1use crate::SheetId;
2
3mod criteria;
4mod exact;
5mod family;
6mod kernels;
7mod lift;
8mod memo;
9use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
10#[cfg(test)]
11use crate::engine::Scheduler;
12use crate::engine::arena::AstNodeId;
13use crate::engine::eval_delta::{
14 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
15};
16use crate::engine::graph::editor::change_log::MutationCapture;
17use crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan;
18use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
19use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
20use crate::engine::live_graph::analyze_live_graph;
21use crate::engine::lookup_index_cache::{
22 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
23 LookupIndexKey, estimate_bytes,
24};
25use crate::engine::named_range::{NameScope, NamedDefinition};
26use crate::engine::range_view::RangeView;
27use crate::engine::row_visibility::RowVisibilityState;
28use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
29use crate::engine::target_preparation::{
30 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
31};
32use crate::engine::used_extent::{
33 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
34};
35use crate::engine::virtual_deps::VirtualDepBuilder;
36use family::{LayerUnit, layer_units, unit_members};
37
38#[path = "freshness.rs"]
39mod freshness;
40use crate::engine::template::region::Region;
41use crate::engine::{
42 ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
43 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceRequestStats,
44 FormulaDirtyLeaseOutcome, FormulaIngestBatch, FormulaIngestRecord, FormulaIngestReport,
45 FormulaParseDiagnostic, FormulaParsePolicy, FormulaPlaneMode, ResourceLedger,
46 RowVisibilitySource, ScheduleUnit, VertexId, VertexKind, VisibilityMaskMode,
47};
48use crate::function::FnCaps;
49use crate::interpreter::Interpreter;
50use crate::reference::{CellRef, Coord, RangeRef};
51use crate::traits::FunctionProvider;
52use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
53use formualizer_common::{
54 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
55};
56use formualizer_parse::parser::ReferenceType;
57use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
58use rayon::ThreadPoolBuilder;
59use rustc_hash::{FxHashMap, FxHashSet};
60use std::collections::{BTreeMap, BTreeSet, VecDeque};
61use std::sync::Arc;
62use std::sync::atomic::{AtomicBool, Ordering};
63use std::time::Instant;
64
65type StagedFormulaEntry = (u32, u32, String);
66type StagedSheetParts = (
67 Vec<StagedFormulaEntry>,
68 Option<crate::engine::DeferredFormulaPackage>,
69);
70
71#[derive(Default)]
81pub(crate) struct StagedSheet {
82 entries: Vec<StagedFormulaEntry>,
83 index: FxHashMap<(u32, u32), usize>,
84 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
85}
86
87impl StagedSheet {
88 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
89 package
90 .invalidated
91 .extend(package.families.iter().map(|family| family.source_id));
92 package.invalidated.extend(
93 package
94 .partitioned_families
95 .iter()
96 .map(|family| family.source_id),
97 );
98 }
99
100 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
101 let Some(package) = self.deferred_package.as_mut() else {
102 return;
103 };
104 if package.suppressed.contains(&(row, col)) {
105 return;
108 }
109 if package.source_geometry_complete
110 && package.families.is_empty()
111 && package.partitioned_families.is_empty()
112 {
113 if let Some((row0, col0)) = row.checked_sub(1).zip(col.checked_sub(1))
114 && package
115 .source_coordinates
116 .binary_search(&crate::engine::SourceCoord {
117 row: row0,
118 col: col0,
119 })
120 .is_ok()
121 {
122 package.suppressed.insert((row, col));
123 }
124 return;
125 }
126 let family = package
127 .replay
128 .lock()
129 .ok()
130 .and_then(|mut replay| replay.formula_at(row, col).ok())
131 .flatten()
132 .and_then(|record| record.partition_owner.or(record.family));
133 if let Some(family) = family {
134 package.invalidated.insert(family);
135 } else {
136 Self::invalidate_all_deferred_families(package);
140 }
141 package.suppressed.insert((row, col));
142 }
143
144 fn reconcile_attached_deferred_package(&mut self) {
145 let Some(package) = self.deferred_package.as_mut() else {
146 return;
147 };
148 if self.entries.is_empty() {
149 return;
150 }
151
152 let replayed = (|| {
153 let mut disposition = crate::engine::FormulaReplayDisposition::default();
154 for partition in &package.partitioned_families {
155 disposition.register_partition(partition, false)?;
156 }
157 package
158 .replay
159 .lock()
160 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
161 .replay_partitioned(&disposition, &package.partitioned_families)
162 })();
163
164 let Ok(records) = replayed else {
165 Self::invalidate_all_deferred_families(package);
166 package
167 .suppressed
168 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
169 return;
170 };
171 let mut owners = FxHashMap::default();
172 for record in &records {
173 owners
174 .entry((record.row, record.col))
175 .or_insert(record.partition_owner.or(record.family));
176 }
177 for (row, col, _) in &self.entries {
178 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
179 package.invalidated.insert(family);
180 } else {
181 Self::invalidate_all_deferred_families(package);
182 }
183 package.suppressed.insert((*row, *col));
184 }
185 package.reconciliation_replay = Some(records);
186 }
187
188 fn stage(&mut self, row: u32, col: u32, text: String) {
189 self.invalidate_deferred_at(row, col);
190 match self.index.entry((row, col)) {
191 std::collections::hash_map::Entry::Occupied(slot) => {
192 self.entries[*slot.get()].2 = text;
193 }
194 std::collections::hash_map::Entry::Vacant(slot) => {
195 slot.insert(self.entries.len());
196 self.entries.push((row, col, text));
197 }
198 }
199 }
200
201 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
202 let deferred_text = self.get(row, col);
203 self.invalidate_deferred_at(row, col);
204 let Some(idx) = self.index.remove(&(row, col)) else {
205 return deferred_text;
206 };
207 let (_, _, text) = self.entries.remove(idx);
208 for slot in self.index.values_mut() {
210 if *slot > idx {
211 *slot -= 1;
212 }
213 }
214 Some(text)
215 }
216
217 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
218 self.index
219 .get(&(row, col))
220 .map(|&i| self.entries[i].2.as_str())
221 }
222
223 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
224 let idx = self.index.remove(&(row, col))?;
225 let (_, _, text) = self.entries.remove(idx);
226 for slot in self.index.values_mut() {
227 if *slot > idx {
228 *slot -= 1;
229 }
230 }
231 Some(text)
232 }
233
234 fn get(&self, row: u32, col: u32) -> Option<String> {
235 if let Some(text) = self.get_ordinary(row, col) {
236 return Some(text.to_string());
237 }
238 let package = self.deferred_package.as_ref()?;
239 if package.suppressed.contains(&(row, col)) {
240 return None;
241 }
242 package
243 .replay
244 .lock()
245 .ok()?
246 .formula_at(row, col)
247 .ok()?
248 .map(|record| record.text)
249 }
250
251 fn len(&self) -> usize {
252 self.deferred_package
253 .as_ref()
254 .map_or(self.entries.len(), |package| {
255 (if package.source_geometry_complete
256 && (package.coordinates_cover_families
257 || (package.families.is_empty() && package.partitioned_families.is_empty()))
258 {
259 package.source_coordinates.len()
260 } else {
261 usize::try_from(package.report.source_formula_records_spooled)
262 .unwrap_or(usize::MAX)
263 })
264 .saturating_sub(package.suppressed.len())
265 .saturating_add(self.entries.len())
266 })
267 }
268
269 fn is_empty(&self) -> bool {
270 self.entries.is_empty() && self.deferred_package.is_none()
271 }
272
273 fn into_parts(self) -> StagedSheetParts {
274 (self.entries, self.deferred_package)
275 }
276}
277
278type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
279
280#[derive(Clone, Debug, PartialEq, Eq, Hash)]
281struct PreparationRegion {
282 sheet: String,
283 sheet_id: SheetId,
284 start_row: u32,
285 start_col: u32,
286 end_row: u32,
287 end_col: u32,
288}
289
290struct OrderedTargetProducers {
292 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
293 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
294}
295
296impl OrderedTargetProducers {
297 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
298 let mut ordered = Vec::new();
299 ordered.try_reserve(capacity)?;
300 let mut seen = FxHashSet::default();
301 seen.try_reserve(capacity)?;
302 Ok(Self { ordered, seen })
303 }
304
305 fn from_ordered(
306 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
307 ) -> Result<Self, std::collections::TryReserveError> {
308 let mut seen = FxHashSet::default();
309 seen.try_reserve(ordered.len())?;
310 seen.extend(ordered.iter().copied());
311 Ok(Self { ordered, seen })
312 }
313
314 fn push(
315 &mut self,
316 producer: crate::engine::target_preparation::TargetProducer,
317 ) -> Result<bool, std::collections::TryReserveError> {
318 #[cfg(test)]
319 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
320 if self.seen.contains(&producer) {
321 return Ok(false);
322 }
323 self.seen.try_reserve(1)?;
324 self.ordered.try_reserve(1)?;
325 self.seen.insert(producer);
326 self.ordered.push(producer);
327 Ok(true)
328 }
329
330 fn len(&self) -> usize {
331 self.ordered.len()
332 }
333
334 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
335 self.ordered
336 }
337}
338
339#[cfg(test)]
340thread_local! {
341 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
342}
343
344fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
345 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
346 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
347 limit: u64::MAX,
348 observed: observed as u64,
349 request_id,
350 })
351 .into_excel_error()
352}
353
354#[derive(Debug)]
355struct PreparedOrdinaryStagedFormula {
356 sheet: String,
357 sheet_id: SheetId,
358 lease: StagedFormulaLease,
359 ast_id: Option<AstNodeId>,
360 plan: Option<DependencyPlanRow>,
361}
362
363struct PreparedTargetSourcePackage {
364 sheet: String,
365 sheet_id: SheetId,
366 lease: StagedPackageLease,
367 selected_points: Option<BTreeSet<(u32, u32)>>,
368 complete_selections: BTreeSet<crate::engine::SourceFamilyId>,
369 deferred_shared: bool,
370 direct_domains: Vec<(
371 crate::engine::SourceFamilyId,
372 crate::engine::PlacementDomainTransport,
373 )>,
374 source_report: crate::engine::FormulaCompressedSourceReport,
375 replay_records: Vec<crate::engine::DeferredReplayFormula>,
376 spool_replays: u64,
377 disposition: crate::engine::FormulaReplayDisposition,
378 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
379 direct_families: usize,
380 direct_cells: u64,
381 direct_fragments: u64,
382 direct_complete_families: u64,
383 direct_complete_cells: u64,
384 direct_partition_families: u64,
385 direct_partition_cells: u64,
386 anchor_parses: u64,
387 anchor_asts: u64,
388 anchor_analyses: u64,
389}
390
391impl PreparedTargetSourcePackage {
392 fn empty_selection(sheet: &str, sheet_id: SheetId, lease: StagedPackageLease) -> Self {
393 Self {
394 sheet: sheet.to_owned(),
395 sheet_id,
396 lease,
397 selected_points: Some(BTreeSet::new()),
398 complete_selections: Default::default(),
399 deferred_shared: false,
400 direct_domains: Vec::new(),
401 source_report: Default::default(),
402 replay_records: Vec::new(),
403 spool_replays: 0,
404 disposition: Default::default(),
405 legacy: Vec::new(),
406 direct_families: 0,
407 direct_cells: 0,
408 direct_fragments: 0,
409 direct_complete_families: 0,
410 direct_complete_cells: 0,
411 direct_partition_families: 0,
412 direct_partition_cells: 0,
413 anchor_parses: 0,
414 anchor_asts: 0,
415 anchor_analyses: 0,
416 }
417 }
418
419 fn direct_contains(&self, row: u32, col: u32) -> bool {
420 self.direct_domains.iter().any(|(_, domain)| {
421 let rect = domain.rect();
422 row > rect.start.row
423 && row <= rect.end.row + 1
424 && col > rect.start.col
425 && col <= rect.end.col + 1
426 })
427 }
428
429 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
430 self.replay_records.iter().filter(|record| {
431 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
432 return true;
433 };
434 let coord = crate::engine::SourceCoord { row, col };
435 let disposition = match record.family {
436 Some(family) => self.disposition.shared_disposition(family, coord),
437 None => self.disposition.ordinary_disposition(coord).0,
438 };
439 !matches!(
440 disposition,
441 crate::engine::FormulaReplayCoordinateDisposition::Direct
442 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
443 )
444 })
445 }
446}
447
448type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
449type StagedFormulaBatches = Vec<(String, StagedSheet)>;
450
451type CompressedReplayBatch = (
452 FormulaIngestBatch,
453 crate::engine::FormulaCompressedSourceBatch,
454);
455type PreparedSourceBatch = (
456 FormulaIngestBatch,
457 crate::engine::FormulaCompressedSourceReport,
458 crate::engine::FormulaCompressedPreparation,
459);
460type PreparedStagedFormulaBatches = (
461 PreparedFormulaBatches,
462 Vec<CompressedReplayBatch>,
463 Vec<PreparedSourceBatch>,
464 FxHashSet<crate::engine::arena::AstNodeId>,
466);
467
468#[doc(hidden)]
471pub struct SourceFormulaIngress<'a, R> {
472 engine: &'a mut Engine<R>,
473}
474
475impl<R> SourceFormulaIngress<'_, R>
476where
477 R: EvaluationContext,
478{
479 pub fn prepare_families(
480 &mut self,
481 sheet_name: &str,
482 families: &[crate::engine::SourceFormulaFamily],
483 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
484 self.engine.observe_function_semantic_epoch()?;
485 Ok(self
486 .engine
487 .prepare_source_formula_families(sheet_name, families))
488 }
489
490 pub fn prepare_eager_proposals(
491 &mut self,
492 sheet_name: &str,
493 families: &[crate::engine::SourceFormulaFamily],
494 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
495 formula_record_count: u64,
496 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
497 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
498 self.engine.observe_function_semantic_epoch()?;
499 let replay = Arc::new(std::sync::Mutex::new(replay));
500 self.engine.prepare_source_formula_proposals(
501 sheet_name,
502 families,
503 partitions,
504 partitions,
505 formula_record_count,
506 replay,
507 &BTreeSet::new(),
508 &Default::default(),
509 None,
510 )
511 }
512
513 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
514 self.engine.stage_deferred_formula_package(package);
515 }
516
517 pub fn ingest_replay_batches(
518 &mut self,
519 batches: Vec<(
520 FormulaIngestBatch,
521 crate::engine::FormulaCompressedSourceBatch,
522 )>,
523 ) -> Result<FormulaIngestReport, ExcelError> {
524 self.engine
525 .ingest_compressed_formula_source_batches(batches)
526 }
527
528 pub fn finish_prepared(
529 &mut self,
530 batches: Vec<(
531 FormulaIngestBatch,
532 crate::engine::FormulaCompressedSourceReport,
533 crate::engine::FormulaCompressedPreparation,
534 )>,
535 ) -> Result<FormulaIngestReport, ExcelError> {
536 self.engine.finish_compressed_formula_sources(batches)
537 }
538}
539
540const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
544
545#[cfg(debug_assertions)]
547fn same_value_bits(a: &LiteralValue, b: &LiteralValue) -> bool {
548 match (a, b) {
549 (LiteralValue::Number(x), LiteralValue::Number(y)) => x.to_bits() == y.to_bits(),
550 _ => a == b,
551 }
552}
553
554fn buffer_layer_writes(layer: &crate::engine::scheduler::Layer) -> bool {
557 !layer.sequential && layer.vertices.len() >= COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH
558}
559const PARALLEL_SCHEDULE_MIN_CANDIDATES: usize = 16 * 1024;
567const BASE_SCHEDULE_MIN_VERTICES: usize = 64;
571const BASE_SCHEDULE_RATIO: usize = 256;
572const BASE_SCHEDULE_MIN_REQUEST: usize = 128;
577const PARALLEL_LAYER_PROBE: std::time::Duration = std::time::Duration::from_micros(300);
578const PARALLEL_LAYER_WORTH: std::time::Duration = std::time::Duration::from_micros(150);
579const EXPENSIVE_VERTEX_NS: u128 = 10_000;
581const UNIT_CANCELLED: &str = "Evaluation cancelled; the evaluated unit was not committed";
583const GROUP_CANCELLED: &str =
584 "Parallel evaluation cancelled; the evaluated group was not committed";
585const SCC_CANCELLED: &str = "Evaluation cancelled; the evaluated cycle was not completed";
586
587#[derive(Debug, Clone, PartialEq)]
588pub(crate) enum ComputedWrite {
589 Cell {
590 seq: u64,
591 sheet_id: SheetId,
592 row0: u32,
593 col0: u32,
594 value: OverlayValue,
595 format_id: Option<crate::format::FormatId>,
596 },
597 Rect {
598 seq: u64,
599 sheet_id: SheetId,
600 sr0: u32,
601 sc0: u32,
602 values: Vec<Vec<OverlayValue>>,
603 },
604 Run {
607 seq: u64,
608 sheet_id: SheetId,
609 row0: u32,
610 col0: u32,
611 entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
612 },
613}
614
615impl ComputedWrite {
616 #[inline]
617 pub(crate) fn seq(&self) -> u64 {
618 match self {
619 ComputedWrite::Cell { seq, .. }
620 | ComputedWrite::Rect { seq, .. }
621 | ComputedWrite::Run { seq, .. } => *seq,
622 }
623 }
624}
625
626#[derive(Debug, Default)]
627pub(crate) struct ComputedWriteBuffer {
628 writes: Vec<ComputedWrite>,
629 next_seq: u64,
630 estimated_bytes: usize,
631 formats_present: bool,
632}
633
634impl ComputedWriteBuffer {
635 const ENTRY_BASE_BYTES: usize = 32;
636
637 #[inline]
638 pub(crate) fn is_empty(&self) -> bool {
639 self.writes.is_empty()
640 }
641
642 #[inline]
643 pub(crate) fn len(&self) -> usize {
644 self.writes.len()
645 }
646
647 #[inline]
648 pub(crate) fn writes(&self) -> &[ComputedWrite] {
649 &self.writes
650 }
651
652 #[inline]
653 pub(crate) fn estimated_bytes(&self) -> usize {
654 self.estimated_bytes
655 }
656
657 #[cfg(feature = "tracing")]
658 fn traced_cell_count(&self) -> usize {
659 self.writes
660 .iter()
661 .map(|write| match write {
662 ComputedWrite::Cell { .. } => 1,
663 ComputedWrite::Rect { values, .. } => values.iter().map(Vec::len).sum(),
664 ComputedWrite::Run { entries, .. } => entries.len(),
665 })
666 .sum()
667 }
668
669 pub(crate) fn push_cell(
670 &mut self,
671 sheet_id: SheetId,
672 row0: u32,
673 col0: u32,
674 value: OverlayValue,
675 ) {
676 self.push_cell_with_format(sheet_id, row0, col0, value, None);
677 }
678
679 pub(crate) fn push_cell_with_format(
680 &mut self,
681 sheet_id: SheetId,
682 row0: u32,
683 col0: u32,
684 value: OverlayValue,
685 format_id: Option<crate::format::FormatId>,
686 ) {
687 let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
688 self.formats_present |= format_id.is_some();
689 let seq = self.next_sequence();
690 self.estimated_bytes = self
691 .estimated_bytes
692 .saturating_add(Self::estimate_value_bytes(&value));
693 self.writes.push(ComputedWrite::Cell {
694 seq,
695 sheet_id,
696 row0,
697 col0,
698 value,
699 format_id,
700 });
701 }
702
703 pub(crate) fn push_column_run(
704 &mut self,
705 sheet_id: SheetId,
706 row0: u32,
707 col0: u32,
708 mut entries: Vec<(OverlayValue, Option<crate::format::FormatId>)>,
709 ) {
710 let seq = self.next_sequence();
711 let mut added = 0usize;
712 for (value, format_id) in entries.iter_mut() {
713 *format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
714 self.formats_present |= format_id.is_some();
715 added = added.saturating_add(Self::estimate_value_bytes(value));
716 }
717 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
718 self.writes.push(ComputedWrite::Run {
719 seq,
720 sheet_id,
721 row0,
722 col0,
723 entries,
724 });
725 }
726
727 pub(crate) fn push_rect(
728 &mut self,
729 sheet_id: SheetId,
730 sr0: u32,
731 sc0: u32,
732 values: Vec<Vec<OverlayValue>>,
733 ) {
734 let seq = self.next_sequence();
735 let added = values
736 .iter()
737 .flat_map(|row| row.iter())
738 .map(Self::estimate_value_bytes)
739 .fold(0usize, usize::saturating_add);
740 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
741 self.writes.push(ComputedWrite::Rect {
742 seq,
743 sheet_id,
744 sr0,
745 sc0,
746 values,
747 });
748 }
749
750 pub(crate) fn clear(&mut self) {
751 self.writes.clear();
752 self.estimated_bytes = 0;
753 self.formats_present = false;
754 }
755
756 fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
757 self.estimated_bytes = 0;
758 let formats_present = std::mem::take(&mut self.formats_present);
759 (std::mem::take(&mut self.writes), formats_present)
760 }
761
762 fn next_sequence(&mut self) -> u64 {
763 let seq = self.next_seq;
764 self.next_seq = self.next_seq.wrapping_add(1);
765 seq
766 }
767
768 #[inline]
769 fn estimate_value_bytes(value: &OverlayValue) -> usize {
770 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
771 }
772}
773
774#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
775struct ComputedWriteChunkKey {
776 sheet_id: SheetId,
777 col0: u32,
778 chunk_idx: usize,
779 chunk_start_row0: u32,
780}
781
782#[derive(Debug, Clone, PartialEq)]
783pub(crate) struct ComputedWriteChunkEntryPlan {
784 pub(crate) row_in_chunk: usize,
785 pub(crate) seq: u64,
786 pub(crate) value: OverlayValue,
787 pub(crate) format_id: Option<crate::format::FormatId>,
788}
789
790#[derive(Debug, Clone, PartialEq, Eq)]
791pub(crate) enum ComputedWriteChunkPlanShape {
792 Point,
793 SparseOffsets {
794 entries: usize,
795 span_len: usize,
796 },
797 DenseRange {
798 start: usize,
799 len: usize,
800 },
801 RunRange {
802 start: usize,
803 len: usize,
804 runs: usize,
805 },
806}
807
808#[derive(Debug, Clone, PartialEq, Eq)]
809pub(crate) enum ComputedWriteFormatClear {
810 Range { start: usize, end: usize },
811 Offsets(Vec<usize>),
812}
813
814#[derive(Debug, Clone, PartialEq)]
815pub(crate) enum ComputedWriteChunkFormatEffect {
816 NoFormatWork,
817 ClearStale(ComputedWriteFormatClear),
818 SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
819}
820
821#[derive(Debug, Clone, PartialEq)]
822pub(crate) struct ComputedWriteChunkPlan {
823 pub(crate) sheet_id: SheetId,
824 pub(crate) col0: u32,
825 pub(crate) chunk_idx: usize,
826 pub(crate) chunk_start_row0: u32,
827 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
828 pub(crate) shape: ComputedWriteChunkPlanShape,
829 pub(crate) format_effect: ComputedWriteChunkFormatEffect,
830}
831
832#[derive(Debug, Clone, Default, PartialEq)]
833pub(crate) struct ComputedWriteCoalescingPlan {
834 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
835 pub(crate) input_cells: usize,
836 pub(crate) coalesced_cells: usize,
837 pub(crate) overwritten_cells: usize,
838}
839
840impl ComputedWriteCoalescingPlan {
841 #[inline]
842 pub(crate) fn is_empty(&self) -> bool {
843 self.chunks.is_empty()
844 }
845}
846
847impl ComputedWriteChunkPlan {
848 fn from_group(
849 key: ComputedWriteChunkKey,
850 mut entries: Vec<ComputedWriteChunkEntryPlan>,
851 formats_present: bool,
852 computed_lane_has_formats: bool,
853 ) -> (Self, usize) {
854 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
855 let input_len = entries.len();
856 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
857 for entry in entries {
858 if let Some(prev) = coalesced.last_mut()
859 && prev.row_in_chunk == entry.row_in_chunk
860 {
861 *prev = entry;
862 continue;
863 }
864 coalesced.push(entry);
865 }
866 let overwritten = input_len.saturating_sub(coalesced.len());
867 let shape = Self::classify_shape(&coalesced);
868 let format_effect = Self::classify_format_effect(
869 &coalesced,
870 &shape,
871 formats_present,
872 computed_lane_has_formats,
873 );
874 (
875 Self {
876 sheet_id: key.sheet_id,
877 col0: key.col0,
878 chunk_idx: key.chunk_idx,
879 chunk_start_row0: key.chunk_start_row0,
880 entries: coalesced,
881 shape,
882 format_effect,
883 },
884 overwritten,
885 )
886 }
887
888 fn classify_format_effect(
889 entries: &[ComputedWriteChunkEntryPlan],
890 shape: &ComputedWriteChunkPlanShape,
891 formats_present: bool,
892 computed_lane_has_formats: bool,
893 ) -> ComputedWriteChunkFormatEffect {
894 if !formats_present {
895 return if computed_lane_has_formats {
896 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
897 } else {
898 ComputedWriteChunkFormatEffect::NoFormatWork
899 };
900 }
901
902 if entries.iter().all(|entry| entry.format_id.is_none()) {
903 return if computed_lane_has_formats {
904 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
905 } else {
906 ComputedWriteChunkFormatEffect::NoFormatWork
907 };
908 }
909
910 ComputedWriteChunkFormatEffect::SetExplicit(
911 entries
912 .iter()
913 .map(|entry| (entry.row_in_chunk, entry.format_id))
914 .collect(),
915 )
916 }
917
918 fn format_clear_spec(
919 entries: &[ComputedWriteChunkEntryPlan],
920 shape: &ComputedWriteChunkPlanShape,
921 ) -> ComputedWriteFormatClear {
922 match shape {
923 ComputedWriteChunkPlanShape::DenseRange { start, len }
924 | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
925 ComputedWriteFormatClear::Range {
926 start: *start,
927 end: start.saturating_add(*len),
928 }
929 }
930 ComputedWriteChunkPlanShape::Point
931 | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
932 ComputedWriteFormatClear::Offsets(
933 entries.iter().map(|entry| entry.row_in_chunk).collect(),
934 )
935 }
936 }
937 }
938
939 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
940 debug_assert!(!entries.is_empty());
941 if entries.len() == 1 {
942 return ComputedWriteChunkPlanShape::Point;
943 }
944
945 let start = entries[0].row_in_chunk;
946 let end = entries[entries.len() - 1].row_in_chunk;
947 let span_len = end.saturating_sub(start).saturating_add(1);
948 if span_len != entries.len() {
949 return ComputedWriteChunkPlanShape::SparseOffsets {
950 entries: entries.len(),
951 span_len,
952 };
953 }
954
955 let runs = Self::run_count(entries);
956 if runs < entries.len() {
957 ComputedWriteChunkPlanShape::RunRange {
958 start,
959 len: entries.len(),
960 runs,
961 }
962 } else {
963 ComputedWriteChunkPlanShape::DenseRange {
964 start,
965 len: entries.len(),
966 }
967 }
968 }
969
970 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
971 let mut runs = 0usize;
972 let mut prev: Option<&OverlayValue> = None;
973 for entry in entries {
974 if prev != Some(&entry.value) {
975 runs = runs.saturating_add(1);
976 prev = Some(&entry.value);
977 }
978 }
979 runs
980 }
981}
982
983#[cfg(feature = "tracing")]
984#[derive(Default)]
985struct TraceEvaluationCounters {
986 computed_vertices: usize,
987 cycles: usize,
988}
989
990pub struct Engine<R> {
991 pub(crate) graph: DependencyGraph,
992 resolver: R,
993 pub config: EvalConfig,
994 workbook_load_limits: crate::engine::WorkbookLoadLimits,
995 clock: crate::timezone::SnapshotClock,
1000 thread_pool: Option<Arc<rayon::ThreadPool>>,
1001 pub recalc_epoch: u64,
1002 snapshot_id: std::sync::atomic::AtomicU64,
1003 topology_epoch: u64,
1004 cached_static_schedule: Option<CachedScheduleEntry>,
1005 recent_schedules: Vec<CachedScheduleEntry>,
1010 base_schedule: Option<CachedScheduleEntry>,
1015 #[cfg(any(test, feature = "benchmark_internal"))]
1016 recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1017 spill_mgr: ShimSpillManager,
1018 arrow_sheets: SheetStore,
1020 format_registry: crate::format::FormatRegistry,
1022 derived_formats: crate::engine::derived_formats::DerivedFormats,
1024 #[cfg(test)]
1025 derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1026 #[cfg(test)]
1027 family_members_for_test: std::sync::atomic::AtomicU64,
1028 #[cfg(test)]
1029 invariant_bound_members_for_test: std::sync::atomic::AtomicU64,
1030 #[cfg(test)]
1031 lifted_members_for_test: std::sync::atomic::AtomicU64,
1032 #[cfg(test)]
1033 chained_members_for_test: std::sync::atomic::AtomicU64,
1034 #[cfg(test)]
1035 lane_clean_reads_for_test: std::sync::atomic::AtomicU64,
1036 #[cfg(test)]
1037 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64,
1038 #[cfg(test)]
1039 memo_hits_for_test: std::sync::atomic::AtomicU64,
1040 compressed_at_build: Option<u64>,
1042 #[cfg(test)]
1043 computed_overlay_set_explicit_entry_operations_for_test: u64,
1044 #[cfg(test)]
1045 computed_overlay_stale_clear_range_effects_for_test: u64,
1046 #[cfg(test)]
1047 computed_overlay_stale_clear_offset_attempts_for_test: u64,
1048 #[cfg(test)]
1049 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1050 has_edited: bool,
1052 overlay_compactions: u64,
1054
1055 computed_overlay_bytes_estimate: usize,
1057 computed_overlay_mirroring_disabled: bool,
1058 pub(crate) force_materialize_range_views: bool,
1061 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1063 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1065 lookup_index_cache: LookupIndexCache,
1066 source_cache: Arc<std::sync::RwLock<SourceCache>>,
1067 source_formula_token: Arc<()>,
1069 recalc_plan_token: Arc<()>,
1071 staged_formulas: StagedFormulaMap,
1073 staged_formula_index: StagedFormulaIndex,
1075 blocked_pending_spills: Vec<(VertexId, CellRef, Region)>,
1077 spill_blocker_waiters: SpillBlockerWaiters,
1081 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1083 row_visibility_mask_cache: std::sync::RwLock<
1085 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1086 >,
1087 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1089 last_formula_ingest_report: Option<FormulaIngestReport>,
1091 formula_ingest_report_total: FormulaIngestReport,
1093 active_cancel_flag: Option<crate::engine::CancelToken>,
1095 active_evaluation_deadline: Option<Instant>,
1097
1098 action_depth: u32,
1103
1104 last_virtual_dep_telemetry: VirtualDepTelemetry,
1106 virtual_dep_fallback_activations: u64,
1107
1108 last_cycle_telemetry: CycleTelemetry,
1110
1111 next_evaluation_resource_request_id: u64,
1113 evaluation_resource_request_depth: usize,
1114 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1115 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1116 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1117 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1118 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1119 evaluation_resource_config_diagnostic:
1120 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1121 active_resource_ledger: Option<ResourceLedger>,
1122 source_cache_footprints: Vec<std::sync::Weak<std::sync::atomic::AtomicU64>>,
1123 source_cache_accounted: u64,
1124
1125 pending_iterative_redirty: Vec<VertexId>,
1144 snapshot_evaluation_active: bool,
1146 snapshot_volatile_redirty_pending: bool,
1147 retained_scc_members: FxHashMap<VertexId, u64>,
1165 next_retained_scc_id: u64,
1166 retained_scc_config_fingerprint: u64,
1173 retained_scc_function_epoch_seen: u64,
1178 retained_scc_provider_revision_seen: Option<u64>,
1179 retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1186
1187 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1200
1201 function_semantic_epoch_seen: u64,
1203 function_provider_revision_seen: Option<u64>,
1205
1206 #[cfg(feature = "tracing")]
1207 trace_evaluation_counters: TraceEvaluationCounters,
1208 #[cfg(test)]
1209 evaluation_request_begin_count_for_test: u64,
1210 #[cfg(any(test, feature = "test-support"))]
1211 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1212 #[cfg(test)]
1213 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1214 #[cfg(test)]
1215 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1216 #[cfg(test)]
1217 inject_target_semantic_stale_once_for_test: bool,
1218 #[cfg(test)]
1219 force_virtual_dep_changes_remaining_for_test: usize,
1220 freshness: freshness::Freshness,
1222 #[cfg(test)]
1223 fail_evaluation_commit_preflight_once_for_test: bool,
1224 #[cfg(test)]
1225 target_preparation_fault_for_test:
1226 Option<crate::engine::target_preparation::TargetPreparationFault>,
1227 #[cfg(test)]
1228 force_non_cycle_schedule_fallback_for_test: bool,
1229 #[cfg(test)]
1230 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1231 #[cfg(test)]
1232 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1233}
1234
1235impl<R: EvaluationContext> Engine<R> {
1238 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1239 self.graph.ingest_pipeline(&self.resolver)
1240 }
1241}
1242
1243pub struct EngineAction<'a, R>
1244where
1245 R: EvaluationContext,
1246{
1247 engine: &'a mut Engine<R>,
1248 name: String,
1249 capture: Option<*mut MutationCapture>,
1252 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1255 atomic_policy: bool,
1257}
1258
1259impl<'a, R> EngineAction<'a, R>
1260where
1261 R: EvaluationContext,
1262{
1263 #[inline]
1264 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1265 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1266 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1267 crate::reference::CellRef::new(sheet_id, coord)
1268 }
1269
1270 #[inline]
1271 pub fn name(&self) -> &str {
1272 &self.name
1273 }
1274
1275 #[inline]
1276 pub fn set_cell_value(
1277 &mut self,
1278 sheet: &str,
1279 row: u32,
1280 col: u32,
1281 value: LiteralValue,
1282 ) -> Result<(), crate::engine::EditorError> {
1283 if self.capture.is_some() {
1284 let old_value = self.engine.read_cell_value(sheet, row, col);
1285 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1286 let addr = self.addr_for(sheet, row, col);
1287 let Some(capture_ptr) = self.capture else {
1288 return Err(crate::engine::EditorError::TransactionFailed {
1289 reason: "action_with_logger: missing mutation capture".to_string(),
1290 });
1291 };
1292
1293 let old_comp = if self.arrow_undo.is_some() {
1296 self.engine.read_computed_overlay_cell(sheet, row, col)
1297 } else {
1298 None
1299 };
1300
1301 if self.engine.graph_admission_enabled() {
1302 let admission =
1303 self.engine
1304 .graph
1305 .preview_value_mutation(addr.sheet_id, row, col)?;
1306 self.engine.preflight_graph_admission(admission)?;
1307 }
1308 if old_formula.is_none() {
1309 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1310 }
1311
1312 let delta_old_sem = if old_formula.is_some() {
1313 None
1314 } else {
1315 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1316 };
1317
1318 let start_len = unsafe { (&*capture_ptr).len() };
1319
1320 let capture = unsafe { &mut *capture_ptr };
1322 self.engine.edit_with_capture(capture, |editor| {
1323 editor.set_cell_value_with_old_state(
1324 addr,
1325 value.clone(),
1326 old_value.clone(),
1327 old_formula.clone(),
1328 );
1329 })?;
1330 self.engine.record_structural_change(StructuralScope::Cell {
1331 sheet: addr.sheet_id,
1332 row: addr.coord.row(),
1333 col: addr.coord.col(),
1334 });
1335
1336 if let Some(undo_ptr) = self.arrow_undo {
1337 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1339 let undo = unsafe { &mut *undo_ptr };
1340 self.engine
1341 .record_spill_ops_into_arrow_undo(undo, new_events);
1342
1343 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1345 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1346 let row0 = row.saturating_sub(1);
1347 let col0 = col.saturating_sub(1);
1348 let delta_new_sem = Some(value.clone());
1349 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1350 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1351 }
1352 Ok(())
1353 } else {
1354 self.engine
1355 .set_cell_value(sheet, row, col, value)
1356 .map_err(crate::engine::EditorError::from)
1357 }
1358 }
1359
1360 #[inline]
1361 pub fn set_cell_formula(
1362 &mut self,
1363 sheet: &str,
1364 row: u32,
1365 col: u32,
1366 ast: ASTNode,
1367 ) -> Result<(), crate::engine::EditorError> {
1368 if self.capture.is_some() {
1369 let old_value = self.engine.read_cell_value(sheet, row, col);
1370 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1371 let addr = self.addr_for(sheet, row, col);
1372 let Some(capture_ptr) = self.capture else {
1373 return Err(crate::engine::EditorError::TransactionFailed {
1374 reason: "action_with_logger: missing mutation capture".to_string(),
1375 });
1376 };
1377
1378 let admitted_formula = if self.engine.graph_admission_enabled() {
1379 let placement =
1380 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1381 let ingested = self.engine.ingest_pipeline().ingest_formula(
1382 FormulaAstInput::Tree(ast.clone()),
1383 placement,
1384 None,
1385 )?;
1386 let admission = self.engine.graph.preview_formula_mutations(&[(
1387 addr.sheet_id,
1388 row,
1389 col,
1390 ingested.dep_plan.clone(),
1391 )])?;
1392 self.engine.preflight_graph_admission(admission)?;
1393 Some((ingested.ast_id, ingested.dep_plan))
1394 } else {
1395 None
1396 };
1397 if old_formula.is_none() {
1398 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1399 }
1400 let delta_old = if self.arrow_undo.is_some() {
1401 if old_formula.is_some() {
1402 None
1403 } else {
1404 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1405 }
1406 } else {
1407 None
1408 };
1409 let start_len = unsafe { (&*capture_ptr).len() };
1410
1411 let capture = unsafe { &mut *capture_ptr };
1413 self.engine.edit_with_capture(capture, |editor| {
1414 if let Some((ast_id, plan)) = admitted_formula {
1415 editor.set_cell_formula_with_prepared_plan(
1416 addr,
1417 ast.clone(),
1418 old_value,
1419 old_formula,
1420 ast_id,
1421 plan,
1422 );
1423 } else {
1424 editor.set_cell_formula_with_old_state(
1425 addr,
1426 ast.clone(),
1427 old_value,
1428 old_formula,
1429 );
1430 }
1431 })?;
1432 self.engine.record_structural_change(StructuralScope::Cell {
1433 sheet: addr.sheet_id,
1434 row: addr.coord.row(),
1435 col: addr.coord.col(),
1436 });
1437
1438 if let Some(undo_ptr) = self.arrow_undo {
1439 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1440 let undo = unsafe { &mut *undo_ptr };
1441 self.engine
1442 .record_spill_ops_into_arrow_undo(undo, new_events);
1443 let delta_new: Option<LiteralValue> = None;
1444 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1445 let row0 = row.saturating_sub(1);
1446 let col0 = col.saturating_sub(1);
1447 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1448 }
1449 Ok(())
1450 } else {
1451 self.engine
1452 .set_cell_formula(sheet, row, col, ast)
1453 .map_err(crate::engine::EditorError::from)
1454 }
1455 }
1456
1457 #[inline]
1458 pub fn set_row_hidden(
1459 &mut self,
1460 sheet: &str,
1461 row_1based: u32,
1462 hidden: bool,
1463 source: RowVisibilitySource,
1464 ) -> Result<(), crate::engine::EditorError> {
1465 if self.capture.is_some() {
1466 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1467 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1468 let old_hidden = self
1469 .engine
1470 .row_visibility
1471 .get(&sheet_id)
1472 .map(|state| state.is_row_hidden(row0, Some(source)))
1473 .unwrap_or(false);
1474 if old_hidden == hidden {
1475 return Ok(());
1476 }
1477
1478 let _ = self
1479 .engine
1480 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1481
1482 let Some(capture_ptr) = self.capture else {
1483 return Err(crate::engine::EditorError::TransactionFailed {
1484 reason: "action_with_logger: missing mutation capture".to_string(),
1485 });
1486 };
1487 unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1488 sheet_id,
1489 row0,
1490 source,
1491 old_hidden,
1492 new_hidden: hidden,
1493 });
1494
1495 Ok(())
1496 } else {
1497 self.engine
1498 .set_row_hidden(sheet, row_1based, hidden, source)
1499 }
1500 }
1501
1502 #[inline]
1503 pub fn set_rows_hidden(
1504 &mut self,
1505 sheet: &str,
1506 start_row_1based: u32,
1507 end_row_1based: u32,
1508 hidden: bool,
1509 source: RowVisibilitySource,
1510 ) -> Result<(), crate::engine::EditorError> {
1511 if self.capture.is_some() {
1512 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1513 let (start_row0, end_row0) =
1514 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1515
1516 let Some(capture_ptr) = self.capture else {
1517 return Err(crate::engine::EditorError::TransactionFailed {
1518 reason: "action_with_logger: missing mutation capture".to_string(),
1519 });
1520 };
1521 let capture = unsafe { &mut *capture_ptr };
1522
1523 for row0 in start_row0..=end_row0 {
1524 let old_hidden = self
1525 .engine
1526 .row_visibility
1527 .get(&sheet_id)
1528 .map(|state| state.is_row_hidden(row0, Some(source)))
1529 .unwrap_or(false);
1530 if old_hidden == hidden {
1531 continue;
1532 }
1533
1534 let _ = self
1535 .engine
1536 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1537
1538 capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1539 sheet_id,
1540 row0,
1541 source,
1542 old_hidden,
1543 new_hidden: hidden,
1544 });
1545 }
1546
1547 Ok(())
1548 } else {
1549 self.engine
1550 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1551 }
1552 }
1553
1554 #[inline]
1555 pub fn insert_rows(
1556 &mut self,
1557 sheet: &str,
1558 before: u32,
1559 count: u32,
1560 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1561 if count == 0 {
1562 return Ok(crate::engine::ShiftSummary::default());
1563 }
1564 if self.capture.is_some() {
1565 let Some(capture_ptr) = self.capture else {
1566 return Err(crate::engine::EditorError::TransactionFailed {
1567 reason: "action_atomic: missing mutation capture".to_string(),
1568 });
1569 };
1570
1571 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1572 let before0 = before.saturating_sub(1);
1573 let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1574 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1575
1576 let summary = {
1578 let capture = unsafe { &mut *capture_ptr };
1579 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1580 Ok(crate::engine::ShiftSummary::default());
1581 self.engine.edit_with_capture(capture, |editor| {
1582 editor.set_structural_occupancy(occupancy);
1583 out = editor.insert_rows(sheet_id, before0, count);
1584 })?;
1585 out?
1586 };
1587
1588 self.engine.ensure_arrow_sheet(sheet);
1590 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1591 asheet.insert_rows(before0 as usize, count as usize);
1592 }
1593 self.engine
1594 .purge_derived_formats_after_row(sheet_id, before0);
1595 self.engine
1596 .shift_row_visibility_insert(sheet_id, before0, count);
1597 self.engine.mark_moved_formula_vertices_dirty(&summary);
1598 self.engine
1599 .clear_computed_overlay_after_row(sheet, before0 as usize);
1600 self.engine
1601 .record_structural_change(StructuralScope::Region(affected_region));
1602 if let Some(undo_ptr) = self.arrow_undo {
1603 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1604 }
1605 Ok(summary)
1606 } else {
1607 self.engine.insert_rows(sheet, before, count)
1608 }
1609 }
1610
1611 #[inline]
1612 pub fn delete_rows(
1613 &mut self,
1614 sheet: &str,
1615 start: u32,
1616 count: u32,
1617 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1618 if count == 0 {
1619 return Ok(crate::engine::ShiftSummary::default());
1620 }
1621 if self.atomic_policy {
1622 return Err(crate::engine::EditorError::TransactionUnsupported {
1623 reason:
1624 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1625 .to_string(),
1626 });
1627 }
1628 self.engine.delete_rows(sheet, start, count)
1629 }
1630
1631 #[inline]
1632 pub fn insert_columns(
1633 &mut self,
1634 sheet: &str,
1635 before: u32,
1636 count: u32,
1637 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1638 if count == 0 {
1639 return Ok(crate::engine::ShiftSummary::default());
1640 }
1641 if self.capture.is_some() {
1642 let Some(capture_ptr) = self.capture else {
1643 return Err(crate::engine::EditorError::TransactionFailed {
1644 reason: "action_atomic: missing mutation capture".to_string(),
1645 });
1646 };
1647
1648 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1649 let before0 = before.saturating_sub(1);
1650 let occupancy = self.engine.structural_column_occupancy();
1651 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1652
1653 let summary = {
1654 let capture = unsafe { &mut *capture_ptr };
1655 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1656 Ok(crate::engine::ShiftSummary::default());
1657 self.engine.edit_with_capture(capture, |editor| {
1658 editor.set_structural_occupancy(occupancy);
1659 out = editor.insert_columns(sheet_id, before0, count);
1660 })?;
1661 out?
1662 };
1663
1664 self.engine.ensure_arrow_sheet(sheet);
1665 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1666 asheet.insert_columns(before0 as usize, count as usize);
1667 }
1668 self.engine
1669 .purge_derived_formats_after_col(sheet_id, before0);
1670 self.engine.mark_moved_formula_vertices_dirty(&summary);
1671 self.engine
1672 .clear_computed_overlay_after_col(sheet, before0 as usize);
1673 self.engine
1674 .record_structural_change(StructuralScope::Region(affected_region));
1675 if let Some(undo_ptr) = self.arrow_undo {
1676 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1677 }
1678 Ok(summary)
1679 } else {
1680 self.engine.insert_columns(sheet, before, count)
1681 }
1682 }
1683
1684 #[inline]
1685 pub fn delete_columns(
1686 &mut self,
1687 sheet: &str,
1688 start: u32,
1689 count: u32,
1690 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1691 if count == 0 {
1692 return Ok(crate::engine::ShiftSummary::default());
1693 }
1694 if self.atomic_policy {
1695 return Err(crate::engine::EditorError::TransactionUnsupported {
1696 reason:
1697 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1698 .to_string(),
1699 });
1700 }
1701 self.engine.delete_columns(sheet, start, count)
1702 }
1703
1704 #[inline]
1709 pub fn action<T>(
1710 &mut self,
1711 name: impl AsRef<str>,
1712 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1713 ) -> Result<T, crate::engine::EditorError> {
1714 self.engine.action(name, f)
1715 }
1716}
1717
1718struct ActionDepthGuard<'a, R> {
1719 engine: *mut Engine<R>,
1720 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1721}
1722
1723impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1724 fn drop(&mut self) {
1725 unsafe {
1728 let e = &mut *self.engine;
1729 e.action_depth = e.action_depth.saturating_sub(1);
1730 }
1731 }
1732}
1733
1734#[derive(Default)]
1735struct SourceCache {
1736 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1737 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1738}
1739
1740#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1741struct VisibilityMaskCacheKey {
1742 sheet_id: SheetId,
1743 start_row0: u32,
1744 end_row0: u32,
1745 mode: VisibilityMaskMode,
1746 version: u64,
1747}
1748
1749#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1750enum StructuralScope {
1751 Cell { sheet: SheetId, row: u32, col: u32 },
1752 Region(Region),
1753 Sheet(SheetId),
1754 RemovedSheet(SheetId),
1755 OpaqueGlobal,
1756 AllSheets,
1757}
1758
1759#[derive(Default)]
1762struct SpillBlockerWaiters {
1763 by_cell: FxHashMap<CellRef, Vec<VertexId>>,
1764 by_anchor: FxHashMap<VertexId, CellRef>,
1765}
1766
1767impl SpillBlockerWaiters {
1768 const REGION_PROBE_CELLS: u64 = 4096;
1770
1771 fn is_empty(&self) -> bool {
1772 self.by_anchor.is_empty()
1773 }
1774
1775 fn wait(&mut self, anchor: VertexId, blocker: CellRef) {
1776 match self.by_anchor.insert(anchor, blocker) {
1777 Some(old) if old == blocker => return,
1778 Some(old) => self.unlink(anchor, old),
1779 None => {}
1780 }
1781 self.by_cell.entry(blocker).or_default().push(anchor);
1782 }
1783
1784 fn forget(&mut self, anchor: VertexId) {
1785 if self.by_anchor.is_empty() {
1786 return;
1787 }
1788 if let Some(old) = self.by_anchor.remove(&anchor) {
1789 self.unlink(anchor, old);
1790 }
1791 }
1792
1793 fn unlink(&mut self, anchor: VertexId, cell: CellRef) {
1794 if let Some(waiters) = self.by_cell.get_mut(&cell) {
1795 waiters.retain(|&v| v != anchor);
1796 if waiters.is_empty() {
1797 self.by_cell.remove(&cell);
1798 }
1799 }
1800 }
1801
1802 fn take_affected(&mut self, scope: StructuralScope) -> Vec<VertexId> {
1804 let cells: Vec<CellRef> = match scope {
1805 StructuralScope::Cell { sheet, row, col } => {
1806 let cell = CellRef::new(sheet, Coord::new(row, col, true, true));
1807 if self.by_cell.contains_key(&cell) {
1808 vec![cell]
1809 } else {
1810 Vec::new()
1811 }
1812 }
1813 StructuralScope::Region(region) => {
1814 let (rows, cols) = region.axis_ranges();
1815 let (r0, r1) = rows.query_bounds();
1816 let (c0, c1) = cols.query_bounds();
1817 let area = u64::from(r1.saturating_sub(r0) + 1)
1818 .saturating_mul(u64::from(c1.saturating_sub(c0) + 1));
1819 if area <= Self::REGION_PROBE_CELLS {
1820 let sheet = region.sheet_id();
1821 (r0..=r1)
1822 .flat_map(|r| (c0..=c1).map(move |c| (r, c)))
1823 .map(|(r, c)| CellRef::new(sheet, Coord::new(r, c, true, true)))
1824 .filter(|cell| self.by_cell.contains_key(cell))
1825 .collect()
1826 } else {
1827 self.by_cell
1828 .keys()
1829 .filter(|cell| {
1830 region.intersects(&Region::point(
1831 cell.sheet_id,
1832 cell.coord.row(),
1833 cell.coord.col(),
1834 ))
1835 })
1836 .copied()
1837 .collect()
1838 }
1839 }
1840 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => self
1841 .by_cell
1842 .keys()
1843 .filter(|cell| cell.sheet_id == sheet)
1844 .copied()
1845 .collect(),
1846 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => {
1847 self.by_cell.keys().copied().collect()
1848 }
1849 };
1850 let mut woken = Vec::new();
1851 for cell in cells {
1852 if let Some(waiters) = self.by_cell.remove(&cell) {
1853 for anchor in waiters {
1854 self.by_anchor.remove(&anchor);
1855 woken.push(anchor);
1856 }
1857 }
1858 }
1859 woken
1860 }
1861}
1862
1863#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1864enum LoggedEditImpact {
1865 NoOp,
1866 DataOnly,
1867 Topology,
1868}
1869
1870#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1871enum LoggedEditDirection {
1872 Original,
1873 InverseReplay,
1874 ForwardReplay,
1875}
1876
1877#[derive(Clone, Copy)]
1878struct InvalidationBaseline {
1879 snapshot_id: u64,
1880 topology_epoch: u64,
1881}
1882
1883struct SourceCacheSession {
1884 cache: Arc<std::sync::RwLock<SourceCache>>,
1885}
1886
1887impl Drop for SourceCacheSession {
1888 fn drop(&mut self) {
1889 if let Ok(mut g) = self.cache.write() {
1890 *g = SourceCache::default();
1891 }
1892 }
1893}
1894
1895#[derive(Debug)]
1896#[non_exhaustive]
1897pub struct EvalResult {
1898 pub computed_vertices: usize,
1899 pub cycle_errors: usize,
1900 pub elapsed: std::time::Duration,
1901}
1902
1903#[derive(Clone, Debug, PartialEq, Eq)]
1904#[non_exhaustive]
1905pub struct TableMetadata {
1906 pub name: String,
1907 pub sheet: String,
1908 pub start_row: u32,
1909 pub start_col: u32,
1910 pub end_row: u32,
1911 pub end_col: u32,
1912 pub header_row: bool,
1913 pub headers: Vec<String>,
1914 pub totals_row: bool,
1915}
1916
1917#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1925#[non_exhaustive]
1926pub struct EngineBaselineStats {
1927 pub graph_vertex_count: usize,
1928 pub graph_formula_vertex_count: usize,
1929 pub graph_edge_count: usize,
1930 pub dirty_vertex_count: usize,
1931 pub evaluation_vertex_count: usize,
1932 pub formula_ast_root_count: usize,
1933 pub formula_ast_node_count: usize,
1934 pub staged_formula_count: usize,
1935 pub formula_plane_active_span_count: usize,
1936 pub formula_plane_producer_result_entries: usize,
1937 pub formula_plane_consumer_read_entries: usize,
1938 pub formula_plane_mixed_topology_cache_builds: u64,
1939 pub formula_plane_mixed_topology_cache_hits: u64,
1940 pub formula_plane_mixed_topology_cache_overflows: u64,
1941 pub formula_plane_dirty_pending_events: usize,
1942 pub formula_plane_dirty_region_events_recorded: u64,
1943 pub formula_plane_dirty_span_region_events_recorded: u64,
1944 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1945 pub formula_plane_dirty_global_invalidations: u64,
1946 pub formula_plane_structural_span_candidates: u64,
1947 pub formula_plane_cycle_member_span_demotions: u64,
1948 pub formula_plane_array_result_span_demotions: u64,
1949 pub retained_scc_members: usize,
1952}
1953
1954#[derive(Debug, Clone, Default)]
1955#[non_exhaustive]
1956pub struct VirtualDepTelemetry {
1957 pub candidate_vertices_total: usize,
1958 pub vdeps_vertices_total: usize,
1959 pub vdeps_edges_total: usize,
1960 pub builder_elapsed_ms_total: u128,
1961 pub schedule_virtual_passes: usize,
1962 pub schedule_static_passes: usize,
1963 pub schedule_cache_hits: usize,
1964 pub schedule_cache_misses: usize,
1965 pub reused_schedule_vertices_total: usize,
1966 pub replan_iterations: usize,
1967 pub changed_vdeps_total: usize,
1968 pub bailout_reason: Option<&'static str>,
1969 pub fallback_mode_activations: u64,
1970}
1971
1972#[derive(Debug, Clone, Default, PartialEq)]
1981#[non_exhaustive]
1982pub struct CycleTelemetry {
1983 pub static_sccs: usize,
1985 pub phantom_sccs: usize,
1987 pub live_cycles_witnessed: usize,
1989 pub circ_cells_stamped: usize,
1991 pub settle_passes_total: usize,
1994 pub max_passes_single_scc: usize,
1996 pub iterated_sccs: usize,
1999 pub converged_sccs: usize,
2002 pub capped_sccs: usize,
2008 pub max_abs_delta_at_stop: f64,
2012 pub nan_converged: usize,
2015 pub reused_sccs: usize,
2020 pub reused_scc_members: usize,
2022 pub elapsed_ms: u128,
2024}
2025
2026#[derive(Debug, Clone, Copy)]
2027struct ScheduleBuildMeta {
2028 candidate_vertices: usize,
2029 vdeps_vertices: usize,
2030 vdeps_edges: usize,
2031 builder_elapsed_ms: u128,
2032 used_virtual_schedule: bool,
2033 schedule_cache_hit: bool,
2034 schedule_cache_eligible: bool,
2035}
2036
2037#[cfg(any(test, feature = "benchmark_internal"))]
2038#[doc(hidden)]
2039#[derive(Debug, Clone, Default)]
2040pub struct RecalcReuseProbe {
2041 pub schedule_requests: usize,
2042 pub schedule_cache_hits: usize,
2043 pub schedule_cache_misses: usize,
2044 pub schedule_cache_ineligible: usize,
2045 pub schedule_builds: usize,
2046 pub schedule_base_restrictions: usize,
2048 pub schedule_shared_handles: usize,
2049 pub schedule_retained_bytes: usize,
2050 pub legacy_target_requests: usize,
2051 pub target_schedule_builds: usize,
2052 pub demand_builds: usize,
2053 pub demand_vertices: usize,
2054 pub demand_clean_formulas: usize,
2055 pub demand_explicit_edges: usize,
2056 pub demand_virtual_builder_calls: usize,
2057}
2058
2059#[cfg(any(test, feature = "benchmark_internal"))]
2060fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2061 fn vector_bytes<T>(values: &Vec<T>) -> usize {
2062 values.capacity() * std::mem::size_of::<T>()
2063 }
2064
2065 [
2066 vector_bytes(&schedule.units),
2067 vector_bytes(&schedule.layers),
2068 vector_bytes(&schedule.cycles),
2069 ]
2070 .into_iter()
2071 .chain(
2072 schedule
2073 .layers
2074 .iter()
2075 .map(|layer| vector_bytes(&layer.vertices)),
2076 )
2077 .chain(schedule.cycles.iter().map(vector_bytes))
2078 .sum()
2079}
2080
2081#[derive(Debug, Clone)]
2082struct CachedScheduleEntry {
2083 topology_epoch: u64,
2084 authority_revision: (u64, u64),
2087 candidate_vertices: VertexIdRuns,
2090 schedule: Arc<crate::engine::scheduler::Schedule>,
2091}
2092
2093#[derive(Debug, Clone, Default)]
2096struct VertexIdRuns(Vec<(u32, u32)>);
2097
2098impl VertexIdRuns {
2099 fn from_slice(ids: &[VertexId]) -> Self {
2100 let mut runs: Vec<(u32, u32)> = Vec::new();
2101 for v in ids {
2102 match runs.last_mut() {
2103 Some((first, len)) if first.checked_add(*len) == Some(v.0) => *len += 1,
2104 _ => runs.push((v.0, 1)),
2105 }
2106 }
2107 runs.shrink_to_fit();
2108 Self(runs)
2109 }
2110
2111 fn equals(&self, ids: &[VertexId]) -> bool {
2112 let mut rest = ids;
2113 for &(first, len) in &self.0 {
2114 let len = len as usize;
2115 if rest.len() < len {
2116 return false;
2117 }
2118 let (head, tail) = rest.split_at(len);
2119 if head
2120 .iter()
2121 .enumerate()
2122 .any(|(i, v)| v.0 != first.wrapping_add(i as u32))
2123 {
2124 return false;
2125 }
2126 rest = tail;
2127 }
2128 rest.is_empty()
2129 }
2130
2131 fn heap_bytes(&self) -> usize {
2132 self.0.capacity() * std::mem::size_of::<(u32, u32)>()
2133 }
2134
2135 fn len(&self) -> usize {
2137 self.0.iter().map(|&(_, len)| len as usize).sum()
2138 }
2139}
2140
2141#[cfg(test)]
2142mod vertex_id_runs_tests {
2143 use super::{VertexId, VertexIdRuns};
2144
2145 #[test]
2146 fn runs_compare_like_the_list() {
2147 let ids = |v: &[u32]| v.iter().map(|&i| VertexId(i)).collect::<Vec<_>>();
2148 let list = ids(&[5, 6, 7, 2, 3, 9, 10, 10]);
2149 let runs = VertexIdRuns::from_slice(&list);
2150 assert_eq!(runs.0, vec![(5, 3), (2, 2), (9, 2), (10, 1)]);
2151 assert!(runs.equals(&list));
2152 assert!(!runs.equals(&list[..7]));
2153 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 11])));
2154 assert!(!runs.equals(&ids(&[5, 6, 7, 2, 3, 9, 10, 10, 11])));
2155 assert!(VertexIdRuns::from_slice(&[]).equals(&[]));
2156 assert!(!VertexIdRuns::from_slice(&[]).equals(&list));
2157 let edge = ids(&[u32::MAX - 1, u32::MAX, 0]);
2158 assert!(VertexIdRuns::from_slice(&edge).equals(&edge));
2159 }
2160}
2161
2162enum EvaluationSchedule {
2164 Owned(crate::engine::scheduler::Schedule),
2165 Shared(Arc<crate::engine::scheduler::Schedule>),
2166}
2167
2168impl std::ops::Deref for EvaluationSchedule {
2169 type Target = crate::engine::scheduler::Schedule;
2170
2171 fn deref(&self) -> &Self::Target {
2172 match self {
2173 Self::Owned(schedule) => schedule,
2174 Self::Shared(schedule) => schedule,
2175 }
2176 }
2177}
2178
2179type ScheduleBuildOutput = (
2180 crate::engine::scheduler::Schedule,
2181 FxHashMap<VertexId, Vec<VertexId>>,
2182 ScheduleBuildMeta,
2183);
2184
2185type EvaluationScheduleBuildOutput = (
2186 EvaluationSchedule,
2187 FxHashMap<VertexId, Vec<VertexId>>,
2188 ScheduleBuildMeta,
2189);
2190
2191#[derive(Debug)]
2193pub struct RecalcPlan {
2194 key: RecalcPlanKey,
2195 kind: RecalcPlanKind,
2196}
2197
2198#[derive(Debug)]
2199struct RecalcPlanKey {
2200 engine_token: Arc<()>,
2201 revisions: PlanningRevisionSnapshot,
2202}
2203
2204#[derive(Clone, Debug, PartialEq, Eq)]
2205struct PlanningRevisionSnapshot {
2206 engine_topology_epoch: u64,
2207 graph_topology_revision: u64,
2208 staged: u64,
2209 symbols: u64,
2210 semantic: u64,
2211 provider: Option<u64>,
2212 deterministic_mode: crate::engine::DeterministicMode,
2213 budgets: crate::engine::EvaluationBudgets,
2214}
2215
2216#[derive(Debug)]
2217enum RecalcPlanKind {
2218 CompatibilityFull {
2219 schedule: crate::engine::Schedule,
2220 has_dynamic_refs: bool,
2221 },
2222 Target {
2223 targets: Vec<crate::engine::EvaluationTarget>,
2224 scope: crate::engine::PrepareScope,
2225 topology: RecalcTopology,
2226 dynamic_policy: DynamicPlanPolicy,
2227 },
2228}
2229
2230#[derive(Debug)]
2231enum RecalcTopology {
2232 RunLocalRecipe,
2233 Workbook,
2234}
2235
2236#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2237enum DynamicPlanPolicy {
2238 BoundedTargetReplan,
2239}
2240
2241impl RecalcPlan {
2242 pub fn layer_count(&self) -> usize {
2245 match &self.kind {
2246 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2247 RecalcPlanKind::Target { .. } => 0,
2248 }
2249 }
2250
2251 pub fn has_dynamic_refs(&self) -> bool {
2252 match &self.kind {
2253 RecalcPlanKind::CompatibilityFull {
2254 has_dynamic_refs, ..
2255 } => *has_dynamic_refs,
2256 RecalcPlanKind::Target { .. } => false,
2257 }
2258 }
2259
2260 #[cfg(test)]
2261 pub(crate) fn force_stale_reasons_for_test(
2262 &mut self,
2263 reasons: &[formualizer_common::PlanStaleReason],
2264 ) {
2265 use formualizer_common::PlanStaleReason;
2266 for reason in reasons {
2267 match reason {
2268 PlanStaleReason::Engine => {
2269 self.key.engine_token = Arc::new(());
2270 }
2271 PlanStaleReason::Provider => {
2272 self.key.revisions.provider = Some(
2273 self.key
2274 .revisions
2275 .provider
2276 .unwrap_or_default()
2277 .wrapping_add(1),
2278 );
2279 }
2280 PlanStaleReason::Semantic => {
2281 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2282 }
2283 PlanStaleReason::Budget => {
2284 let current = self.key.revisions.budgets.work.max_work_units;
2285 self.key.revisions.budgets.work.max_work_units =
2286 Some(current.unwrap_or_default().wrapping_add(1));
2287 }
2288 PlanStaleReason::Staged => {
2289 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2290 }
2291 PlanStaleReason::Symbols => {
2292 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2293 }
2294 PlanStaleReason::Graph => {
2295 self.key.revisions.graph_topology_revision =
2296 self.key.revisions.graph_topology_revision.wrapping_add(1);
2297 }
2298 _ => {}
2299 }
2300 }
2301 }
2302}
2303
2304#[cfg(any(test, feature = "test-support"))]
2305pub(crate) mod criteria_mask_test_hooks {
2306 use std::cell::Cell;
2307
2308 thread_local! {
2309 static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2310 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2311 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2312 }
2313
2314 pub(crate) fn take_mask_work() -> (usize, usize) {
2315 MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2316 }
2317
2318 pub(crate) fn note_mask(rows: usize) {
2319 MASK_CALLS_ROWS.with(|c| {
2320 let (calls, work) = c.get();
2321 c.set((calls + 1, work + rows));
2322 });
2323 }
2324
2325 pub fn reset_text_segment_counters() {
2326 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2327 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2328 }
2329
2330 pub fn text_segment_counters() -> (usize, usize) {
2331 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2332 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2333 (a, b)
2334 }
2335
2336 pub(crate) fn inc_total() {
2337 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2338 }
2339 pub(crate) fn inc_all_null() {
2340 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2341 }
2342}
2343
2344#[cfg(test)]
2345pub(crate) mod visibility_mask_test_hooks {
2346 use std::cell::Cell;
2347
2348 thread_local! {
2349 static HITS: Cell<usize> = const { Cell::new(0) };
2350 static MISSES: Cell<usize> = const { Cell::new(0) };
2351 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2352 }
2353
2354 pub fn reset() {
2355 HITS.with(|c| c.set(0));
2356 MISSES.with(|c| c.set(0));
2357 EVICTIONS.with(|c| c.set(0));
2358 }
2359
2360 pub fn counters() -> (usize, usize, usize) {
2361 let hits = HITS.with(|c| c.get());
2362 let misses = MISSES.with(|c| c.get());
2363 let evictions = EVICTIONS.with(|c| c.get());
2364 (hits, misses, evictions)
2365 }
2366
2367 pub(crate) fn inc_hit() {
2368 HITS.with(|c| c.set(c.get() + 1));
2369 }
2370
2371 pub(crate) fn inc_miss() {
2372 MISSES.with(|c| c.set(c.get() + 1));
2373 }
2374
2375 pub(crate) fn inc_eviction() {
2376 EVICTIONS.with(|c| c.set(c.get() + 1));
2377 }
2378}
2379
2380fn is_numeric_text_equality(pred: &crate::args::CriteriaPredicate) -> bool {
2381 match pred {
2382 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(text)) => {
2383 text.trim().parse::<f64>().is_ok_and(f64::is_finite)
2384 }
2385 _ => false,
2386 }
2387}
2388
2389fn fill_numeric_equality_nulls(
2394 view: &RangeView<'_>,
2395 row_start: usize,
2396 row_len: usize,
2397 col_in_view: usize,
2398 mask: &arrow_array::BooleanArray,
2399 n: f64,
2400 ne: bool,
2401) -> Option<arrow_array::BooleanArray> {
2402 use arrow_array::Array as _;
2403 let texts = view.slice_lowered_text(row_start, row_len);
2404 let text = texts.get(col_in_view).and_then(|t| t.as_ref());
2405 match text {
2406 None => {
2407 if !ne {
2408 return Some(mask.clone());
2410 }
2411 let nulls = arrow::compute::is_null(mask).ok()?;
2412 crate::compute_prelude::boolean::or_kleene(mask, &nulls).ok()
2413 }
2414 Some(text) => {
2415 if text.len() != mask.len() {
2416 return None;
2417 }
2418 let mut out = arrow_array::builder::BooleanBuilder::with_capacity(mask.len());
2419 for i in 0..mask.len() {
2420 if mask.is_valid(i) {
2421 out.append_value(mask.value(i));
2422 } else if text.is_valid(i) {
2423 let equal = crate::locale::Locale::invariant()
2424 .parse_number_invariant(text.value(i))
2425 .is_some_and(|x| (x - n).abs() < 1e-12);
2426 out.append_value(equal != ne);
2427 } else {
2428 out.append_value(ne);
2429 }
2430 }
2431 Some(out.finish())
2432 }
2433 }
2434}
2435
2436fn compute_criteria_mask(
2437 view: &RangeView<'_>,
2438 col_in_view: usize,
2439 pred: &crate::args::CriteriaPredicate,
2440) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2441 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2442 use arrow::compute::kernels::comparison::{ilike, nilike};
2443 use arrow_array::{
2444 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2445 };
2446
2447 fn apply_numeric_pred(
2449 chunk: &Float64Array,
2450 pred: &crate::args::CriteriaPredicate,
2451 ) -> Option<BooleanArray> {
2452 match pred {
2453 crate::args::CriteriaPredicate::Gt(n) => {
2454 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2455 }
2456 crate::args::CriteriaPredicate::Ge(n) => {
2457 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2458 }
2459 crate::args::CriteriaPredicate::Lt(n) => {
2460 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2461 }
2462 crate::args::CriteriaPredicate::Le(n) => {
2463 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2464 }
2465 crate::args::CriteriaPredicate::Eq(v) => match v {
2466 formualizer_common::LiteralValue::Number(x) => {
2467 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2468 }
2469 formualizer_common::LiteralValue::Int(i) => {
2470 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2471 }
2472 _ => None,
2473 },
2474 crate::args::CriteriaPredicate::Ne(v) => match v {
2475 formualizer_common::LiteralValue::Number(x) => {
2476 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2477 }
2478 formualizer_common::LiteralValue::Int(i) => {
2479 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2480 }
2481 _ => None,
2482 },
2483 _ => None,
2484 }
2485 }
2486
2487 let is_numeric_pred = matches!(
2489 pred,
2490 crate::args::CriteriaPredicate::Gt(_)
2491 | crate::args::CriteriaPredicate::Ge(_)
2492 | crate::args::CriteriaPredicate::Lt(_)
2493 | crate::args::CriteriaPredicate::Le(_)
2494 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2495 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2496 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2497 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2498 );
2499
2500 if is_numeric_pred {
2504 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2505 let number = |v: &formualizer_common::LiteralValue| match v {
2509 formualizer_common::LiteralValue::Number(n) => Some(*n),
2510 formualizer_common::LiteralValue::Int(i) => Some(*i as f64),
2511 _ => None,
2512 };
2513 let equality = match pred {
2514 crate::args::CriteriaPredicate::Eq(v) => number(v).map(|n| (n, false)),
2515 crate::args::CriteriaPredicate::Ne(v) => number(v).map(|n| (n, true)),
2516 _ => None,
2517 };
2518 for res in view.numbers_slices() {
2519 let (rs, rl, cols_seg) = res.ok()?;
2520 if col_in_view < cols_seg.len() {
2521 let chunk = cols_seg[col_in_view].as_ref();
2522 let mut mask = apply_numeric_pred(chunk, pred)?;
2523 if let Some((n, ne)) = equality
2524 && mask.null_count() > 0
2525 {
2526 mask = fill_numeric_equality_nulls(view, rs, rl, col_in_view, &mask, n, ne)?;
2527 }
2528 bool_parts.push(mask);
2529 }
2530 }
2531
2532 if bool_parts.is_empty() {
2533 return None;
2534 } else if bool_parts.len() == 1 {
2535 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2536 } else {
2537 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2539 .iter()
2540 .map(|a| a as &dyn arrow_array::Array)
2541 .collect();
2542 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2543 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2544 return Some(std::sync::Arc::new(ba));
2545 }
2546 }
2547
2548 if is_numeric_text_equality(pred)
2552 || matches!(pred, crate::args::CriteriaPredicate::TextLike { .. })
2553 {
2554 for tags in view.type_tags_slices() {
2555 let (_, _, cols) = tags.ok()?;
2556 let tags = cols.get(col_in_view)?;
2557 if tags.values().iter().any(|tag| {
2558 *tag == crate::arrow_store::TypeTag::Empty as u8
2559 || *tag == crate::arrow_store::TypeTag::Number as u8
2560 || *tag == crate::arrow_store::TypeTag::Boolean as u8
2561 }) {
2562 let mut mask = BooleanBuilder::new();
2563 for chunk in view.iter_row_chunks() {
2564 let chunk = chunk.ok()?;
2565 for row in chunk.row_start..chunk.row_start + chunk.row_len {
2566 mask.append_value(crate::builtins::criteria_match(
2567 pred,
2568 &view.get_cell(row, col_in_view),
2569 ));
2570 }
2571 }
2572 return Some(std::sync::Arc::new(mask.finish()));
2573 }
2574 }
2575 }
2576
2577 if matches!(pred, crate::args::CriteriaPredicate::TextLike { pattern, .. }
2579 if pattern.contains('~'))
2580 {
2581 return None;
2582 }
2583
2584 if matches!(
2588 pred,
2589 crate::args::CriteriaPredicate::IsBlank | crate::args::CriteriaPredicate::NotBlank
2590 ) {
2591 let want_blank = matches!(pred, crate::args::CriteriaPredicate::IsBlank);
2592 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2593 for tags in view.type_tags_slices() {
2594 let (_, _, cols) = tags.ok()?;
2595 let tags = cols.get(col_in_view)?;
2596 let mut bb = BooleanBuilder::with_capacity(tags.len());
2597 for i in 0..tags.len() {
2598 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8;
2599 bb.append_value(blank == want_blank);
2600 }
2601 bool_parts.push(bb.finish());
2602 }
2603 return match bool_parts.len() {
2604 0 => None,
2605 1 => Some(std::sync::Arc::new(bool_parts.remove(0))),
2606 _ => {
2607 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2608 .iter()
2609 .map(|a| a as &dyn arrow_array::Array)
2610 .collect();
2611 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2612 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2613 Some(std::sync::Arc::new(ba))
2614 }
2615 };
2616 }
2617
2618 let like_pattern = |text: &str, wildcards: bool| {
2620 let mut pattern = String::with_capacity(text.len());
2621 for ch in text.chars() {
2622 match ch {
2623 '%' | '_' | '\\' => {
2624 pattern.push('\\');
2625 pattern.push(ch);
2626 }
2627 '*' if wildcards => pattern.push('%'),
2628 '?' if wildcards => pattern.push('_'),
2629 _ => pattern.push(ch),
2630 }
2631 }
2632 pattern
2633 };
2634 let (text_kind, text_pat, empty_special) = match pred {
2635 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2636 (0u8, like_pattern(&t.to_lowercase(), false), t.is_empty())
2637 }
2638 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2639 (1u8, like_pattern(&t.to_lowercase(), false), false)
2640 }
2641 crate::args::CriteriaPredicate::TextLike {
2642 pattern,
2643 case_insensitive,
2644 } => {
2645 let p = if *case_insensitive {
2646 pattern.to_lowercase()
2647 } else {
2648 pattern.clone()
2649 };
2650 (2u8, like_pattern(&p, true), false)
2651 }
2652 _ => return None,
2653 };
2654
2655 let text_pat_is_empty = text_pat.is_empty();
2656 let ne_matches_blank = text_kind == 1 && !text_pat_is_empty;
2657 let pat = StringArray::new_scalar(text_pat);
2658 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2659
2660 let mut tag_slices = view.type_tags_slices();
2661 for res in view.iter_row_chunks() {
2662 let cs = res.ok()?;
2663 if cs.row_len == 0 {
2664 continue;
2665 }
2666 #[cfg(test)]
2667 criteria_mask_test_hooks::inc_total();
2668
2669 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2670 if col_in_view >= slices.len() {
2671 return None;
2672 }
2673
2674 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2675 if empty_special || (text_kind == 1 && text_pat_is_empty) {
2676 let (tag_start, tag_len, tags) = tag_slices.next()?.ok()?;
2677 if tag_start != cs.row_start || tag_len != cs.row_len {
2678 return None;
2679 }
2680 let tags = tags.get(col_in_view)?;
2681 let strings = seg_opt.and_then(|a| a.as_any().downcast_ref::<StringArray>());
2685 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2686 for i in 0..cs.row_len {
2687 let blank = tags.value(i) == crate::arrow_store::TypeTag::Empty as u8
2688 || (tags.value(i) == crate::arrow_store::TypeTag::Text as u8
2689 && strings.is_some_and(|s| s.is_valid(i) && s.value(i).is_empty()));
2690 bb.append_value(if text_kind == 0 { blank } else { !blank });
2691 }
2692 #[cfg(test)]
2693 if seg_opt.is_none() {
2694 criteria_mask_test_hooks::inc_all_null();
2695 }
2696 bool_parts.push(bb.finish());
2697 continue;
2698 }
2699 let seg = match seg_opt {
2700 Some(s) => s,
2701 None => {
2702 #[cfg(test)]
2703 criteria_mask_test_hooks::inc_all_null();
2704 if (text_kind == 0 && empty_special) || ne_matches_blank {
2705 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2707 bb.append_n(cs.row_len, true);
2708 bool_parts.push(bb.finish());
2709 } else {
2710 bool_parts.push(BooleanArray::new_null(cs.row_len));
2712 }
2713 continue;
2714 }
2715 };
2716
2717 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2718 let mut m = match text_kind {
2719 0 => ilike(seg_sa, &pat).ok()?,
2720 1 => nilike(seg_sa, &pat).ok()?,
2721 2 => ilike(seg_sa, &pat).ok()?,
2722 _ => return None,
2723 };
2724
2725 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2731 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2733 for i in 0..seg_sa.len() {
2734 bb.append_value(seg_sa.is_null(i));
2735 }
2736 let nulls = bb.finish();
2737 m = boolean::or_kleene(&m, &nulls).ok()?;
2738 }
2739
2740 bool_parts.push(m);
2741 }
2742
2743 if bool_parts.is_empty() {
2744 None
2745 } else if bool_parts.len() == 1 {
2746 Some(std::sync::Arc::new(bool_parts.remove(0)))
2747 } else {
2748 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2749 .iter()
2750 .map(|a| a as &dyn arrow_array::Array)
2751 .collect();
2752 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2753 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2754 Some(std::sync::Arc::new(ba))
2755 }
2756}
2757
2758#[derive(Debug, Clone)]
2759pub struct LayerInfo {
2760 pub vertex_count: usize,
2761 pub parallel_eligible: bool,
2762 pub sample_cells: Vec<String>, }
2764
2765#[derive(Debug, Clone)]
2766pub struct EvalPlan {
2767 pub total_vertices_to_evaluate: usize,
2768 pub layers: Vec<LayerInfo>,
2769 pub cycles_detected: usize,
2770 pub dirty_count: usize,
2771 pub volatile_count: usize,
2772 pub parallel_enabled: bool,
2773 pub estimated_parallel_layers: usize,
2774 pub target_cells: Vec<String>,
2775}
2776
2777#[inline]
2780fn plane_mode_ignored() -> bool {
2781 true
2782}
2783
2784#[cfg(feature = "test-support")]
2787#[doc(hidden)]
2788pub fn formula_plane_mode_ignored_for_test() -> bool {
2789 plane_mode_ignored()
2790}
2791
2792impl<R> Engine<R>
2793where
2794 R: EvaluationContext,
2795{
2796 pub fn new(resolver: R, config: EvalConfig) -> Self {
2803 let config = if plane_mode_ignored() {
2808 EvalConfig {
2809 formula_plane_mode: FormulaPlaneMode::Off,
2810 ..config
2811 }
2812 } else {
2813 config
2814 };
2815 if let Err(msg) = config.cycle.validate() {
2816 panic!("invalid CycleConfig: {msg}");
2817 }
2818 crate::builtins::load_builtins();
2819 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2820 &config.evaluation_budgets,
2821 config.max_vertices,
2822 config.max_memory_mb,
2823 config.max_eval_time,
2824 );
2825
2826 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2827 #[cfg(feature = "system-clock")]
2828 {
2829 Arc::new(crate::timezone::SystemClock::new(
2830 crate::timezone::TimeZoneSpec::default(),
2831 ))
2832 }
2833 #[cfg(not(feature = "system-clock"))]
2834 {
2835 Arc::new(crate::timezone::FixedClock::new(
2836 chrono::DateTime::UNIX_EPOCH,
2837 crate::timezone::TimeZoneSpec::Utc,
2838 ))
2839 }
2840 });
2841
2842 let thread_pool = if config.enable_parallel {
2844 let mut builder = ThreadPoolBuilder::new();
2845 if let Some(max_threads) = config.max_threads {
2846 builder = builder.num_threads(max_threads);
2847 }
2848
2849 match builder.build() {
2850 Ok(pool) => Some(Arc::new(pool)),
2851 Err(_) => {
2852 None
2854 }
2855 }
2856 } else {
2857 None
2858 };
2859
2860 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2862 let function_provider_revision_seen = resolver.planning_semantic_revision();
2863 let mut engine = Self {
2864 graph: DependencyGraph::new_with_config(config.clone()),
2865 resolver,
2866 config,
2867 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2868 clock: crate::timezone::SnapshotClock::new(clock),
2869 thread_pool,
2870 recalc_epoch: 0,
2871 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2872 topology_epoch: 0,
2873 cached_static_schedule: None,
2874 recent_schedules: Vec::new(),
2875 base_schedule: None,
2876 #[cfg(any(test, feature = "benchmark_internal"))]
2877 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2878 spill_mgr: ShimSpillManager::default(),
2879 arrow_sheets: SheetStore::default(),
2880 format_registry: crate::format::FormatRegistry::default(),
2881 derived_formats: Default::default(),
2882 #[cfg(test)]
2883 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2884 #[cfg(test)]
2885 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
2886 #[cfg(test)]
2887 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
2888 #[cfg(test)]
2889 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
2890 #[cfg(test)]
2891 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
2892 #[cfg(test)]
2893 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
2894 #[cfg(test)]
2895 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
2896 #[cfg(test)]
2897 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
2898 compressed_at_build: None,
2899 #[cfg(test)]
2900 computed_overlay_set_explicit_entry_operations_for_test: 0,
2901 #[cfg(test)]
2902 computed_overlay_stale_clear_range_effects_for_test: 0,
2903 #[cfg(test)]
2904 computed_overlay_stale_clear_offset_attempts_for_test: 0,
2905 #[cfg(test)]
2906 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2907 has_edited: false,
2908 overlay_compactions: 0,
2909 computed_overlay_bytes_estimate: 0,
2910 computed_overlay_mirroring_disabled: false,
2911 force_materialize_range_views: false,
2912 row_bounds_cache: std::sync::RwLock::new(None),
2913 used_axis_bounds_cache: std::sync::RwLock::new(None),
2914 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2915 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2916 source_formula_token: Arc::new(()),
2917 recalc_plan_token: Arc::new(()),
2918 staged_formulas: std::collections::HashMap::new(),
2919 staged_formula_index: StagedFormulaIndex::default(),
2920 blocked_pending_spills: Vec::new(),
2921 spill_blocker_waiters: SpillBlockerWaiters::default(),
2922 row_visibility: FxHashMap::default(),
2923 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2924 formula_parse_diagnostics: Vec::new(),
2925 last_formula_ingest_report: None,
2926 formula_ingest_report_total: FormulaIngestReport::default(),
2927 active_cancel_flag: None,
2928 active_evaluation_deadline: None,
2929 action_depth: 0,
2930 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2931 virtual_dep_fallback_activations: 0,
2932 last_cycle_telemetry: CycleTelemetry::default(),
2933 next_evaluation_resource_request_id: 1,
2934 evaluation_resource_request_depth: 0,
2935 active_evaluation_resource_request: None,
2936 last_evaluation_resource_request: None,
2937 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2938 evaluation_resource_request_started_at: None,
2939 evaluation_resource_budgets: resolved_resources.budgets,
2940 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2941 active_resource_ledger: None,
2942 source_cache_footprints: Vec::new(),
2943 source_cache_accounted: 0,
2944 pending_iterative_redirty: Vec::new(),
2945 snapshot_evaluation_active: false,
2946 snapshot_volatile_redirty_pending: false,
2947 retained_scc_members: FxHashMap::default(),
2948 next_retained_scc_id: 0,
2949 retained_scc_config_fingerprint: 0,
2950 retained_scc_function_epoch_seen: 0,
2951 retained_scc_provider_revision_seen: None,
2952 retained_scc_dirty_at_begin: Vec::new(),
2953 iterative_state_values: FxHashMap::default(),
2954 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2955 function_provider_revision_seen,
2956 #[cfg(feature = "tracing")]
2957 trace_evaluation_counters: TraceEvaluationCounters::default(),
2958 #[cfg(test)]
2959 evaluation_request_begin_count_for_test: 0,
2960 #[cfg(any(test, feature = "test-support"))]
2961 before_prepared_span_commit_hook: None,
2962 #[cfg(test)]
2963 before_target_preparation_commit_hook: None,
2964 #[cfg(test)]
2965 before_target_planning_snapshot_hook: None,
2966 #[cfg(test)]
2967 inject_target_semantic_stale_once_for_test: false,
2968 #[cfg(test)]
2969 force_virtual_dep_changes_remaining_for_test: 0,
2970 freshness: Default::default(),
2971 #[cfg(test)]
2972 fail_evaluation_commit_preflight_once_for_test: false,
2973 #[cfg(test)]
2974 target_preparation_fault_for_test: None,
2975 #[cfg(test)]
2976 force_non_cycle_schedule_fallback_for_test: false,
2977 #[cfg(test)]
2978 before_legacy_fallback_final_provider_sample_hook: None,
2979 #[cfg(test)]
2980 after_eager_proposal_commit_hook: None,
2981 };
2982 engine.config.arrow_storage_enabled = true;
2984 engine.config.delta_overlay_enabled = true;
2985 engine.config.write_formula_overlay_enabled = true;
2986 let default_sheet = engine.graph.default_sheet_name().to_string();
2987 engine.ensure_arrow_sheet(&default_sheet);
2988 engine
2989 }
2990
2991 pub fn with_thread_pool(
2996 resolver: R,
2997 config: EvalConfig,
2998 thread_pool: Arc<rayon::ThreadPool>,
2999 ) -> Self {
3000 if let Err(msg) = config.cycle.validate() {
3001 panic!("invalid CycleConfig: {msg}");
3002 }
3003 crate::builtins::load_builtins();
3004 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3005 &config.evaluation_budgets,
3006 config.max_vertices,
3007 config.max_memory_mb,
3008 config.max_eval_time,
3009 );
3010 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3011 #[cfg(feature = "system-clock")]
3012 {
3013 Arc::new(crate::timezone::SystemClock::new(
3014 crate::timezone::TimeZoneSpec::default(),
3015 ))
3016 }
3017 #[cfg(not(feature = "system-clock"))]
3018 {
3019 Arc::new(crate::timezone::FixedClock::new(
3020 chrono::DateTime::UNIX_EPOCH,
3021 crate::timezone::TimeZoneSpec::Utc,
3022 ))
3023 }
3024 });
3025 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3027 let function_provider_revision_seen = resolver.planning_semantic_revision();
3028 let mut engine = Self {
3029 graph: DependencyGraph::new_with_config(config.clone()),
3030 resolver,
3031 config,
3032 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3033 clock: crate::timezone::SnapshotClock::new(clock),
3034 thread_pool: Some(thread_pool),
3035 recalc_epoch: 0,
3036 snapshot_id: std::sync::atomic::AtomicU64::new(1),
3037 topology_epoch: 0,
3038 cached_static_schedule: None,
3039 recent_schedules: Vec::new(),
3040 base_schedule: None,
3041 #[cfg(any(test, feature = "benchmark_internal"))]
3042 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3043 spill_mgr: ShimSpillManager::default(),
3044 arrow_sheets: SheetStore::default(),
3045 format_registry: crate::format::FormatRegistry::default(),
3046 derived_formats: Default::default(),
3047 #[cfg(test)]
3048 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3049 #[cfg(test)]
3050 family_members_for_test: std::sync::atomic::AtomicU64::new(0),
3051 #[cfg(test)]
3052 invariant_bound_members_for_test: std::sync::atomic::AtomicU64::new(0),
3053 #[cfg(test)]
3054 lifted_members_for_test: std::sync::atomic::AtomicU64::new(0),
3055 #[cfg(test)]
3056 chained_members_for_test: std::sync::atomic::AtomicU64::new(0),
3057 #[cfg(test)]
3058 lane_clean_reads_for_test: std::sync::atomic::AtomicU64::new(0),
3059 #[cfg(test)]
3060 criteria_kernel_members_for_test: std::sync::atomic::AtomicU64::new(0),
3061 #[cfg(test)]
3062 memo_hits_for_test: std::sync::atomic::AtomicU64::new(0),
3063 compressed_at_build: None,
3064 #[cfg(test)]
3065 computed_overlay_set_explicit_entry_operations_for_test: 0,
3066 #[cfg(test)]
3067 computed_overlay_stale_clear_range_effects_for_test: 0,
3068 #[cfg(test)]
3069 computed_overlay_stale_clear_offset_attempts_for_test: 0,
3070 #[cfg(test)]
3071 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3072 has_edited: false,
3073 overlay_compactions: 0,
3074 computed_overlay_bytes_estimate: 0,
3075 computed_overlay_mirroring_disabled: false,
3076 force_materialize_range_views: false,
3077 row_bounds_cache: std::sync::RwLock::new(None),
3078 used_axis_bounds_cache: std::sync::RwLock::new(None),
3079 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3080 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3081 source_formula_token: Arc::new(()),
3082 recalc_plan_token: Arc::new(()),
3083 staged_formulas: std::collections::HashMap::new(),
3084 staged_formula_index: StagedFormulaIndex::default(),
3085 blocked_pending_spills: Vec::new(),
3086 spill_blocker_waiters: SpillBlockerWaiters::default(),
3087 row_visibility: FxHashMap::default(),
3088 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3089 formula_parse_diagnostics: Vec::new(),
3090 last_formula_ingest_report: None,
3091 formula_ingest_report_total: FormulaIngestReport::default(),
3092 active_cancel_flag: None,
3093 active_evaluation_deadline: None,
3094 action_depth: 0,
3095 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3096 virtual_dep_fallback_activations: 0,
3097 last_cycle_telemetry: CycleTelemetry::default(),
3098 next_evaluation_resource_request_id: 1,
3099 evaluation_resource_request_depth: 0,
3100 active_evaluation_resource_request: None,
3101 last_evaluation_resource_request: None,
3102 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3103 evaluation_resource_request_started_at: None,
3104 evaluation_resource_budgets: resolved_resources.budgets,
3105 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3106 active_resource_ledger: None,
3107 source_cache_footprints: Vec::new(),
3108 source_cache_accounted: 0,
3109 pending_iterative_redirty: Vec::new(),
3110 snapshot_evaluation_active: false,
3111 snapshot_volatile_redirty_pending: false,
3112 retained_scc_members: FxHashMap::default(),
3113 next_retained_scc_id: 0,
3114 retained_scc_config_fingerprint: 0,
3115 retained_scc_function_epoch_seen: 0,
3116 retained_scc_provider_revision_seen: None,
3117 retained_scc_dirty_at_begin: Vec::new(),
3118 iterative_state_values: FxHashMap::default(),
3119 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3120 function_provider_revision_seen,
3121 #[cfg(feature = "tracing")]
3122 trace_evaluation_counters: TraceEvaluationCounters::default(),
3123 #[cfg(test)]
3124 evaluation_request_begin_count_for_test: 0,
3125 #[cfg(any(test, feature = "test-support"))]
3126 before_prepared_span_commit_hook: None,
3127 #[cfg(test)]
3128 before_target_preparation_commit_hook: None,
3129 #[cfg(test)]
3130 before_target_planning_snapshot_hook: None,
3131 #[cfg(test)]
3132 inject_target_semantic_stale_once_for_test: false,
3133 #[cfg(test)]
3134 force_virtual_dep_changes_remaining_for_test: 0,
3135 freshness: Default::default(),
3136 #[cfg(test)]
3137 fail_evaluation_commit_preflight_once_for_test: false,
3138 #[cfg(test)]
3139 target_preparation_fault_for_test: None,
3140 #[cfg(test)]
3141 force_non_cycle_schedule_fallback_for_test: false,
3142 #[cfg(test)]
3143 before_legacy_fallback_final_provider_sample_hook: None,
3144 #[cfg(test)]
3145 after_eager_proposal_commit_hook: None,
3146 };
3147 engine.config.arrow_storage_enabled = true;
3149 engine.config.delta_overlay_enabled = true;
3150 engine.config.write_formula_overlay_enabled = true;
3151 let default_sheet = engine.graph.default_sheet_name().to_string();
3152 engine.ensure_arrow_sheet(&default_sheet);
3153 engine
3154 }
3155
3156 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3157 &self.workbook_load_limits
3158 }
3159
3160 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3161 self.workbook_load_limits = limits;
3162 }
3163
3164 fn clear_source_cache(&self) {
3165 if let Ok(mut g) = self.source_cache.write() {
3166 *g = SourceCache::default();
3167 }
3168 }
3169
3170 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3171 &self.last_virtual_dep_telemetry
3172 }
3173
3174 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3178 &self.last_cycle_telemetry
3179 }
3180
3181 pub fn last_evaluation_resource_request_stats(
3183 &self,
3184 ) -> Option<&EvaluationResourceRequestStats> {
3185 self.last_evaluation_resource_request.as_ref()
3186 }
3187
3188 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3190 self.evaluation_resource_baseline
3191 }
3192
3193 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3194 &self.evaluation_resource_budgets
3195 }
3196
3197 pub fn evaluation_resource_config_diagnostic(
3199 &self,
3200 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3201 self.evaluation_resource_config_diagnostic.as_ref()
3202 }
3203
3204 pub fn reset_evaluation_resource_telemetry(&mut self) {
3206 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3207 self.last_evaluation_resource_request = None;
3208 }
3209
3210 fn reconcile_source_cache_footprints(&mut self) -> Result<(), ExcelError> {
3213 self.blocked_pending_spills.retain(|&(vertex, anchor, _)| {
3214 self.graph.vertex_exists(vertex)
3215 && self.graph.get_cell_ref(vertex) == Some(anchor)
3216 && matches!(
3217 self.graph.get_vertex_kind(vertex),
3218 VertexKind::FormulaScalar | VertexKind::FormulaArray
3219 )
3220 });
3221 if self.blocked_pending_spills.is_empty() {
3222 self.blocked_pending_spills = Vec::new();
3223 }
3224 let mut bytes = (self.blocked_pending_spills.capacity()
3225 * std::mem::size_of::<(VertexId, CellRef, Region)>()) as u64;
3226 self.source_cache_footprints.retain(|weak| {
3227 let Some(footprint) = weak.upgrade() else {
3228 return false;
3229 };
3230 bytes = bytes.saturating_add(footprint.load(std::sync::atomic::Ordering::Acquire));
3231 true
3232 });
3233 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3234 ledger
3235 .release_retained(self.source_cache_accounted)
3236 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3237 ledger.observe_retained(bytes);
3239 self.source_cache_accounted = bytes;
3240 ledger
3241 .reserve_retained(0)
3242 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3243 }
3244 Ok(())
3245 }
3246
3247 fn duration_ns(duration: std::time::Duration) -> u64 {
3248 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3249 }
3250
3251 fn observe_evaluation_resource_request<T>(
3252 &mut self,
3253 kind: EvaluationRequestKind,
3254 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3255 ) -> Result<T, ExcelError> {
3256 let outermost = self.evaluation_resource_request_depth == 0;
3257 if outermost && self.snapshot_volatile_redirty_pending {
3258 self.snapshot_volatile_redirty_pending = false;
3259 self.graph.redirty_volatiles();
3260 }
3261 #[cfg(feature = "tracing")]
3262 if outermost {
3263 self.trace_evaluation_counters = TraceEvaluationCounters::default();
3264 }
3265 #[cfg(feature = "tracing")]
3266 let request_kind = if matches!(
3267 kind,
3268 EvaluationRequestKind::Full
3269 | EvaluationRequestKind::FullWithDelta
3270 | EvaluationRequestKind::FullCancellable
3271 | EvaluationRequestKind::FullLogged
3272 ) {
3273 "full"
3274 } else {
3275 "targeted"
3276 };
3277 #[cfg(feature = "tracing")]
3278 let _request_span = outermost.then(|| {
3279 crate::engine::trace::fz_span!(
3280 tracing::Level::INFO,
3281 "evaluate",
3282 "evaluate.request",
3283 kind = request_kind,
3284 mode = ?FormulaPlaneMode::Off
3285 )
3286 });
3287 if outermost {
3288 let request_id = self.next_evaluation_resource_request_id;
3289 self.next_evaluation_resource_request_id = request_id
3290 .checked_add(1)
3291 .expect("evaluation resource request ID exhausted");
3292 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3293 request_id,
3294 kind,
3295 FormulaPlaneMode::Off,
3296 self.staged_formula_count(),
3297 ));
3298 self.evaluation_resource_baseline.record_started(request_id);
3299 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3300 self.source_cache_accounted = 0;
3301 self.active_resource_ledger = Some(ResourceLedger::new(
3302 Some(request_id),
3303 self.evaluation_resource_budgets.clone(),
3304 ));
3305 }
3306 self.evaluation_resource_request_depth =
3307 self.evaluation_resource_request_depth.saturating_add(1);
3308 let result = if outermost {
3309 self.reconcile_source_cache_footprints()
3310 .and_then(|()| self.resource_checkpoint(0))
3311 .and_then(|()| evaluate(self))
3312 } else {
3313 evaluate(self)
3314 };
3315 if outermost && result.is_err() {
3316 self.freshness_abort_pass();
3317 }
3318 self.evaluation_resource_request_depth =
3319 self.evaluation_resource_request_depth.saturating_sub(1);
3320
3321 let reconciliation = if outermost {
3322 self.reconcile_source_cache_footprints()
3323 } else {
3324 Ok(())
3325 };
3326 let result = result.and_then(|value| reconciliation.map(|()| value));
3327 if outermost {
3328 let total_ns = self
3329 .evaluation_resource_request_started_at
3330 .take()
3331 .map(|start| Self::duration_ns(start.elapsed()))
3332 .unwrap_or(0);
3333 let mut stats = self
3334 .active_evaluation_resource_request
3335 .take()
3336 .expect("outer evaluation resource request has active stats");
3337 let mut ledger = self
3338 .active_resource_ledger
3339 .take()
3340 .expect("outer evaluation resource request has active ledger");
3341 ledger.release_all_scratch();
3342 stats.ledger.update(ledger.snapshot());
3343 stats.outcome = match &result {
3344 Ok(_) => EvaluationRequestOutcome::Success,
3345 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3346 EvaluationRequestOutcome::Cancelled
3347 }
3348 Err(_) => EvaluationRequestOutcome::Error,
3349 };
3350 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3351 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3352 FormulaDirtyLeaseOutcome::RetainedOnCancellation
3353 } else {
3354 FormulaDirtyLeaseOutcome::RetainedOnError
3355 };
3356 }
3357 stats.phases.total_ns = total_ns;
3358 let attributed = stats
3359 .phases
3360 .staged_prepare_ns
3361 .saturating_add(stats.phases.topology_ns)
3362 .saturating_add(stats.phases.materialization_ns);
3363 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3364 self.evaluation_resource_baseline.record_finished(&stats);
3365 self.last_evaluation_resource_request = Some(stats);
3366 crate::engine::trace::fz_event!(
3367 tracing::Level::INFO,
3368 "evaluate",
3369 "evaluate.summary",
3370 computed_vertices = self.trace_evaluation_counters.computed_vertices,
3371 cycles = self.trace_evaluation_counters.cycles,
3372 cancelled = matches!(
3373 &result,
3374 Err(error) if error.kind == ExcelErrorKind::Cancelled
3375 )
3376 );
3377 }
3378 result
3379 }
3380
3381 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3382 self.evaluation_resource_budgets = budgets.clone();
3383 self.config.evaluation_budgets = budgets.clone();
3384 self.graph.set_evaluation_budgets(budgets);
3385 }
3386
3387 #[cfg(test)]
3388 pub(crate) fn set_evaluation_budgets_for_test(
3389 &mut self,
3390 budgets: crate::engine::EvaluationBudgets,
3391 ) {
3392 self.set_evaluation_resource_budgets(budgets);
3393 }
3394
3395 fn preflight_evaluation_commit_window(
3396 &mut self,
3397 bounded_writes: usize,
3398 ) -> Result<crate::instant::FzInstant, ExcelError> {
3399 #[cfg(test)]
3400 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3401 return Err(crate::engine::ResourceLedgerError::Exhausted(
3402 formualizer_common::ResourceExhaustionDetail {
3403 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3404 limit: 0,
3405 observed: 1,
3406 request_id: self
3407 .active_evaluation_resource_request
3408 .as_ref()
3409 .map(|stats| stats.request_id),
3410 },
3411 )
3412 .into_excel_error());
3413 }
3414 let estimate = std::time::Duration::from_nanos(
3415 u64::try_from(bounded_writes)
3416 .unwrap_or(u64::MAX)
3417 .saturating_mul(100),
3418 );
3419 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3420 ledger
3421 .preflight_commit_window(estimate)
3422 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3423 }
3424 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3425 stats.evaluation_commit_preflight_count =
3426 stats.evaluation_commit_preflight_count.saturating_add(1);
3427 stats.evaluation_commit_estimated_ns = stats
3428 .evaluation_commit_estimated_ns
3429 .saturating_add(Self::duration_ns(estimate));
3430 }
3431 Ok(crate::instant::FzInstant::now())
3432 }
3433
3434 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3435 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3436 stats.evaluation_commit_actual_ns = stats
3437 .evaluation_commit_actual_ns
3438 .saturating_add(Self::duration_ns(started.elapsed()));
3439 }
3440 }
3441
3442 fn live_cancellation_after_work(&self, message: &'static str) -> Result<(), ExcelError> {
3454 if self
3455 .active_cancel_flag
3456 .as_ref()
3457 .is_some_and(|cancel| cancel.is_cancelled())
3458 {
3459 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3460 }
3461 Ok(())
3462 }
3463
3464 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3465 if self
3466 .active_cancel_flag
3467 .as_ref()
3468 .is_some_and(|cancel| cancel.is_cancelled())
3469 {
3470 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3471 }
3472 if self
3473 .active_evaluation_deadline
3474 .is_some_and(|deadline| Instant::now() >= deadline)
3475 {
3476 return Err(crate::engine::ResourceLedgerError::Exhausted(
3477 formualizer_common::ResourceExhaustionDetail {
3478 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3479 limit: 0,
3480 observed: 1,
3481 request_id: self
3482 .active_evaluation_resource_request
3483 .as_ref()
3484 .map(|request| request.request_id),
3485 },
3486 )
3487 .into_excel_error()
3488 .with_message(message));
3489 }
3490 Ok(())
3491 }
3492
3493 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3494 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3495 return Ok(());
3496 };
3497 ledger
3498 .charge_work(work_units)
3499 .and_then(|()| ledger.checkpoint_deadline())
3500 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3501 }
3502
3503 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3504 if work_units == 0 {
3505 return self.resource_checkpoint(0);
3506 }
3507 while work_units > 0 {
3508 let chunk = work_units.min(256);
3509 self.resource_checkpoint(chunk)?;
3510 work_units -= chunk;
3511 }
3512 Ok(())
3513 }
3514
3515 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3516 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3517 ledger.observe_scratch(bytes);
3520 }
3521 Ok(())
3522 }
3523
3524 fn release_request_scratch(&mut self, bytes: u64) {
3525 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3526 let released = ledger.release_scratch(bytes);
3527 debug_assert!(
3528 released.is_ok(),
3529 "request scratch release exceeded the outstanding reservation"
3530 );
3531 }
3532 }
3533
3534 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3535 self.active_resource_ledger
3536 .as_mut()
3537 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3538 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3539 }
3540
3541 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3542 self.active_resource_ledger
3543 .as_mut()
3544 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3545 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3546 }
3547
3548 fn with_request_scratch<T>(
3549 &mut self,
3550 bytes: u64,
3551 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3552 ) -> Result<T, ExcelError> {
3553 self.reserve_request_scratch(bytes)?;
3554 let result = work(self);
3555 self.release_request_scratch(bytes);
3556 result
3557 }
3558
3559 fn observe_staged_preparation(
3560 &mut self,
3561 selected: usize,
3562 retained: usize,
3563 elapsed: std::time::Duration,
3564 ) {
3565 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3566 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3567 stats.staged_retained = retained as u64;
3568 stats.phases.staged_prepare_ns = stats
3569 .phases
3570 .staged_prepare_ns
3571 .saturating_add(Self::duration_ns(elapsed));
3572 }
3573 }
3574
3575 fn graph_admission_enabled(&self) -> bool {
3576 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3577 }
3578
3579 fn preflight_graph_admission(
3580 &mut self,
3581 usage: crate::engine::resource_ledger::GraphAdmission,
3582 ) -> Result<(), ExcelError> {
3583 let request_id = self
3584 .active_evaluation_resource_request
3585 .as_ref()
3586 .map(|stats| stats.request_id);
3587 crate::engine::resource_ledger::preflight_graph_admission(
3588 &self.evaluation_resource_budgets,
3589 usage,
3590 request_id,
3591 )
3592 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3593 }
3594
3595 fn prepared_legacy_admission(
3596 &mut self,
3597 plan: &PreparedLegacyGraphPlan,
3598 materialization_cells: u64,
3599 ) -> Result<(), ExcelError> {
3600 if !self.graph_admission_enabled() {
3601 return Ok(());
3602 }
3603 let stats = self.graph.baseline_stats();
3604 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3605 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3606 })?;
3607 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3608 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3609 })?;
3610 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3611 final_vertices: stats
3612 .graph_vertex_count
3613 .checked_add(plan.new_vertex_count())
3614 .ok_or_else(|| {
3615 ExcelError::new(ExcelErrorKind::NImpl)
3616 .with_message("graph vertex count overflow")
3617 })?,
3618 final_edges: stats
3619 .graph_edge_count
3620 .checked_sub(removed_edges)
3621 .and_then(|count| count.checked_add(added_edges))
3622 .ok_or_else(|| {
3623 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3624 })?,
3625 materialization_cells,
3626 added_vertices: plan.new_vertex_count(),
3627 added_edges,
3628 })
3629 }
3630
3631 fn observe_target_admission_failure(
3632 &mut self,
3633 reason: formualizer_common::ResourceExhaustionReason,
3634 ) {
3635 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3636 stats.target_admission_failure = Some(reason);
3637 }
3638 }
3639
3640 fn observe_target_preparation_report(
3641 &mut self,
3642 report: &crate::engine::PreparedTargetGraphReport,
3643 ) {
3644 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3645 let index = match reason {
3646 crate::engine::OpaqueReason::DynamicReference => 0,
3647 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3648 crate::engine::OpaqueReason::UnknownFunction => 2,
3649 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3650 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3651 crate::engine::OpaqueReason::UnresolvedName => 5,
3652 crate::engine::OpaqueReason::UnresolvedTable => 6,
3653 crate::engine::OpaqueReason::FormulaName => 7,
3654 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3655 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3656 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3657 };
3658 1u64 << index
3659 };
3660 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3661 stats.staged_selected = report.selected_staged_cells as u64;
3662 stats.staged_retained = report.retained_staged_cells as u64;
3663 stats.target_requested = report.requested_targets as u64;
3664 stats.target_normalized_regions = report.normalized_regions as u64;
3665 stats.target_scope_level = match &report.widened_scope {
3666 crate::engine::PrepareScope::Exact => 0,
3667 crate::engine::PrepareScope::Sheets(_) => 1,
3668 crate::engine::PrepareScope::Workbook => 2,
3669 };
3670 stats.target_widening_reason_bits = report
3671 .widening_reasons
3672 .iter()
3673 .copied()
3674 .fold(0, |bits, reason| bits | reason_bit(reason));
3675 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3676 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3677 stats.target_commit_estimated_work = report.estimated_commit_work;
3678 stats.target_commit_actual_work = report.actual_commit_work;
3679 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3680 stats.phases.staged_prepare_ns = stats
3681 .phases
3682 .staged_prepare_ns
3683 .saturating_add(Self::duration_ns(report.commit_window));
3684 }
3685 }
3686
3687 fn begin_evaluation_request(&mut self) {
3691 self.freshness_begin_request();
3692 #[cfg(test)]
3693 {
3694 self.evaluation_request_begin_count_for_test = self
3695 .evaluation_request_begin_count_for_test
3696 .saturating_add(1);
3697 }
3698 self.last_cycle_telemetry = CycleTelemetry::default();
3699 self.graph.authority_sync();
3700 self.pending_iterative_redirty.clear();
3703 self.reconcile_retained_sccs_at_request_begin();
3704 self.clock.refresh();
3708 }
3709
3710 fn redirty_for_next_recalc(&mut self) {
3719 if self.snapshot_evaluation_active {
3720 self.snapshot_volatile_redirty_pending = true;
3721 } else {
3722 self.graph.redirty_volatiles();
3723 }
3724 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3725 let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3726 for (vertex, scc) in dirty_at_begin {
3727 if self.retained_scc_members.get(&vertex) == Some(&scc) {
3728 self.retained_scc_members.remove(&vertex);
3734 if !self.graph.is_dirty(vertex) {
3735 self.iterative_state_values.remove(&vertex);
3736 }
3737 }
3738 }
3739 if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3740 let graph = &self.graph;
3741 self.iterative_state_values
3742 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3743 self.retained_scc_members
3744 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3745 }
3746 for &vertex in &pending {
3751 if !self.graph.is_live_formula_vertex(vertex) {
3752 continue;
3753 }
3754 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3755 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3756 match self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3757 Some(value) if !matches!(value, LiteralValue::Empty) => {
3758 self.iterative_state_values.insert(vertex, value);
3759 }
3760 _ => {
3761 self.iterative_state_values.remove(&vertex);
3762 }
3763 }
3764 }
3765 }
3766 if !pending.is_empty() {
3767 self.graph.redirty_iterative_members(&pending);
3768 }
3769 }
3770
3771 fn retained_scc_config_fingerprint(&self) -> u64 {
3776 use std::hash::{Hash, Hasher};
3777 let mut hasher = rustc_hash::FxHasher::default();
3778 let config = &self.config;
3779 std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3780 match config.cycle.policy {
3781 CyclePolicy::Error => 0u8.hash(&mut hasher),
3782 CyclePolicy::Iterate {
3783 max_iterations,
3784 max_change,
3785 } => {
3786 1u8.hash(&mut hasher);
3787 max_iterations.hash(&mut hasher);
3788 max_change.to_bits().hash(&mut hasher);
3789 }
3790 }
3791 config.date_system.hash(&mut hasher);
3792 config.workbook_seed.hash(&mut hasher);
3793 std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3794 format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3797 config.range_expansion_limit.hash(&mut hasher);
3798 config.max_open_ended_rows.hash(&mut hasher);
3799 config.max_open_ended_cols.hash(&mut hasher);
3800 hasher.finish()
3801 }
3802
3803 fn reconcile_retained_sccs_at_request_begin(&mut self) {
3809 self.retained_scc_dirty_at_begin.clear();
3810 if self.retained_scc_members.is_empty() {
3811 return;
3812 }
3813 let graph = &self.graph;
3816 self.retained_scc_members
3817 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3818 if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3819 let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3820 self.retained_scc_members.clear();
3821 self.graph.mark_dirty_many(&members);
3822 return;
3823 }
3824 let changes =
3825 crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3826 let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3827 let provider_revision = self.resolver.planning_semantic_revision();
3828 let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3829 if global_changed || provider_changed {
3830 let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3831 let affected: Vec<VertexId> = self
3832 .retained_scc_members
3833 .keys()
3834 .copied()
3835 .filter(|&vertex| {
3836 let Some(ast) = self.graph.get_formula(vertex) else {
3837 return true;
3838 };
3839 (global_changed
3840 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3841 || (provider_changed && Self::ast_contains_function(&ast))
3842 })
3843 .collect();
3844 for vertex in &affected {
3845 self.retained_scc_members.remove(vertex);
3846 }
3847 if !affected.is_empty() {
3848 self.graph.mark_dirty_many(&affected);
3849 }
3850 self.retained_scc_function_epoch_seen = changes.epoch;
3851 self.retained_scc_provider_revision_seen = provider_revision;
3852 }
3853 let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
3854 let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
3855 for (&vertex, &scc) in &self.retained_scc_members {
3856 all_sccs.insert(scc);
3857 if self.graph.is_dirty(vertex) {
3858 dirty_sccs.insert(scc);
3859 self.retained_scc_dirty_at_begin.push((vertex, scc));
3860 }
3861 }
3862 let reused_members = self
3863 .retained_scc_members
3864 .values()
3865 .filter(|scc| !dirty_sccs.contains(scc))
3866 .count();
3867 let t = &mut self.last_cycle_telemetry;
3868 t.reused_sccs = all_sccs.len() - dirty_sccs.len();
3869 t.reused_scc_members = reused_members;
3870 }
3871
3872 pub fn virtual_dep_fallback_activations(&self) -> u64 {
3873 self.virtual_dep_fallback_activations
3874 }
3875
3876 #[cfg(test)]
3877 pub(crate) fn lookup_index_flights_built_for_test(&self) -> usize {
3878 self.lookup_index_cache
3879 .flights_built
3880 .load(std::sync::atomic::Ordering::Relaxed)
3881 }
3882
3883 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3884 self.lookup_index_cache.report()
3885 }
3886
3887 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3888 let start_row = view.start_row();
3889 let end_row = view.end_row();
3890 let start_col = view.start_col();
3891 let end_col = view.end_col();
3892 for row in start_row..=end_row {
3893 let Ok(row_u32) = u32::try_from(row) else {
3894 return true;
3895 };
3896 for col in start_col..=end_col {
3897 let Ok(col_u32) = u32::try_from(col) else {
3898 return true;
3899 };
3900 let cell_ref = self
3901 .graph
3902 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3903 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3904 && self.graph.is_volatile(vertex_id)
3905 {
3906 return true;
3907 }
3908 }
3909 }
3910 false
3911 }
3912
3913 fn build_lookup_index_impl(
3914 &self,
3915 view: &RangeView<'_>,
3916 axis: LookupAxis,
3917 ) -> Option<Arc<LookupIndex>> {
3918 let (rows, cols) = view.dims();
3919 if rows == 0 || cols == 0 {
3920 self.lookup_index_cache.note_skipped_tiny();
3921 return None;
3922 }
3923 let len = match axis {
3924 LookupAxis::ColumnInView(col) => {
3925 if col >= cols {
3926 self.lookup_index_cache.note_skipped_tiny();
3927 return None;
3928 }
3929 rows
3930 }
3931 LookupAxis::RowInView(row) => {
3932 if row >= rows {
3933 self.lookup_index_cache.note_skipped_tiny();
3934 return None;
3935 }
3936 cols
3937 }
3938 };
3939 if len < 64 {
3940 self.lookup_index_cache.note_skipped_tiny();
3941 return None;
3942 }
3943
3944 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3945 let key = LookupIndexKey {
3946 sheet_id,
3947 start_row: u32::try_from(view.start_row()).ok()?,
3948 start_col: u32::try_from(view.start_col()).ok()?,
3949 end_row: u32::try_from(view.end_row()).ok()?,
3950 end_col: u32::try_from(view.end_col()).ok()?,
3951 axis,
3952 snapshot_id: self.data_snapshot_id(),
3953 };
3954 if let Some(index) = self.lookup_index_cache.get(&key) {
3955 return Some(index);
3956 }
3957 if self
3958 .lookup_index_cache
3959 .would_exceed_cap(estimate_bytes(len, 0))
3960 {
3961 self.lookup_index_cache.note_skipped_cap();
3962 return None;
3963 }
3964 if !self.lookup_index_cache.should_build(key) {
3965 return None;
3966 }
3967 self.lookup_index_cache.single_flight(key, || {
3969 if let Some(index) = self.lookup_index_cache.recheck(&key) {
3970 return Some(index);
3971 }
3972 if self.lookup_index_cache.is_known_volatile(&key) {
3973 self.lookup_index_cache.note_skipped_volatile();
3974 return None;
3975 }
3976 if self.lookup_view_contains_volatile(view, sheet_id) {
3977 self.lookup_index_cache.note_volatile_key(key);
3978 self.lookup_index_cache.note_skipped_volatile();
3979 return None;
3980 }
3981 #[cfg(test)]
3982 self.lookup_index_cache
3983 .flights_built
3984 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
3985 match LookupIndex::build(view, axis, self.config.date_system).ok()? {
3986 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3987 BuildOutcome::ErrorInLookupAxis => {
3988 self.lookup_index_cache.note_skipped_error();
3989 None
3990 }
3991 BuildOutcome::Degenerate => {
3992 self.lookup_index_cache.note_skipped_tiny();
3993 None
3994 }
3995 }
3996 })
3997 }
3998
3999 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4000 if !self.config.enable_virtual_dep_telemetry {
4001 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4002 fallback_mode_activations: self.virtual_dep_fallback_activations,
4003 ..VirtualDepTelemetry::default()
4004 };
4005 }
4006 }
4007
4008 fn source_cache_session(&self) -> SourceCacheSession {
4009 self.clear_source_cache();
4010 SourceCacheSession {
4011 cache: self.source_cache.clone(),
4012 }
4013 }
4014
4015 fn resolve_source_scalar_cached(
4016 &self,
4017 name: &str,
4018 version: Option<u64>,
4019 ) -> Result<LiteralValue, ExcelError> {
4020 let key = (name.to_string(), version);
4021 if let Ok(mut g) = self.source_cache.write() {
4022 if let Some(v) = g.scalars.get(&key) {
4023 return Ok(v.clone());
4024 }
4025
4026 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4027 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4028 ExcelError::new(ExcelErrorKind::Ref)
4029 .with_message(format!("Unresolved source scalar: {name}"))
4030 } else {
4031 err
4032 }
4033 })?;
4034 g.scalars.insert(key, v.clone());
4035 Ok(v)
4036 } else {
4037 self.resolver.resolve_source_scalar(name).map_err(|err| {
4038 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4039 ExcelError::new(ExcelErrorKind::Ref)
4040 .with_message(format!("Unresolved source scalar: {name}"))
4041 } else {
4042 err
4043 }
4044 })
4045 }
4046 }
4047
4048 fn resolve_source_table_cached(
4049 &self,
4050 name: &str,
4051 version: Option<u64>,
4052 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4053 let key = (name.to_string(), version);
4054 if let Ok(mut g) = self.source_cache.write() {
4055 if let Some(t) = g.tables.get(&key) {
4056 return Ok(t.clone());
4057 }
4058
4059 let t = self.resolver.resolve_source_table(name).map_err(|err| {
4060 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4061 ExcelError::new(ExcelErrorKind::Ref)
4062 .with_message(format!("Unresolved source table: {name}"))
4063 } else {
4064 err
4065 }
4066 })?;
4067 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4068 g.tables.insert(key, t.clone());
4069 Ok(t)
4070 } else {
4071 self.resolver
4072 .resolve_source_table(name)
4073 .map_err(|err| {
4074 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4075 ExcelError::new(ExcelErrorKind::Ref)
4076 .with_message(format!("Unresolved source table: {name}"))
4077 } else {
4078 err
4079 }
4080 })
4081 .map(Arc::from)
4082 }
4083 }
4084
4085 fn source_table_to_range_view(
4086 &self,
4087 table: &dyn crate::traits::Table,
4088 spec: &Option<formualizer_parse::parser::TableSpecifier>,
4089 ) -> Result<RangeView<'static>, ExcelError> {
4090 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4091
4092 let owned = match spec {
4093 Some(TableSpecifier::Column(c)) => {
4094 let c = c.trim();
4095 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4096 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4097 "Complex structured references not yet supported".to_string(),
4098 ));
4099 }
4100 table.get_column(c)?.materialise().into_owned()
4101 }
4102 Some(TableSpecifier::ColumnRange(start, end)) => {
4103 let cols = table.columns();
4104 let start = start.trim();
4105 let end = end.trim();
4106 let start_key = start.to_lowercase();
4107 let end_key = end.to_lowercase();
4108 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4109 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4110 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4111 if si > ei {
4112 std::mem::swap(&mut si, &mut ei);
4113 }
4114 let h = table.data_height();
4115 let w = ei - si + 1;
4116 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4117 for (offset, ci) in (si..=ei).enumerate() {
4118 let cname = &cols[ci];
4119 let col_range = table.get_column(cname)?;
4120 let (rh, _) = col_range.dimensions();
4121 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4122 row[offset] = col_range.get(r, 0)?;
4123 }
4124 }
4125 rows
4126 } else {
4127 return Err(ExcelError::new(ExcelErrorKind::Ref)
4128 .with_message("Column range refers to unknown column(s)".to_string()));
4129 }
4130 }
4131 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4132 | Some(TableSpecifier::Headers) => table
4133 .headers_row()
4134 .map(|r| r.materialise().into_owned())
4135 .unwrap_or_default(),
4136 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4137 | Some(TableSpecifier::Totals) => table
4138 .totals_row()
4139 .map(|r| r.materialise().into_owned())
4140 .unwrap_or_default(),
4141 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4142 table
4143 .data_body()
4144 .map(|r| r.materialise().into_owned())
4145 .unwrap_or_default()
4146 }
4147 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4148 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4149 if let Some(h) = table.headers_row() {
4150 out.extend(h.iter_rows());
4151 }
4152 if let Some(body) = table.data_body() {
4153 out.extend(body.iter_rows());
4154 }
4155 if let Some(tr) = table.totals_row() {
4156 out.extend(tr.iter_rows());
4157 }
4158 out
4159 }
4160 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4161 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4162 "@ (This Row) requires table-aware context; not yet supported".to_string(),
4163 ));
4164 }
4165 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4166 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4167 .with_message("Complex structured references not yet supported".to_string()));
4168 }
4169 None => {
4170 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4171 .with_message("Table reference without specifier is unsupported".to_string()));
4172 }
4173 };
4174
4175 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4176 }
4177
4178 pub fn default_sheet_id(&self) -> SheetId {
4179 self.graph.default_sheet_id()
4180 }
4181
4182 pub fn default_sheet_name(&self) -> &str {
4183 self.graph.default_sheet_name()
4184 }
4185
4186 pub fn set_workbook_seed(&mut self, seed: u64) {
4188 self.config.workbook_seed = seed;
4189 }
4190
4191 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4193 self.config.volatile_level = level;
4194 }
4195
4196 pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4198 self.config.temporal_egress = policy;
4199 }
4200
4201 pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4202 self.config.temporal_egress
4203 }
4204
4205 pub fn set_deterministic_mode(
4207 &mut self,
4208 mode: crate::engine::DeterministicMode,
4209 ) -> Result<(), ExcelError> {
4210 let clock = mode.build_clock()?;
4211 self.config.deterministic_mode = mode;
4212 self.clock = crate::timezone::SnapshotClock::new(clock);
4213 Ok(())
4214 }
4215
4216 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4224 self.clock = crate::timezone::SnapshotClock::new(clock);
4225 }
4226
4227 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4228 self.config.deterministic_mode.validate()
4229 }
4230
4231 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4232 self.graph.sheet_id(name)
4233 }
4234
4235 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4236 self.add_sheet(name)
4237 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4238 }
4239
4240 pub fn sheet_name(&self, id: SheetId) -> &str {
4241 self.graph.sheet_name(id)
4242 }
4243
4244 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4245 let id = self.graph.add_sheet(name)?;
4246 self.ensure_arrow_sheet(name);
4247 self.mark_topology_edited();
4248 Ok(id)
4249 }
4250
4251 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4252 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4253 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4254 })?;
4255 if new_name.is_empty() || new_name.len() > 255 {
4256 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4257 }
4258 if self.graph.sheet_id(new_name).is_some() {
4259 return Err(ExcelError::new(ExcelErrorKind::Value)
4260 .with_message(format!("Sheet '{new_name}' already exists")));
4261 }
4262 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4263
4264 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4265 let mut copied_sheet = source_sheet;
4266 copied_sheet.name = Arc::<str>::from(new_name);
4267 self.arrow_sheets.sheets.push(copied_sheet);
4268 } else {
4269 self.ensure_arrow_sheet(new_name);
4270 }
4271
4272 let duplicated_formulas = self
4273 .graph
4274 .formula_vertices()
4275 .into_iter()
4276 .filter(|vertex| {
4277 self.graph
4278 .get_cell_ref(*vertex)
4279 .is_some_and(|cell| cell.sheet_id == new_id)
4280 })
4281 .collect::<Vec<_>>();
4282 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4283 self.mark_topology_edited();
4284 Ok(new_id)
4285 }
4286
4287 fn ensure_arrow_sheet(&mut self, name: &str) {
4288 if self.arrow_sheets.sheet(name).is_some() {
4289 return;
4290 }
4291 self.arrow_sheets
4292 .sheets
4293 .push(crate::arrow_store::ArrowSheet {
4294 name: std::sync::Arc::<str>::from(name),
4295 date_system: self.config.date_system,
4296 columns: Vec::new(),
4297 nrows: 0,
4298 chunk_starts: Vec::new(),
4299 chunk_rows: 32 * 1024,
4300 });
4301 }
4302
4303 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4304 let name = self.graph.sheet_name(sheet_id).to_string();
4305 self.purge_derived_formats_for_sheet(sheet_id);
4306 self.graph.remove_sheet(sheet_id)?;
4307 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4308 self.clear_all_computed_overlays();
4312 self.mark_all_formula_vertices_dirty();
4313 self.clear_staged_formulas_for_sheet(&name);
4314 if self.row_visibility.remove(&sheet_id).is_some() {
4315 self.invalidate_row_visibility_mask_cache();
4316 }
4317 self.record_structural_change(StructuralScope::RemovedSheet(sheet_id));
4318 self.mark_topology_edited();
4319 Ok(())
4320 }
4321
4322 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4324 if let Some(asheet) = self
4325 .arrow_sheets
4326 .sheets
4327 .iter_mut()
4328 .find(|s| s.name.as_ref() == target_name)
4329 {
4330 asheet.name = std::sync::Arc::<str>::from(new_name);
4331 return true;
4332 }
4333 false
4334 }
4335
4336 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4337 let old_name = self.graph.sheet_name(sheet_id).to_string();
4338
4339 self.rename_sheet_in_arrow_store(&old_name, new_name);
4342
4343 match self.graph.rename_sheet(sheet_id, new_name) {
4345 Ok(_) => {
4346 self.rename_staged_formula_sheet(&old_name, new_name);
4347 let sheet_vertices: Vec<VertexId> = self
4349 .graph
4350 .grid_vertices_in_sheet(sheet_id)
4351 .map(|(id, _)| id)
4352 .collect();
4353 for v_id in sheet_vertices {
4354 self.graph.mark_vertex_dirty(v_id);
4355 }
4356 self.mark_topology_edited();
4358 Ok(())
4359 }
4360 Err(e) => {
4361 self.rename_sheet_in_arrow_store(new_name, &old_name);
4363 Err(e)
4364 }
4365 }
4366 }
4367
4368 pub fn named_ranges_iter(
4369 &self,
4370 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4371 self.graph.named_ranges_iter()
4372 }
4373
4374 pub fn sheet_named_ranges_iter(
4375 &self,
4376 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4377 self.graph.sheet_named_ranges_iter()
4378 }
4379
4380 pub fn resolve_name_entry(
4381 &self,
4382 name: &str,
4383 current_sheet: SheetId,
4384 ) -> Option<&crate::engine::named_range::NamedRange> {
4385 self.graph.resolve_name_entry(name, current_sheet)
4386 }
4387
4388 pub(crate) fn name_query_scope(
4398 &self,
4399 scope_sheet: Option<&str>,
4400 ) -> Result<NameScope, ExcelError> {
4401 match scope_sheet {
4402 None => Ok(NameScope::Workbook),
4403 Some(sheet) => self
4404 .graph
4405 .sheet_id(sheet)
4406 .map(NameScope::Sheet)
4407 .ok_or_else(|| {
4408 ExcelError::new(ExcelErrorKind::Ref)
4409 .with_message(format!("name scope sheet not found: {sheet}"))
4410 }),
4411 }
4412 }
4413
4414 fn resolve_sheet_locator(
4427 &self,
4428 locator: &crate::reference::SharedSheetLocator<'_>,
4429 context_sheet: SheetId,
4430 ) -> Result<SheetId, ExcelError> {
4431 self.graph
4432 .sheet_reg()
4433 .resolve_locator(locator, context_sheet)
4434 }
4435
4436 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4437 let Ok(scope) = self.name_query_scope(scope_sheet) else {
4438 return false;
4439 };
4440 self.graph
4441 .resolve_name_entry_in_scope(name, scope)
4442 .is_some()
4443 }
4444
4445 pub fn resolved_name_value(
4449 &self,
4450 name: &str,
4451 scope_sheet: Option<&str>,
4452 ) -> Option<LiteralValue> {
4453 let scope = self.name_query_scope(scope_sheet).ok()?;
4454 let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4455 self.graph.get_value(entry.vertex)
4456 }
4457
4458 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4459 let entry = self.graph.resolve_table_entry(name)?;
4460 Some(TableMetadata {
4461 name: entry.name.clone(),
4462 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4463 start_row: entry.range.start.coord.row() + 1,
4464 start_col: entry.range.start.coord.col() + 1,
4465 end_row: entry.range.end.coord.row() + 1,
4466 end_col: entry.range.end.coord.col() + 1,
4467 header_row: entry.header_row,
4468 headers: entry.headers.clone(),
4469 totals_row: entry.totals_row,
4470 })
4471 }
4472
4473 pub fn tables(&self) -> Vec<TableMetadata> {
4475 self.graph
4476 .table_names()
4477 .into_iter()
4478 .filter_map(|name| self.table_metadata(&name))
4479 .collect()
4480 }
4481
4482 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4483 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4484
4485 for (name, named) in self.graph.named_ranges_iter() {
4486 out.push(crate::engine::named_range::NamedRangeSnapshot {
4487 name: name.clone(),
4488 scope: NameScope::Workbook,
4489 definition: named.definition.clone(),
4490 });
4491 }
4492
4493 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4494 out.push(crate::engine::named_range::NamedRangeSnapshot {
4495 name: name.clone(),
4496 scope: NameScope::Sheet(*sheet_id),
4497 definition: named.definition.clone(),
4498 });
4499 }
4500
4501 out.sort_by(|a, b| {
4502 let a_scope = match a.scope {
4503 NameScope::Workbook => (0u8, 0u32),
4504 NameScope::Sheet(id) => (1u8, u32::from(id)),
4505 };
4506 let b_scope = match b.scope {
4507 NameScope::Workbook => (0u8, 0u32),
4508 NameScope::Sheet(id) => (1u8, u32::from(id)),
4509 };
4510 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4511 });
4512
4513 out
4514 }
4515
4516 pub fn named_ranges_snapshot_for_sheet(
4517 &self,
4518 sheet_id: SheetId,
4519 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4520 self.named_ranges_snapshot()
4521 .into_iter()
4522 .filter(|entry| match entry.scope {
4523 NameScope::Workbook => true,
4524 NameScope::Sheet(id) => id == sheet_id,
4525 })
4526 .collect()
4527 }
4528
4529 pub fn define_name(
4530 &mut self,
4531 name: &str,
4532 definition: NamedDefinition,
4533 scope: NameScope,
4534 ) -> Result<(), ExcelError> {
4535 self.graph.validate_define_name(name, scope)?;
4536 self.graph.define_name(name, definition, scope)?;
4537 self.record_structural_change(StructuralScope::AllSheets);
4538
4539 self.mark_topology_edited();
4540
4541 Ok(())
4542 }
4543
4544 pub fn update_name(
4545 &mut self,
4546 name: &str,
4547 definition: NamedDefinition,
4548 scope: NameScope,
4549 ) -> Result<(), ExcelError> {
4550 self.graph.validate_existing_name(name, scope)?;
4551 self.graph.update_name(name, definition, scope)?;
4552 self.record_structural_change(StructuralScope::AllSheets);
4553
4554 self.mark_topology_edited();
4555
4556 Ok(())
4557 }
4558
4559 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4560 self.graph.validate_existing_name(name, scope)?;
4561 self.graph.delete_name(name, scope)?;
4562 self.record_structural_change(StructuralScope::AllSheets);
4563
4564 self.mark_topology_edited();
4565
4566 Ok(())
4567 }
4568
4569 pub fn define_table(
4570 &mut self,
4571 name: &str,
4572 range: crate::reference::RangeRef,
4573 header_row: bool,
4574 headers: Vec<String>,
4575 totals_row: bool,
4576 ) -> Result<(), ExcelError> {
4577 self.graph
4578 .define_table(name, range, header_row, headers, totals_row)?;
4579 self.record_structural_change(StructuralScope::AllSheets);
4580 self.mark_topology_edited();
4581 Ok(())
4582 }
4583
4584 pub fn define_source_scalar(
4585 &mut self,
4586 name: &str,
4587 version: Option<u64>,
4588 ) -> Result<(), ExcelError> {
4589 self.graph.define_source_scalar(name, version)?;
4590 self.record_structural_change(StructuralScope::OpaqueGlobal);
4591 self.mark_topology_edited();
4592 Ok(())
4593 }
4594
4595 pub fn define_source_table(
4596 &mut self,
4597 name: &str,
4598 version: Option<u64>,
4599 ) -> Result<(), ExcelError> {
4600 self.graph.define_source_table(name, version)?;
4601 self.record_structural_change(StructuralScope::OpaqueGlobal);
4602 self.mark_topology_edited();
4603 Ok(())
4604 }
4605
4606 pub fn set_source_scalar_version(
4607 &mut self,
4608 name: &str,
4609 version: Option<u64>,
4610 ) -> Result<(), ExcelError> {
4611 self.graph.set_source_scalar_version(name, version)?;
4612 Ok(())
4613 }
4614
4615 pub fn set_source_table_version(
4616 &mut self,
4617 name: &str,
4618 version: Option<u64>,
4619 ) -> Result<(), ExcelError> {
4620 self.graph.set_source_table_version(name, version)?;
4621 Ok(())
4622 }
4623
4624 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4625 self.graph.invalidate_source(name)?;
4626 Ok(())
4627 }
4628
4629 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4630 self.graph.get_value(vertex)
4631 }
4632
4633 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4634 self.graph.get_cell_value(sheet, row, col)
4635 }
4636
4637 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4638 self.graph.get_vertex_for_cell(cell)
4639 }
4640
4641 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4642 self.graph.get_evaluation_vertices()
4643 }
4644
4645 pub fn baseline_stats(&self) -> EngineBaselineStats {
4647 let graph = self.graph.baseline_stats();
4648 EngineBaselineStats {
4649 graph_vertex_count: graph.graph_vertex_count,
4650 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4651 graph_edge_count: graph.graph_edge_count,
4652 dirty_vertex_count: graph.dirty_vertex_count,
4653 evaluation_vertex_count: graph.evaluation_vertex_count,
4654 formula_ast_root_count: graph.formula_ast_root_count,
4655 formula_ast_node_count: graph.formula_ast_node_count,
4656 staged_formula_count: self.staged_formula_count(),
4657 formula_plane_active_span_count: 0,
4658 formula_plane_producer_result_entries: 0,
4659 formula_plane_consumer_read_entries: 0,
4660 formula_plane_mixed_topology_cache_builds: 0,
4661 formula_plane_mixed_topology_cache_hits: 0,
4662 formula_plane_mixed_topology_cache_overflows: 0,
4663 formula_plane_dirty_pending_events: 0,
4664 formula_plane_dirty_region_events_recorded: 0,
4665 formula_plane_dirty_span_region_events_recorded: 0,
4666 formula_plane_dirty_whole_span_seeds_recorded: 0,
4667 formula_plane_dirty_global_invalidations: 0,
4668 formula_plane_structural_span_candidates: 0,
4669 formula_plane_cycle_member_span_demotions: 0,
4670 formula_plane_array_result_span_demotions: 0,
4671 retained_scc_members: self.retained_scc_members.len(),
4672 }
4673 }
4674
4675 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4677 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4678 }
4679
4680 #[cfg(test)]
4681 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4682 self.used_axis_bounds_cache
4683 .read()
4684 .ok()
4685 .and_then(|guard| {
4686 guard.as_ref().map(|cache| {
4687 (
4688 cache.row_hits.load(Ordering::Relaxed),
4689 cache.row_misses.load(Ordering::Relaxed),
4690 cache.col_hits.load(Ordering::Relaxed),
4691 cache.col_misses.load(Ordering::Relaxed),
4692 )
4693 })
4694 })
4695 .unwrap_or((0, 0, 0, 0))
4696 }
4697
4698 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4699 self.graph.set_first_load_assume_new(enabled);
4700 }
4701
4702 pub fn first_load_assume_new(&self) -> bool {
4703 self.graph.first_load_assume_new()
4704 }
4705
4706 pub fn reset_ensure_touched(&mut self) {
4707 self.graph.reset_ensure_touched();
4708 }
4709
4710 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4711 self.graph.finalize_sheet_index(sheet);
4712 }
4713
4714 pub fn action<T>(
4723 &mut self,
4724 name: impl AsRef<str>,
4725 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4726 ) -> Result<T, crate::engine::EditorError> {
4727 if self.action_depth != 0 {
4728 return Err(crate::engine::EditorError::TransactionFailed {
4729 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4730 .to_string(),
4731 });
4732 }
4733
4734 self.action_depth = 1;
4735 let engine_ptr: *mut Engine<R> = self;
4736 let _guard = ActionDepthGuard {
4737 engine: engine_ptr,
4738 _marker: std::marker::PhantomData,
4739 };
4740
4741 let mut tx = EngineAction {
4742 engine: self,
4743 name: name.as_ref().to_string(),
4744 capture: None,
4745 arrow_undo: None,
4746 atomic_policy: false,
4747 };
4748 f(&mut tx)
4749 }
4750
4751 pub fn action_atomic<T>(
4755 &mut self,
4756 name: impl Into<String>,
4757 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4758 ) -> Result<T, crate::engine::EditorError> {
4759 let (v, _j) = self.action_atomic_journal(name, f)?;
4760 Ok(v)
4761 }
4762
4763 pub fn action_atomic_journal<T>(
4765 &mut self,
4766 name: impl Into<String>,
4767 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4768 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4769 if self.action_depth != 0 {
4770 return Err(crate::engine::EditorError::TransactionFailed {
4771 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4772 .to_string(),
4773 });
4774 }
4775
4776 self.action_depth = 1;
4777 let engine_ptr: *mut Engine<R> = self;
4778 let _guard = ActionDepthGuard {
4779 engine: engine_ptr,
4780 _marker: std::marker::PhantomData,
4781 };
4782
4783 let name_str = name.into();
4784 let mut capture = MutationCapture::new(Default::default());
4785 let start_len = capture.len();
4786 self.action_atomic_impl(&mut capture, start_len, true, name_str, f)
4787 }
4788
4789 fn action_atomic_impl<T>(
4790 &mut self,
4791 capture: &mut MutationCapture,
4792 start_len: usize,
4793 expand_runs: bool,
4794 name: String,
4795 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4796 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4797 let invalidation_baseline = self.invalidation_baseline();
4798 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4799 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4800
4801 let capture_ptr: *mut MutationCapture = capture;
4802 let mut tx = EngineAction {
4803 engine: self,
4804 name: name.clone(),
4805 capture: Some(capture_ptr),
4806 arrow_undo: Some(arrow_ptr),
4807 atomic_policy: true,
4808 };
4809
4810 let res = f(&mut tx);
4811
4812 let capture_ref = unsafe { &*capture_ptr };
4819 let has_runs = capture_ref.lazy_len() > 0;
4820 let graph_events: Vec<crate::engine::ChangeEvent> = if expand_runs || res.is_err() {
4821 capture_ref.expanded_events_from(start_len, 0)
4822 } else {
4823 capture_ref.events()[start_len..].to_vec()
4824 };
4825 let graph_batch = crate::engine::GraphUndoBatch {
4826 events: graph_events,
4827 };
4828 let affected_cells = arrow_undo.ops.len();
4829 let journal = crate::engine::ActionJournal {
4830 name,
4831 graph: graph_batch,
4832 arrow: arrow_undo,
4833 affected_cells,
4834 };
4835
4836 match res {
4837 Ok(v) => {
4838 if !journal.graph.is_empty() || !journal.arrow.is_empty() || has_runs {
4839 for event in &journal.graph.events {
4840 self.record_change_for_event(event);
4841 }
4842 let mut impact = Self::classify_change_events(
4843 &journal.graph.events,
4844 LoggedEditDirection::Original,
4845 )
4846 .max(Self::classify_arrow_undo(&journal.arrow));
4847 if has_runs {
4848 impact = impact.max(LoggedEditImpact::Topology);
4849 }
4850 self.apply_logged_edit_impact(impact, invalidation_baseline);
4851 }
4852 Ok((v, journal))
4853 }
4854 Err(e) => {
4855 if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
4856 {
4857 return Err(crate::engine::EditorError::TransactionFailed {
4858 reason: format!(
4859 "Engine::action_atomic rollback failed after error '{e}': {rb}"
4860 ),
4861 });
4862 }
4863 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4864 for event in &journal.graph.events {
4865 self.record_change_for_event(event);
4866 }
4867 }
4868 Err(e)
4869 }
4870 }
4871 }
4872
4873 pub fn action_with_logger<T>(
4880 &mut self,
4881 log: &mut crate::engine::ChangeLog,
4882 name: impl AsRef<str>,
4883 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4884 ) -> Result<T, crate::engine::EditorError> {
4885 if self.action_depth != 0 {
4886 return Err(crate::engine::EditorError::TransactionFailed {
4887 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4888 .to_string(),
4889 });
4890 }
4891
4892 self.action_depth = 1;
4893 let engine_ptr: *mut Engine<R> = self;
4894 let _guard = ActionDepthGuard {
4895 engine: engine_ptr,
4896 _marker: std::marker::PhantomData,
4897 };
4898
4899 let name_str = name.as_ref().to_string();
4900 let mut capture = MutationCapture::new(log.current_meta());
4901 let start_len = capture.len();
4902 capture.begin_compound(name_str.clone());
4903
4904 let res = self.action_atomic_impl(&mut capture, start_len, false, name_str, f);
4907 capture.close_compounds();
4908
4909 match res {
4910 Ok((v, _journal)) => {
4911 log.publish_capture(capture);
4912 Ok(v)
4913 }
4914 Err(e) => {
4915 log.discard_capture(capture);
4917 Err(e)
4918 }
4919 }
4920 }
4921
4922 fn rollback_from_action_journal(
4923 &mut self,
4924 journal: &crate::engine::ActionJournal,
4925 invalidation_baseline: InvalidationBaseline,
4926 ) -> Result<(), crate::engine::EditorError> {
4927 self.invalidate_for_action_journal(
4930 journal,
4931 LoggedEditDirection::InverseReplay,
4932 invalidation_baseline,
4933 );
4934 journal.graph.undo(&mut self.graph)?;
4936 self.apply_inverse_row_visibility_events(&journal.graph.events);
4938 self.apply_arrow_undo_batch(&journal.arrow, true);
4940 Ok(())
4941 }
4942
4943 fn rollback_from_change_events(
4944 &mut self,
4945 events: &[crate::engine::ChangeEvent],
4946 invalidation_baseline: InvalidationBaseline,
4947 ) -> Result<(), crate::engine::EditorError> {
4948 use crate::engine::ChangeEvent;
4949
4950 self.invalidate_for_change_events(
4953 events,
4954 LoggedEditDirection::InverseReplay,
4955 invalidation_baseline,
4956 );
4957
4958 self.graph
4960 .authority_set_replay(crate::engine::authority::history::Replay::Undo);
4961 let rolled_back = (|| {
4962 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4963 let mut compound_stack: Vec<usize> = Vec::new();
4964 for (i, ev) in events.iter().enumerate().rev() {
4965 match ev {
4966 ChangeEvent::CompoundEnd { depth } => {
4967 compound_stack.push(*depth);
4968 if let Some(description) =
4969 crate::engine::graph::editor::change_log::compound_start_description(
4970 i,
4971 |j| &events[j],
4972 )
4973 {
4974 editor.inverse_compound_end(description);
4975 }
4976 }
4977 ChangeEvent::CompoundStart { depth, .. } => {
4978 if compound_stack.last() == Some(depth) {
4979 compound_stack.pop();
4980 }
4981 editor.apply_inverse(ev.clone())?;
4982 }
4983 ChangeEvent::SetRowVisibility { .. } => {
4984 }
4986 _ => {
4987 editor.apply_inverse(ev.clone())?;
4988 }
4989 }
4990 }
4991 Ok::<_, crate::engine::EditorError>(())
4992 })();
4993 self.graph
4994 .authority_set_replay(crate::engine::authority::history::Replay::Forward);
4995 rolled_back?;
4996
4997 for ev in events.iter().rev() {
4999 self.apply_inverse_row_visibility_event(ev);
5000 }
5001
5002 for ev in events.iter().rev() {
5004 self.mirror_inverse_change_to_arrow(ev);
5005 }
5006
5007 Ok(())
5008 }
5009
5010 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5011 let sheet_id = self.graph.sheet_id(sheet)?;
5012 let coord = Coord::from_excel(row, col, true, true);
5013 let cell = CellRef::new(sheet_id, coord);
5014 let vid = self.graph.get_vertex_for_cell(&cell)?;
5015 self.graph.get_formula(vid)
5016 }
5017
5018 pub fn define_name_with_logger(
5019 &mut self,
5020 log: &mut crate::engine::ChangeLog,
5021 name: &str,
5022 definition: NamedDefinition,
5023 scope: NameScope,
5024 ) -> Result<(), crate::engine::EditorError> {
5025 self.graph
5026 .validate_define_name(name, scope)
5027 .map_err(crate::engine::EditorError::Excel)?;
5028
5029 {
5030 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5031 editor.define_name(name, definition, scope)?;
5032 }
5033 self.record_structural_change(StructuralScope::AllSheets);
5034
5035 self.mark_topology_edited();
5036
5037 Ok(())
5038 }
5039
5040 pub fn update_name_with_logger(
5041 &mut self,
5042 log: &mut crate::engine::ChangeLog,
5043 name: &str,
5044 definition: NamedDefinition,
5045 scope: NameScope,
5046 ) -> Result<(), crate::engine::EditorError> {
5047 self.graph
5048 .validate_existing_name(name, scope)
5049 .map_err(crate::engine::EditorError::Excel)?;
5050 {
5051 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5052 editor.update_name(name, definition, scope)?;
5053 }
5054 self.record_structural_change(StructuralScope::AllSheets);
5055
5056 self.mark_topology_edited();
5057
5058 Ok(())
5059 }
5060
5061 pub fn delete_name_with_logger(
5062 &mut self,
5063 log: &mut crate::engine::ChangeLog,
5064 name: &str,
5065 scope: NameScope,
5066 ) -> Result<(), crate::engine::EditorError> {
5067 self.graph
5068 .validate_existing_name(name, scope)
5069 .map_err(crate::engine::EditorError::Excel)?;
5070 {
5071 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5072 editor.delete_name(name, scope)?;
5073 }
5074 self.record_structural_change(StructuralScope::AllSheets);
5075
5076 self.mark_topology_edited();
5077
5078 Ok(())
5079 }
5080
5081 pub fn edit_with_logger<T>(
5082 &mut self,
5083 log: &mut crate::engine::ChangeLog,
5084 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5085 ) -> Result<T, crate::engine::EditorError> {
5086 let mut capture = MutationCapture::new(log.current_meta());
5087 let result = self.edit_with_capture(&mut capture, f);
5088 capture.close_compounds();
5089 match result {
5090 Ok(value) => {
5091 log.publish_capture(capture);
5092 Ok(value)
5093 }
5094 Err(error) => {
5095 log.discard_capture(capture);
5096 Err(error)
5097 }
5098 }
5099 }
5100
5101 fn edit_with_capture<T>(
5102 &mut self,
5103 capture: &mut MutationCapture,
5104 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5105 ) -> Result<T, crate::engine::EditorError> {
5106 let invalidation_baseline = self.invalidation_baseline();
5107 let start_len = capture.len();
5108 let lazy_start = capture.lazy_len();
5109
5110 struct ArrowSpillReader<'a> {
5113 sheets: &'a crate::arrow_store::SheetStore,
5114 }
5115 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5116 fn read_cell_value(
5117 &self,
5118 sheet: &str,
5119 row: u32,
5120 col: u32,
5121 ) -> Option<formualizer_common::LiteralValue> {
5122 use formualizer_common::LiteralValue;
5123 let asheet = self.sheets.sheet(sheet)?;
5124 let r0 = row.saturating_sub(1) as usize;
5125 let c0 = col.saturating_sub(1) as usize;
5126 let v = asheet.get_cell_value(r0, c0);
5127 if matches!(v, LiteralValue::Empty) {
5128 None
5129 } else {
5130 Some(v)
5131 }
5132 }
5133 }
5134
5135 let ret = {
5136 let spill_reader = ArrowSpillReader {
5137 sheets: &self.arrow_sheets,
5138 };
5139 let mut editor = crate::engine::VertexEditor::with_capture_and_spill_reader(
5140 &mut self.graph,
5141 capture,
5142 &spill_reader,
5143 );
5144 f(&mut editor)
5145 };
5146
5147 let new_events = capture.events()[start_len..].to_vec();
5151 let new_runs = capture.lazy_len() > lazy_start;
5152 if new_events.iter().any(|event| {
5153 matches!(
5154 event,
5155 ChangeEvent::DefineName { .. }
5156 | ChangeEvent::UpdateName { .. }
5157 | ChangeEvent::DeleteName { .. }
5158 )
5159 }) {
5160 let all = capture.expanded_events_from(start_len, lazy_start);
5161 self.rollback_from_change_events(&all, invalidation_baseline)?;
5162 return Err(crate::engine::EditorError::TransactionUnsupported {
5163 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5164 });
5165 }
5166
5167 self.clear_logged_cell_format_states(&new_events);
5170 for ev in &new_events {
5171 self.mirror_forward_change_to_arrow(ev);
5172 }
5173 for ev in &new_events {
5174 self.record_change_for_event(ev);
5175 }
5176
5177 if self.action_depth == 0 {
5180 let mut impact =
5181 Self::classify_change_events(&new_events, LoggedEditDirection::Original);
5182 if new_runs {
5183 impact = impact.max(LoggedEditImpact::Topology);
5184 }
5185 self.apply_logged_edit_impact(impact, invalidation_baseline);
5186 }
5187
5188 Ok(ret)
5189 }
5190
5191 pub(crate) fn preflight_replay_admission(
5192 &mut self,
5193 events: &[ChangeEvent],
5194 forward: bool,
5195 ) -> Result<(), crate::engine::EditorError> {
5196 if !self.graph_admission_enabled() {
5197 return Ok(());
5198 }
5199 let mut vertex_delta = 0i128;
5200 let mut edge_delta = 0i128;
5201 let mut added_vertices = 0usize;
5202 let mut added_edges = 0usize;
5203 let mut formula_cells = BTreeSet::new();
5204 for event in events {
5205 match event {
5206 ChangeEvent::AddVertex {
5207 formula,
5208 coord,
5209 sheet_id,
5210 ..
5211 } => {
5212 let delta = if forward { 1 } else { -1 };
5213 vertex_delta += delta;
5214 if forward {
5215 added_vertices = added_vertices.saturating_add(1);
5216 if formula.is_some() {
5217 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5218 }
5219 }
5220 }
5221 ChangeEvent::RemoveVertex {
5222 old_formula,
5223 coord,
5224 sheet_id,
5225 ..
5226 } => {
5227 let delta = if forward { -1 } else { 1 };
5228 vertex_delta += delta;
5229 if !forward {
5230 added_vertices = added_vertices.saturating_add(1);
5231 if old_formula.is_some()
5232 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5233 {
5234 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5235 }
5236 }
5237 }
5238 ChangeEvent::EdgeAdded { .. } => {
5239 let delta = if forward { 1 } else { -1 };
5240 edge_delta += delta;
5241 if forward {
5242 added_edges = added_edges.saturating_add(1);
5243 }
5244 }
5245 ChangeEvent::EdgeRemoved { .. } => {
5246 let delta = if forward { -1 } else { 1 };
5247 edge_delta += delta;
5248 if !forward {
5249 added_edges = added_edges.saturating_add(1);
5250 }
5251 }
5252 ChangeEvent::SetFormula {
5253 addr, old_formula, ..
5254 } if (forward || old_formula.is_some()) => {
5255 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5256 }
5257 _ => {}
5258 }
5259 }
5260 let stats = self.graph.baseline_stats();
5261 let final_vertices = i128::try_from(stats.graph_vertex_count)
5262 .ok()
5263 .and_then(|count| count.checked_add(vertex_delta))
5264 .and_then(|count| usize::try_from(count).ok())
5265 .ok_or_else(|| {
5266 crate::engine::EditorError::Excel(
5267 ExcelError::new(ExcelErrorKind::NImpl)
5268 .with_message("replay vertex count overflow"),
5269 )
5270 })?;
5271 let final_edges = i128::try_from(stats.graph_edge_count)
5272 .ok()
5273 .and_then(|count| count.checked_add(edge_delta))
5274 .and_then(|count| usize::try_from(count).ok())
5275 .ok_or_else(|| {
5276 crate::engine::EditorError::Excel(
5277 ExcelError::new(ExcelErrorKind::NImpl)
5278 .with_message("replay edge count overflow"),
5279 )
5280 })?;
5281 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5282 final_vertices,
5283 final_edges,
5284 materialization_cells: formula_cells.len() as u64,
5285 added_vertices,
5286 added_edges,
5287 })
5288 .map_err(crate::engine::EditorError::Excel)
5289 }
5290
5291 pub fn undo_logged(
5296 &mut self,
5297 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5298 log: &mut crate::engine::ChangeLog,
5299 ) -> Result<(), crate::engine::EditorError> {
5300 let pending_events = log
5301 .last_group_indices()
5302 .into_iter()
5303 .map(|index| log.events()[index].clone())
5304 .collect::<Vec<_>>();
5305 self.preflight_replay_admission(&pending_events, false)?;
5306 let invalidation_baseline = self.invalidation_baseline();
5307 self.invalidate_for_change_events(
5310 &pending_events,
5311 LoggedEditDirection::InverseReplay,
5312 invalidation_baseline,
5313 );
5314 let batch = undo.undo(&mut self.graph, log)?;
5315 for item in batch.iter().rev() {
5316 self.apply_inverse_row_visibility_event(&item.event);
5317 self.apply_inverse_staged_formula_event(&item.event);
5318 }
5319 if !batch.is_empty() {
5320 let events = batch
5321 .iter()
5322 .map(|item| item.event.clone())
5323 .collect::<Vec<_>>();
5324 self.clear_logged_cell_format_states(&events);
5325 }
5326 self.mirror_undo_batch_to_arrow(&batch);
5327 if !batch.is_empty() {
5328 for item in &batch {
5329 self.record_change_for_event(&item.event);
5330 }
5331 }
5332 crate::engine::trace::fz_event!(
5333 tracing::Level::INFO,
5334 "history",
5335 "history.replay",
5336 op = "undo",
5337 events_replayed = batch.len(),
5338 ownership_resyncs = batch.len()
5339 );
5340 Ok(())
5341 }
5342
5343 pub fn redo_logged(
5344 &mut self,
5345 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5346 log: &mut crate::engine::ChangeLog,
5347 ) -> Result<(), crate::engine::EditorError> {
5348 let pending_events = undo.pending_redo_events();
5349 self.preflight_replay_admission(&pending_events, true)?;
5350 let invalidation_baseline = self.invalidation_baseline();
5351 self.invalidate_for_change_events(
5352 &pending_events,
5353 LoggedEditDirection::ForwardReplay,
5354 invalidation_baseline,
5355 );
5356 let batch = undo.redo(&mut self.graph, log)?;
5357 for item in &batch {
5358 self.apply_forward_row_visibility_event(&item.event);
5359 self.apply_forward_staged_formula_event(&item.event);
5360 }
5361 if !batch.is_empty() {
5362 let events = batch
5363 .iter()
5364 .map(|item| item.event.clone())
5365 .collect::<Vec<_>>();
5366 self.clear_logged_cell_format_states(&events);
5367 }
5368 self.mirror_redo_batch_to_arrow(&batch);
5369 if !batch.is_empty() {
5370 for item in &batch {
5371 self.record_change_for_event(&item.event);
5372 }
5373 }
5374 crate::engine::trace::fz_event!(
5375 tracing::Level::INFO,
5376 "history",
5377 "history.replay",
5378 op = "redo",
5379 events_replayed = batch.len(),
5380 ownership_resyncs = batch.len()
5381 );
5382 Ok(())
5383 }
5384
5385 pub fn undo_action(
5389 &mut self,
5390 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5391 ) -> Result<(), crate::engine::EditorError> {
5392 let Some(journal) = undo.pop_undo_action() else {
5393 return Ok(());
5394 };
5395 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5396 undo.push_done_action(journal);
5397 return Err(error);
5398 }
5399 let invalidation_baseline = self.invalidation_baseline();
5400
5401 self.invalidate_for_action_journal(
5402 &journal,
5403 LoggedEditDirection::InverseReplay,
5404 invalidation_baseline,
5405 );
5406 journal.graph.undo(&mut self.graph)?;
5407 self.apply_inverse_row_visibility_events(&journal.graph.events);
5408 self.apply_arrow_undo_batch(&journal.arrow, true);
5409 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5410 for event in &journal.graph.events {
5411 self.record_change_for_event(event);
5412 }
5413 }
5414
5415 #[cfg(feature = "tracing")]
5416 let events_replayed = journal.graph.events.len();
5417 crate::engine::trace::fz_event!(
5418 tracing::Level::INFO,
5419 "history",
5420 "history.replay",
5421 op = "undo",
5422 events_replayed,
5423 ownership_resyncs = events_replayed
5424 );
5425 undo.push_redo_action(journal);
5426 Ok(())
5427 }
5428
5429 pub fn redo_action(
5433 &mut self,
5434 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5435 ) -> Result<(), crate::engine::EditorError> {
5436 let Some(journal) = undo.pop_redo_action() else {
5437 return Ok(());
5438 };
5439 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5440 undo.push_redo_action(journal);
5441 return Err(error);
5442 }
5443 let invalidation_baseline = self.invalidation_baseline();
5444 self.invalidate_for_action_journal(
5445 &journal,
5446 LoggedEditDirection::ForwardReplay,
5447 invalidation_baseline,
5448 );
5449 journal.graph.redo(&mut self.graph)?;
5450 self.apply_forward_row_visibility_events(&journal.graph.events);
5451 self.apply_arrow_undo_batch(&journal.arrow, false);
5452 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5453 for event in &journal.graph.events {
5454 self.record_change_for_event(event);
5455 }
5456 }
5457
5458 #[cfg(feature = "tracing")]
5459 let events_replayed = journal.graph.events.len();
5460 crate::engine::trace::fz_event!(
5461 tracing::Level::INFO,
5462 "history",
5463 "history.replay",
5464 op = "redo",
5465 events_replayed,
5466 ownership_resyncs = events_replayed
5467 );
5468 undo.push_done_action(journal);
5469 Ok(())
5470 }
5471
5472 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5473 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5474 let row = addr.coord.row() + 1;
5476 let col = addr.coord.col() + 1;
5477 (sheet, row, col)
5478 }
5479
5480 fn mirror_undo_batch_to_arrow(
5481 &mut self,
5482 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5483 ) {
5484 for item in batch.iter().rev() {
5486 self.mirror_inverse_change_to_arrow(&item.event);
5487 }
5488 }
5489
5490 fn mirror_redo_batch_to_arrow(
5491 &mut self,
5492 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5493 ) {
5494 for item in batch.iter() {
5496 self.mirror_forward_change_to_arrow(&item.event);
5497 }
5498 }
5499
5500 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5501 use crate::engine::ChangeEvent;
5502 use formualizer_common::LiteralValue;
5503
5504 match ev {
5505 ChangeEvent::SetValue {
5506 addr,
5507 old_value,
5508 old_formula,
5509 ..
5510 } => {
5511 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5512 if old_formula.is_some() {
5513 self.clear_delta_overlay_cell(&sheet, row, col);
5514 } else {
5515 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5516 self.mirror_value_to_overlay(&sheet, row, col, &v);
5517 }
5518 }
5519 ChangeEvent::SetFormula {
5520 addr,
5521 old_value,
5522 old_formula,
5523 ..
5524 } => {
5525 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5526 if old_formula.is_some() {
5527 self.clear_delta_overlay_cell(&sheet, row, col);
5528 } else {
5529 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5530 self.mirror_value_to_overlay(&sheet, row, col, &v);
5531 }
5532 }
5533 ChangeEvent::SpillCommitted { old, new, .. } => {
5534 self.mirror_spill_snapshot(new, true);
5536 if let Some(snap) = old {
5537 self.mirror_spill_snapshot(snap, false);
5538 }
5539 }
5540 ChangeEvent::SpillCleared { old, .. } => {
5541 self.mirror_spill_snapshot(old, false);
5543 }
5544 ChangeEvent::SetRowVisibility { .. } => {
5545 }
5547 _ => {}
5548 }
5549 }
5550
5551 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5552 use crate::engine::ChangeEvent;
5553
5554 match ev {
5555 ChangeEvent::SetValue { addr, new, .. } => {
5556 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5557 self.mirror_value_to_overlay(&sheet, row, col, new);
5558 }
5559 ChangeEvent::SetFormula { addr, .. } => {
5560 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5561 self.clear_delta_overlay_cell(&sheet, row, col);
5562 }
5564 ChangeEvent::SpillCommitted { old, new, .. } => {
5565 if let Some(snap) = old {
5566 self.mirror_spill_snapshot(snap, true);
5567 }
5568 self.mirror_spill_snapshot(new, false);
5569 }
5570 ChangeEvent::SpillCleared { old, .. } => {
5571 self.mirror_spill_snapshot(old, true);
5572 }
5573 ChangeEvent::SetRowVisibility { .. } => {
5574 }
5576 _ => {
5577 }
5579 }
5580 }
5581
5582 fn mirror_spill_snapshot(
5583 &mut self,
5584 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5585 clear_only: bool,
5586 ) {
5587 use formualizer_common::LiteralValue;
5588
5589 let mut i = 0usize;
5590 for row in &snap.values {
5591 for v in row {
5592 if let Some(cell) = snap.target_cells.get(i) {
5593 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5594 let out = if clear_only {
5595 LiteralValue::Empty
5596 } else {
5597 v.clone()
5598 };
5599 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5600 }
5601 i += 1;
5602 }
5603 }
5604 if clear_only {
5606 for cell in snap.target_cells.iter().skip(i) {
5607 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5608 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5609 }
5610 }
5611 }
5612
5613 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5614 self.graph.set_default_sheet_by_name(name);
5615 }
5616
5617 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5618 self.graph.set_default_sheet_by_id(id);
5619 }
5620
5621 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5622 self.graph.set_sheet_index_mode(mode);
5623 }
5624
5625 #[cfg(feature = "test-support")]
5626 #[doc(hidden)]
5627 pub fn mark_all_formulas_dirty_for_test(&mut self) {
5628 self.mark_all_formula_vertices_dirty();
5629 }
5630
5631 #[cfg(feature = "test-support")]
5632 #[doc(hidden)]
5633 pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5634 criteria_mask_test_hooks::take_mask_work()
5635 }
5636
5637 #[cfg(feature = "test-support")]
5638 #[doc(hidden)]
5639 pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5640 self.lookup_index_cache.report()
5641 }
5642
5643 #[cfg(any(test, feature = "benchmark_internal"))]
5644 #[doc(hidden)]
5645 pub fn reset_recalc_reuse_probe(&mut self) {
5646 *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5647 }
5648
5649 #[cfg(any(test, feature = "benchmark_internal"))]
5650 #[doc(hidden)]
5651 pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5652 let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5653 if let Some(cached) = self.cached_static_schedule.as_ref() {
5654 let entry_bytes = |e: &CachedScheduleEntry| {
5655 std::mem::size_of::<CachedScheduleEntry>()
5656 + e.candidate_vertices.heap_bytes()
5657 + std::mem::size_of::<crate::engine::Schedule>()
5658 + 2 * std::mem::size_of::<usize>()
5659 + schedule_probe_retained_bytes(&e.schedule)
5660 };
5661 probe.schedule_retained_bytes = entry_bytes(cached)
5662 + self.recent_schedules.iter().map(entry_bytes).sum::<usize>()
5663 + self.base_schedule.as_ref().map_or(0, entry_bytes)
5664 + self.recent_schedules.capacity() * std::mem::size_of::<CachedScheduleEntry>();
5665 }
5666 probe
5667 }
5668
5669 #[cfg(test)]
5670 pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5671 self.cached_static_schedule
5672 .as_ref()
5673 .map(|cached| Arc::clone(&cached.schedule))
5674 }
5675
5676 fn clear_cached_static_schedule(&mut self) {
5677 self.cached_static_schedule = None;
5678 self.recent_schedules.clear();
5679 self.base_schedule = None;
5680 }
5681
5682 fn retain_recent_schedule(&mut self, entry: CachedScheduleEntry) {
5685 const RECENT_SCHEDULES: usize = 8;
5686 const RECENT_SCHEDULE_VERTICES: usize = 65_536;
5687 let revision = self.schedule_cache_authority_revision();
5688 let epoch = self.topology_epoch;
5689 self.recent_schedules
5690 .retain(|e| e.topology_epoch == epoch && e.authority_revision == revision);
5691 let current =
5692 |e: &CachedScheduleEntry| e.topology_epoch == epoch && e.authority_revision == revision;
5693 if self.base_schedule.as_ref().is_some_and(|b| !current(b)) {
5694 self.base_schedule = None;
5695 }
5696 let mut entry = entry;
5699 if current(&entry)
5700 && entry.candidate_vertices.len() > BASE_SCHEDULE_MIN_VERTICES
5701 && self
5702 .base_schedule
5703 .as_ref()
5704 .is_none_or(|b| entry.candidate_vertices.len() > b.candidate_vertices.len())
5705 {
5706 match self.base_schedule.replace(entry) {
5707 Some(previous) => entry = previous,
5708 None => return,
5709 }
5710 }
5711 if entry.topology_epoch != epoch
5712 || entry.authority_revision != revision
5713 || entry.candidate_vertices.len() > RECENT_SCHEDULE_VERTICES / 4
5714 {
5715 return;
5716 }
5717 self.recent_schedules.push(entry);
5718 let mut total: usize = self
5719 .recent_schedules
5720 .iter()
5721 .map(|e| e.candidate_vertices.len())
5722 .sum();
5723 while self.recent_schedules.len() > RECENT_SCHEDULES || total > RECENT_SCHEDULE_VERTICES {
5724 let oldest = self.recent_schedules.remove(0);
5725 total -= oldest.candidate_vertices.len();
5726 }
5727 }
5728
5729 fn invalidation_baseline(&self) -> InvalidationBaseline {
5730 InvalidationBaseline {
5731 snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5732 topology_epoch: self.topology_epoch,
5733 }
5734 }
5735
5736 fn classify_change_events(
5737 events: &[crate::engine::ChangeEvent],
5738 direction: LoggedEditDirection,
5739 ) -> LoggedEditImpact {
5740 use crate::engine::ChangeEvent;
5741
5742 events
5743 .iter()
5744 .map(|event| match event {
5745 ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5746 LoggedEditImpact::NoOp
5747 }
5748 ChangeEvent::SetRowVisibility { .. }
5749 | ChangeEvent::SetValue {
5750 old_formula: None,
5751 old_value: Some(_),
5752 ..
5753 } => LoggedEditImpact::DataOnly,
5754 ChangeEvent::SetValue {
5758 old_formula: None,
5759 old_value: None,
5760 ..
5761 } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5762 ChangeEvent::SetValue { .. }
5763 | ChangeEvent::SetFormula { .. }
5764 | ChangeEvent::AddVertex { .. }
5765 | ChangeEvent::RemoveVertex { .. }
5766 | ChangeEvent::VertexMoved { .. }
5767 | ChangeEvent::FormulaAdjusted { .. }
5768 | ChangeEvent::NamedRangeAdjusted { .. }
5769 | ChangeEvent::EdgeAdded { .. }
5770 | ChangeEvent::EdgeRemoved { .. }
5771 | ChangeEvent::DefineName { .. }
5772 | ChangeEvent::UpdateName { .. }
5773 | ChangeEvent::DeleteName { .. }
5774 | ChangeEvent::SpillCommitted { .. }
5775 | ChangeEvent::SpillCleared { .. }
5776 | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5777 })
5778 .max()
5779 .unwrap_or(LoggedEditImpact::NoOp)
5780 }
5781
5782 fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5783 use crate::engine::ArrowOp;
5784
5785 arrow
5786 .ops
5787 .iter()
5788 .map(|op| match op {
5789 ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5790 LoggedEditImpact::DataOnly
5791 }
5792 ArrowOp::RestoreComputedRect { .. }
5793 | ArrowOp::InsertRows { .. }
5794 | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5795 })
5796 .max()
5797 .unwrap_or(LoggedEditImpact::NoOp)
5798 }
5799
5800 fn apply_logged_edit_impact(
5801 &mut self,
5802 impact: LoggedEditImpact,
5803 baseline: InvalidationBaseline,
5804 ) {
5805 match impact {
5806 LoggedEditImpact::NoOp => {}
5807 LoggedEditImpact::DataOnly => {
5808 if self.topology_epoch == baseline.topology_epoch
5809 && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
5810 == baseline.snapshot_id
5811 {
5812 self.mark_data_edited();
5813 }
5814 }
5815 LoggedEditImpact::Topology => {
5816 if self.topology_epoch == baseline.topology_epoch {
5819 self.mark_topology_edited();
5820 }
5821 }
5822 }
5823 }
5824
5825 fn invalidate_for_change_events(
5826 &mut self,
5827 events: &[crate::engine::ChangeEvent],
5828 direction: LoggedEditDirection,
5829 baseline: InvalidationBaseline,
5830 ) {
5831 self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
5832 }
5833
5834 fn invalidate_for_action_journal(
5835 &mut self,
5836 journal: &crate::engine::ActionJournal,
5837 direction: LoggedEditDirection,
5838 baseline: InvalidationBaseline,
5839 ) {
5840 let impact = Self::classify_change_events(&journal.graph.events, direction)
5841 .max(Self::classify_arrow_undo(&journal.arrow));
5842 self.apply_logged_edit_impact(impact, baseline);
5843 }
5844
5845 pub fn mark_data_edited(&mut self) {
5848 self.lookup_index_cache.clear();
5849 self.snapshot_id
5850 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5851 self.has_edited = true;
5852 }
5853
5854 pub fn mark_topology_edited(&mut self) {
5856 self.lookup_index_cache.clear();
5857 self.snapshot_id
5858 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5859 self.topology_epoch = self.topology_epoch.wrapping_add(1);
5860 self.graph.bump_topology_revision();
5861 self.clear_cached_static_schedule();
5862 if let Some(ledger) = self.active_resource_ledger.as_mut() {
5863 let released = ledger.account_mixed_cache(0);
5864 debug_assert!(released.is_ok());
5865 }
5866 self.has_edited = true;
5867 self.graph.authority_sync_eager();
5871 }
5872
5873 fn mark_all_formula_vertices_dirty(&mut self) {
5874 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5875 for vertex in vertices {
5876 self.graph.mark_vertex_dirty(vertex);
5877 }
5878 }
5879
5880 fn mark_moved_formula_vertices_dirty(
5881 &mut self,
5882 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5883 ) {
5884 for vertex in &summary.vertices_moved {
5885 if self.graph.has_formula(*vertex) {
5886 self.graph.mark_vertex_dirty(*vertex);
5887 }
5888 }
5889 }
5890
5891 pub fn sheet_store(&self) -> &SheetStore {
5893 &self.arrow_sheets
5894 }
5895
5896 pub(crate) fn has_row_visibility_state(&self) -> bool {
5899 !self.row_visibility.is_empty()
5900 }
5901
5902 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5904 &mut self.arrow_sheets
5905 }
5906
5907 pub fn has_staged_formulas(&self) -> bool {
5908 !self.staged_formulas.is_empty()
5909 }
5910
5911 pub fn staged_formula_count(&self) -> usize {
5912 self.staged_formulas.values().map(StagedSheet::len).sum()
5913 }
5914
5915 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5917 self.staged_formulas
5918 .entry(sheet.to_string())
5919 .or_default()
5920 .stage(row, col, text);
5921 self.staged_formula_index.stage(sheet, row, col);
5922 if let Some(sheet) = self.graph.sheet_id(sheet) {
5923 self.invalidate_pending_spills(StructuralScope::Cell {
5924 sheet,
5925 row: row.saturating_sub(1),
5926 col: col.saturating_sub(1),
5927 });
5928 }
5929 }
5930
5931 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5932 for (row, col, _) in &staged.entries {
5933 self.staged_formula_index.remove(sheet, *row, *col);
5934 }
5935 if staged.deferred_package.is_some() {
5936 self.staged_formula_index.set_package(sheet, None);
5937 }
5938 }
5939
5940 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5941 self.staged_formulas.insert(sheet, staged);
5942 }
5943
5944 #[doc(hidden)]
5945 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5946 SourceFormulaIngress { engine: self }
5947 }
5948
5949 #[doc(hidden)]
5950 #[doc(hidden)]
5953 #[cfg(any(test, feature = "test-support"))]
5954 pub fn set_before_prepared_span_commit_hook(
5955 &mut self,
5956 hook: impl FnOnce() + Send + Sync + 'static,
5957 ) {
5958 self.before_prepared_span_commit_hook = Some(Box::new(hook));
5959 }
5960
5961 #[doc(hidden)]
5962 #[cfg(test)]
5964 pub(crate) fn set_before_target_preparation_commit_hook(
5965 &mut self,
5966 hook: impl FnOnce() + Send + Sync + 'static,
5967 ) {
5968 self.before_target_preparation_commit_hook = Some(Box::new(hook));
5969 }
5970
5971 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5972 if let Ok(replay) = package.replay.lock()
5973 && let Some(footprint) = replay.selection_cache_footprint()
5974 && !self
5975 .source_cache_footprints
5976 .iter()
5977 .any(|known| known.ptr_eq(&footprint))
5978 {
5979 self.source_cache_footprints.push(footprint);
5980 }
5981 let sheet = package.sheet_name.clone();
5982 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5983 debug_assert!(staged.deferred_package.is_none());
5984 staged.deferred_package = Some(package);
5985 staged.reconcile_attached_deferred_package();
5986 self.staged_formula_index
5987 .set_package(&sheet, staged.deferred_package.as_ref());
5988 if let Some(sheet_id) = self.graph.sheet_id(&sheet) {
5989 let package = self
5990 .staged_formulas
5991 .get(&sheet)
5992 .and_then(|staged| staged.deferred_package.as_ref());
5993 for &(vertex, anchor, region) in &self.blocked_pending_spills {
5994 if anchor.sheet_id == sheet_id
5995 && self.graph.vertex_exists(vertex)
5996 && self.graph.get_cell_ref(vertex) == Some(anchor)
5997 && self.staged_formula_index.package_occupies_spill(
5998 &sheet,
5999 (anchor.coord.row() + 1, anchor.coord.col() + 1),
6000 (
6001 region.rows.query_bounds().1 + 1,
6002 region.cols.query_bounds().1 + 1,
6003 ),
6004 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
6005 )
6006 {
6007 self.graph.mark_vertex_dirty(vertex);
6008 }
6009 }
6010 }
6011 }
6012
6013 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6014 let mut removed = None;
6015 let mut remove_sheet = false;
6016 let mut had_package = false;
6017 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6018 had_package = entries.deferred_package.is_some();
6019 removed = entries.remove(row, col);
6020 remove_sheet = entries.is_empty();
6021 }
6022 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6023 if !ordinary_removed && had_package {
6024 self.staged_formula_index.touch_package(sheet);
6025 }
6026 if remove_sheet {
6027 self.staged_formulas.remove(sheet);
6028 self.staged_formula_index.set_package(sheet, None);
6029 }
6030 if (ordinary_removed || had_package)
6031 && let Some(sheet) = self.graph.sheet_id(sheet)
6032 {
6033 self.invalidate_pending_spills(StructuralScope::Cell {
6034 sheet,
6035 row: row.saturating_sub(1),
6036 col: col.saturating_sub(1),
6037 });
6038 }
6039 removed
6040 }
6041
6042 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6043 if self.staged_formulas.remove(sheet).is_some() {
6044 self.staged_formula_index.clear_sheet(sheet);
6045 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
6046 self.invalidate_pending_spills(StructuralScope::Sheet(sheet_id));
6047 }
6048 }
6049 }
6050
6051 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6052 let Some(entries) = self.staged_formulas.remove(old) else {
6053 return;
6054 };
6055 self.staged_formula_index.clear_sheet(old);
6056 let (formulas, mut package) = entries.into_parts();
6057 for (row, col, text) in formulas {
6058 self.stage_formula_text(new, row, col, text);
6059 }
6060 if let Some(package) = package.as_mut() {
6061 package.sheet_name = new.to_string();
6062 }
6063 if let Some(package) = package {
6064 self.stage_deferred_formula_package(package);
6065 }
6066 }
6067
6068 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6070 self.staged_formulas
6071 .get(sheet)
6072 .and_then(|v| v.get(row, col))
6073 }
6074
6075 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6076 &self.formula_parse_diagnostics
6077 }
6078
6079 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6080 std::mem::take(&mut self.formula_parse_diagnostics)
6081 }
6082
6083 pub fn clear_formula_parse_diagnostics(&mut self) {
6084 self.formula_parse_diagnostics.clear();
6085 }
6086
6087 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6088 self.last_formula_ingest_report.as_ref()
6089 }
6090
6091 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6092 &self.formula_ingest_report_total
6093 }
6094
6095 #[cfg(test)]
6096 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6097 &mut self,
6098 hook: impl FnOnce() + Send + Sync + 'static,
6099 ) {
6100 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6101 }
6102
6103 #[cfg(test)]
6104 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6105 self.inject_target_semantic_stale_once_for_test = true;
6106 }
6107
6108 #[cfg(test)]
6109 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6110 self.force_virtual_dep_changes_remaining_for_test = rounds;
6111 }
6112
6113 #[cfg(test)]
6114 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6115 self.fail_evaluation_commit_preflight_once_for_test = true;
6116 }
6117
6118 #[cfg(test)]
6119 pub(crate) fn set_target_preparation_fault_for_test(
6120 &mut self,
6121 fault: crate::engine::target_preparation::TargetPreparationFault,
6122 ) {
6123 self.target_preparation_fault_for_test = Some(fault);
6124 }
6125
6126 #[cfg(test)]
6127 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6128 self.staged_formula_index.revision()
6129 }
6130
6131 #[cfg(test)]
6132 pub(crate) fn deferred_package_for_test(
6133 &self,
6134 sheet: &str,
6135 ) -> &crate::engine::DeferredFormulaPackage {
6136 self.staged_formulas
6137 .get(sheet)
6138 .unwrap()
6139 .deferred_package
6140 .as_ref()
6141 .unwrap()
6142 }
6143
6144 #[cfg(test)]
6145 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6146 let ordinary_storage = self
6147 .staged_formulas
6148 .values()
6149 .map(|sheet| sheet.entries.len())
6150 .sum::<usize>();
6151 let package_storage = self
6152 .staged_formulas
6153 .values()
6154 .filter(|sheet| sheet.deferred_package.is_some())
6155 .count();
6156 ordinary_storage == self.staged_formula_index.ordinary_count()
6157 && package_storage == self.staged_formula_index.package_count()
6158 && self.staged_formulas.iter().all(|(name, sheet)| {
6159 sheet.entries.iter().all(|(row, col, _)| {
6160 let leases = self
6161 .staged_formula_index
6162 .leases_in_region(name, *row, *col, *row, *col);
6163 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6164 })
6165 })
6166 }
6167
6168 #[cfg(test)]
6169 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6170 self.evaluation_request_begin_count_for_test
6171 }
6172
6173 #[cfg(test)]
6174 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6175 &mut self,
6176 hook: impl FnOnce() + Send + Sync + 'static,
6177 ) {
6178 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6179 }
6180
6181 #[cfg(test)]
6182 pub(crate) fn set_after_eager_proposal_commit_hook(
6183 &mut self,
6184 hook: impl FnOnce() + Send + Sync + 'static,
6185 ) {
6186 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6187 }
6188
6189 #[cfg(test)]
6190 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6191 self.topology_epoch
6192 }
6193
6194 #[cfg(test)]
6195 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6196 self.graph.topology_revision()
6197 }
6198
6199 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6200 self.formula_ingest_report_total.mode = report.mode;
6201 self.formula_ingest_report_total.accumulate(&report);
6202 self.last_formula_ingest_report = Some(report);
6203 }
6204
6205 fn collect_planning_function_requests(
6206 ast: &ASTNode,
6207 requests: &mut Vec<(String, String, usize)>,
6208 ) {
6209 match &ast.node_type {
6210 ASTNodeType::Function { name, args } => {
6211 requests.push((String::new(), name.clone(), args.len()));
6212 for arg in args {
6213 Self::collect_planning_function_requests(arg, requests);
6214 }
6215 }
6216 ASTNodeType::BinaryOp { left, right, .. } => {
6217 Self::collect_planning_function_requests(left, requests);
6218 Self::collect_planning_function_requests(right, requests);
6219 }
6220 ASTNodeType::UnaryOp { expr, .. } => {
6221 Self::collect_planning_function_requests(expr, requests);
6222 }
6223 ASTNodeType::Call { callee, args } => {
6224 Self::collect_planning_function_requests(callee, requests);
6225 for arg in args {
6226 Self::collect_planning_function_requests(arg, requests);
6227 }
6228 }
6229 ASTNodeType::Array(rows) => {
6230 for cell in rows.iter().flatten() {
6231 Self::collect_planning_function_requests(cell, requests);
6232 }
6233 }
6234 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6235 }
6236 }
6237
6238 fn prepared_function_semantics_changed(
6239 &self,
6240 preparation: &crate::engine::FormulaCompressedPreparation,
6241 guard: &crate::function_registry::SemanticEpochReadGuard,
6242 ) -> bool {
6243 if preparation.function_semantic_epoch == guard.epoch() {
6244 return false;
6245 }
6246
6247 guard
6248 .semantic_changes_affect_requests_since(preparation.function_semantic_epoch, Vec::new())
6249 }
6250
6251 pub(crate) fn prepare_source_formula_families(
6252 &mut self,
6253 sheet_name: &str,
6254 families: &[crate::engine::SourceFormulaFamily],
6255 ) -> crate::engine::FormulaCompressedPreparation {
6256 crate::engine::FormulaCompressedPreparation {
6257 engine_token: Arc::clone(&self.source_formula_token),
6258 function_semantic_epoch: crate::function_registry::semantic_epoch(),
6259 function_provider_revision: None,
6260 function_semantics_used: false,
6261 sheet_name: Arc::from(sheet_name),
6262 rejected: BTreeMap::new(),
6263 eager_replay: Vec::new(),
6264 preparation_spool_replays: 0,
6265 clean_rejected_anchor_counts: [0; 3],
6266 fragmented_rejected_anchor_counts: [0; 3],
6267 exact_replay: None,
6268 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6269 }
6270 }
6271
6272 fn prepare_source_formula_proposals(
6273 &mut self,
6274 sheet_name: &str,
6275 families: &[crate::engine::SourceFormulaFamily],
6276 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6277 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6278 formula_record_count: u64,
6279 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6280 suppressed: &BTreeSet<(u32, u32)>,
6281 consumed: &FxHashSet<(crate::engine::SourceFamilyId, crate::engine::SourceCoord)>,
6282 consumed_engine: Option<&Arc<()>>,
6283 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6284 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6285 preparation.exact_replay = Some(Arc::clone(&replay));
6286 preparation
6287 .replay_disposition
6288 .register_consumed_members(consumed_engine, consumed.iter().copied());
6289 if !preparation
6290 .replay_disposition
6291 .consumed_engine_matches(&self.source_formula_token)
6292 {
6293 return Err(ExcelError::new(ExcelErrorKind::Value)
6294 .with_message("ResidualConsumedEngineMismatch"));
6295 }
6296 preparation
6297 .replay_disposition
6298 .extend_suppressed_excel_coords(suppressed.iter().copied());
6299 Ok(preparation)
6300 }
6301
6302 fn formula_batch_from_exact_replay(
6303 &mut self,
6304 sheet_name: &str,
6305 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
6306 ) -> Result<FormulaIngestBatch, ExcelError> {
6307 let mut cache = rustc_hash::FxHashMap::default();
6308 let mut formulas = Vec::new();
6309 for record in replayed {
6310 let key = if record.text.starts_with('=') {
6311 record.text
6312 } else {
6313 format!("={}", record.text)
6314 };
6315 let ast_id = if let Some(cached) = cache.get(&key) {
6316 *cached
6317 } else {
6318 let parsed = match formualizer_parse::parser::parse(&key) {
6319 Ok(parsed) => parsed,
6320 Err(error) => {
6321 let Some(parsed) = self.handle_formula_parse_error(
6322 sheet_name,
6323 record.row,
6324 record.col,
6325 &key,
6326 error.to_string(),
6327 )?
6328 else {
6329 continue;
6330 };
6331 parsed
6332 }
6333 };
6334 let ast_id = self.intern_formula_ast(&parsed);
6335 cache.insert(key.clone(), ast_id);
6336 ast_id
6337 };
6338 formulas.push(
6339 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
6340 .with_source_proof(record.source_order, record.family, record.partition_owner),
6341 );
6342 }
6343 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
6344 }
6345
6346 fn prepare_target_combined_legacy_graph(
6347 &self,
6348 packages: &[PreparedTargetSourcePackage],
6349 ordinary: &[PreparedOrdinaryStagedFormula],
6350 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
6351 let mut planned_by_coord = BTreeMap::new();
6352 for package in packages {
6353 for (row, col, ast_id, plan) in &package.legacy {
6354 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
6355 }
6356 }
6357 for formula in ordinary {
6358 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
6359 planned_by_coord.insert(
6360 (formula.sheet_id, formula.lease.row, formula.lease.col),
6361 (ast_id, plan),
6362 );
6363 }
6364 }
6365 let planned = planned_by_coord
6366 .into_iter()
6367 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
6368 .collect::<Vec<_>>();
6369 let formula_count = planned.len();
6370 let graph = self
6371 .graph
6372 .prepare_legacy_graph_plan_multi_sheet(planned)
6373 .map_err(|error| {
6374 ExcelError::new(ExcelErrorKind::Value)
6375 .with_message(format!("target graph preparation failed: {error}"))
6376 })?;
6377 Ok((graph, formula_count))
6378 }
6379
6380 fn replay_target_coordinates(
6381 &mut self,
6382 replay: &Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6383 coordinates: &[(u32, u32)],
6384 deadline: Option<std::time::Instant>,
6385 scratch: &mut u64,
6386 ) -> Result<Option<Vec<crate::engine::DeferredReplayFormula>>, ExcelError> {
6387 if coordinates.is_empty() {
6388 return Ok(Some(Vec::new()));
6389 }
6390 let mut guard = replay.lock().map_err(|_| {
6391 ExcelError::new(ExcelErrorKind::Value)
6392 .with_message("deferred formula spool lock poisoned")
6393 })?;
6394 let retained = guard.selection_cache_footprint().is_some();
6395 let records = guard.replay_selected_exact(coordinates, &mut |work, bytes| {
6396 self.target_preparation_checkpoint(deadline, work)?;
6397 if retained && let Some(ledger) = self.active_resource_ledger.as_mut() {
6398 ledger
6399 .reserve_retained(bytes)
6400 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
6401 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
6402 }
6403 self.reserve_graph_source_scratch(bytes)?;
6404 *scratch = scratch.saturating_add(bytes);
6405 Ok(())
6406 });
6407 drop(guard);
6408 let reconciled = self.reconcile_source_cache_footprints();
6409 let records = records?;
6410 reconciled?;
6411 Ok(records)
6412 }
6413
6414 fn prepare_target_exact_source_selection(
6415 &mut self,
6416 sheet: &str,
6417 lease: StagedPackageLease,
6418 coordinates: Vec<(u32, u32)>,
6419 previous: &BTreeSet<(u32, u32)>,
6420 allow_partial_shared: bool,
6421 deadline: Option<std::time::Instant>,
6422 scratch: &mut u64,
6423 ) -> Result<Option<PreparedTargetSourcePackage>, ExcelError> {
6424 let package = self
6425 .staged_formulas
6426 .get(sheet)
6427 .and_then(|s| s.deferred_package.as_ref())
6428 .unwrap();
6429 if !package.source_geometry_complete
6430 || ((!package.families.is_empty() || !package.partitioned_families.is_empty())
6431 && !package.coordinates_cover_families)
6432 || package.reconciliation_replay.is_some()
6433 {
6434 return Ok(None);
6435 }
6436 let selected_points: BTreeSet<_> = coordinates
6437 .into_iter()
6438 .filter(|point| !package.suppressed.contains(point))
6439 .collect();
6440 let mut prepared = PreparedTargetSourcePackage::empty_selection(
6441 sheet,
6442 self.graph.sheet_id(sheet).unwrap(),
6443 lease,
6444 );
6445 if selected_points.is_empty() {
6446 return Ok(Some(prepared));
6447 }
6448 let mut routing = crate::engine::FormulaReplayDisposition::default();
6449 for partition in &package.partitioned_families {
6450 routing
6451 .register_partition(partition, false)
6452 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
6453 }
6454 routing.extend_suppressed_excel_coords(package.suppressed.iter().copied());
6455 routing.register_consumed_members(
6456 package.consumed_engine.as_ref(),
6457 package.consumed_members.iter().copied(),
6458 );
6459 let replay = Arc::clone(&package.replay);
6460 let source_report = package.accounting_report();
6461 prepared.source_report = source_report;
6462 prepared.disposition = routing.clone();
6463
6464 let replay_points: BTreeSet<_> = selected_points
6465 .iter()
6466 .copied()
6467 .filter(|&(row, col)| !prepared.direct_contains(row, col))
6468 .collect();
6469 let coordinates: Vec<_> = replay_points.iter().copied().collect();
6470 let records = self.replay_target_coordinates(&replay, &coordinates, deadline, scratch)?;
6471 prepared.spool_replays = u64::from(!coordinates.is_empty());
6472 let Some(mut replay_records) = records else {
6473 return Ok(None);
6474 };
6475 for record in &mut replay_records {
6476 if record.family.is_none() {
6477 record.partition_owner = routing
6478 .ordinary_disposition(crate::engine::SourceCoord {
6479 row: record.row.saturating_sub(1),
6480 col: record.col.saturating_sub(1),
6481 })
6482 .1;
6483 }
6484 }
6485 replay_records.sort_by_key(|record| record.source_order);
6486 if replay_records
6487 .iter()
6488 .any(|record| !selected_points.contains(&(record.row, record.col)))
6489 || replay_records
6490 .windows(2)
6491 .any(|records| records[0].source_order == records[1].source_order)
6492 {
6493 return Err(ExcelError::new(ExcelErrorKind::Value)
6494 .with_message("invalid indexed exact source selection"));
6495 }
6496 let represented: BTreeSet<_> = replay_records.iter().map(|r| (r.row, r.col)).collect();
6497 if represented != replay_points {
6498 return Err(ExcelError::new(ExcelErrorKind::Value)
6499 .with_message("incomplete indexed exact source selection"));
6500 }
6501 let source = self
6504 .staged_formulas
6505 .get(sheet)
6506 .unwrap()
6507 .deferred_package
6508 .as_ref()
6509 .unwrap();
6510 let contains = |rect: crate::engine::SourceRect, coord: crate::engine::SourceCoord| {
6511 coord.row >= rect.start.row
6512 && coord.row <= rect.end.row
6513 && coord.col >= rect.start.col
6514 && coord.col <= rect.end.col
6515 };
6516 let mut checked = BTreeSet::new();
6517 for record in replay_records.iter().rev() {
6518 if !checked.insert((record.row, record.col)) {
6519 continue;
6520 }
6521 let coord = crate::engine::SourceCoord {
6522 row: record.row - 1,
6523 col: record.col - 1,
6524 };
6525 let agrees = |owner, shared: bool| {
6526 record.family == shared.then_some(owner)
6527 && record.partition_owner.or(record.family) == Some(owner)
6528 };
6529 let mut valid = true;
6530 for family in &source.families {
6531 let owns = match &family.members {
6532 crate::engine::SourceFamilyMembers::CompleteDomain(domain) => {
6533 contains(domain.rect(), coord)
6534 }
6535 crate::engine::SourceFamilyMembers::ExplicitMembers(members) => {
6536 members.as_slice().binary_search(&coord).is_ok()
6537 }
6538 };
6539 if owns && !agrees(family.source_id, true) {
6540 valid = false;
6541 }
6542 }
6543 for family in &source.partitioned_families {
6544 if family
6545 .fragments
6546 .iter()
6547 .any(|fragment| contains(fragment.rect(), coord))
6548 && !agrees(family.source_id, true)
6549 {
6550 valid = false;
6551 }
6552 if let Some(member) = family
6553 .legacy_members
6554 .as_slice()
6555 .iter()
6556 .find(|member| member.coord == coord)
6557 && !agrees(
6558 family.source_id,
6559 member.kind == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember,
6560 )
6561 {
6562 valid = false;
6563 }
6564 }
6565 if !valid {
6566 return Err(ExcelError::new(ExcelErrorKind::Value)
6567 .with_message("indexed source ownership mismatch"));
6568 }
6569 }
6570 prepared.selected_points = Some(selected_points);
6571 prepared.replay_records = replay_records;
6572 Ok(Some(prepared))
6573 }
6574
6575 fn prepare_target_source_package(
6576 &mut self,
6577 sheet: &str,
6578 lease: StagedPackageLease,
6579 deadline: Option<std::time::Instant>,
6580 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
6581 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
6582 ExcelError::new(ExcelErrorKind::Ref)
6583 .with_message(format!("deferred source sheet not found: {sheet}"))
6584 })?;
6585 let (
6586 source_report,
6587 families,
6588 partitions,
6589 replay,
6590 invalidated,
6591 suppressed,
6592 consumed_members,
6593 consumed_engine,
6594 reconciliation_replay,
6595 ) = {
6596 let package = self
6597 .staged_formulas
6598 .get(sheet)
6599 .and_then(|staged| staged.deferred_package.as_ref())
6600 .ok_or_else(|| {
6601 ExcelError::new(ExcelErrorKind::Value)
6602 .with_message("staged deferred source package is unavailable")
6603 })?;
6604 if package.sheet_name != sheet {
6605 return Err(ExcelError::new(ExcelErrorKind::Value)
6606 .with_message("deferred formula package sheet mismatch"));
6607 }
6608 (
6609 package.accounting_report(),
6610 package.families.clone(),
6611 package.partitioned_families.clone(),
6612 Arc::clone(&package.replay),
6613 package.invalidated.clone(),
6614 package.suppressed.clone(),
6615 package.consumed_members.clone(),
6616 package.consumed_engine.clone(),
6617 package.reconciliation_replay.clone(),
6618 )
6619 };
6620
6621 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
6622 for partition in &partitions {
6623 replay_disposition
6624 .register_partition(partition, false)
6625 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
6626 }
6627 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
6628 replay_disposition
6629 .register_consumed_members(consumed_engine.as_ref(), consumed_members.iter().copied());
6630 if !replay_disposition.consumed_engine_matches(&self.source_formula_token) {
6631 return Err(ExcelError::new(ExcelErrorKind::Value)
6632 .with_message("ResidualConsumedEngineMismatch"));
6633 }
6634 self.target_preparation_checkpoint(deadline, 1)?;
6635 let mut replay_records = if let Some(mut records) = reconciliation_replay {
6636 records.retain(|record| {
6637 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
6638 else {
6639 return true;
6640 };
6641 let coord = crate::engine::SourceCoord { row, col };
6642 let disposition = record.family.map_or_else(
6643 || replay_disposition.ordinary_disposition(coord).0,
6644 |family| replay_disposition.shared_disposition(family, coord),
6645 );
6646 !matches!(
6647 disposition,
6648 crate::engine::FormulaReplayCoordinateDisposition::Direct
6649 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
6650 )
6651 });
6652 records
6653 } else {
6654 replay
6655 .lock()
6656 .map_err(|_| {
6657 ExcelError::new(ExcelErrorKind::Value)
6658 .with_message("deferred formula spool lock poisoned")
6659 })?
6660 .replay_partitioned(&replay_disposition, &partitions)
6661 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
6662 };
6663 replay_records.sort_by_key(|record| record.source_order);
6664 for chunk in replay_records.chunks(256) {
6665 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
6666 }
6667 if replay_records
6668 .windows(2)
6669 .any(|records| records[0].source_order == records[1].source_order)
6670 {
6671 return Err(ExcelError::new(ExcelErrorKind::Value)
6672 .with_message("duplicate deferred source-order proof"));
6673 }
6674
6675 let disposition = replay_disposition;
6676 let direct_families = 0usize;
6677 let direct_cells = 0u64;
6678 let direct_fragments = 0u64;
6679 let direct_complete_families = 0u64;
6680 let direct_complete_cells = 0u64;
6681 let direct_partition_families = 0u64;
6682 let direct_partition_cells = 0u64;
6683 let anchor_parses = 0u64;
6684 let anchor_asts = 0u64;
6685 let anchor_analyses = 0u64;
6686
6687 Ok(PreparedTargetSourcePackage {
6688 sheet: sheet.to_string(),
6689 sheet_id,
6690 lease,
6691 selected_points: None,
6692 complete_selections: Default::default(),
6693 deferred_shared: false,
6694 direct_domains: Vec::new(),
6695 source_report,
6696 replay_records,
6697 spool_replays: 1,
6698 disposition,
6699 legacy: Vec::new(),
6700 direct_families,
6701 direct_cells,
6702 direct_fragments,
6703 direct_complete_families,
6704 direct_complete_cells,
6705 direct_partition_families,
6706 direct_partition_cells,
6707 anchor_parses,
6708 anchor_asts,
6709 anchor_analyses,
6710 })
6711 }
6712
6713 fn fallback_planning_snapshot(
6714 &self,
6715 batch: &FormulaIngestBatch,
6716 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
6717 let mut requests = Vec::new();
6718 for formula in &batch.formulas {
6719 let ast = self
6720 .graph
6721 .data_store()
6722 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
6723 .ok_or_else(|| {
6724 ExcelError::new(ExcelErrorKind::Value)
6725 .with_message("ordered fallback AST is unavailable")
6726 })?;
6727 Self::collect_planning_function_requests(&ast, &mut requests);
6728 }
6729 requests.sort();
6730 requests.dedup();
6731 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6732 &self.resolver,
6733 requests,
6734 )
6735 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
6736 }
6737
6738 fn prepare_legacy_batch_fallback(
6739 &mut self,
6740 batch: FormulaIngestBatch,
6741 function_provider: &dyn crate::traits::FunctionProvider,
6742 ) -> Result<
6743 (
6744 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
6745 u64,
6746 ),
6747 ExcelError,
6748 > {
6749 let formula_count = batch.formulas.len() as u64;
6750 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6751 let mut planned = Vec::with_capacity(batch.formulas.len());
6752 for record in batch.formulas {
6753 let placement = CellRef::new(
6754 sheet_id,
6755 Coord::from_excel(record.row, record.col, true, true),
6756 );
6757 let ingested = self
6758 .graph
6759 .ingest_pipeline(function_provider)
6760 .enable_function_semantics()
6761 .ingest_formula(
6762 FormulaAstInput::RawArena(record.ast_id),
6763 placement,
6764 record.formula_text,
6765 )
6766 .map_err(|error| {
6767 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
6768 })?;
6769 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
6770 }
6771 let plan = self
6772 .graph
6773 .prepare_legacy_graph_plan(sheet_id, planned)
6774 .map_err(|error| {
6775 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
6776 })?;
6777 Ok((plan, formula_count))
6778 }
6779
6780 fn publish_compressed_partial_report(
6781 &mut self,
6782 report: &FormulaIngestReport,
6783 direct_report: &FormulaIngestReport,
6784 ) {
6785 if direct_report.source_family_promoted == 0
6786 && direct_report.graph_formula_cells_materialized == 0
6787 {
6788 return;
6789 }
6790 let mut published = report.clone();
6791 published.accumulate(direct_report);
6792 self.record_formula_ingest_report(published);
6793 }
6794
6795 fn finish_compressed_formula_sources(
6796 &mut self,
6797 batches: Vec<(
6798 FormulaIngestBatch,
6799 crate::engine::FormulaCompressedSourceReport,
6800 crate::engine::FormulaCompressedPreparation,
6801 )>,
6802 ) -> Result<FormulaIngestReport, ExcelError> {
6803 self.observe_function_semantic_epoch()?;
6804 if batches.iter().any(|(_, _, preparation)| {
6805 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
6806 }) {
6807 return Err(ExcelError::new(ExcelErrorKind::Value)
6808 .with_message("compressed source preparation belongs to another engine"));
6809 }
6810 if batches.iter().any(|(fallback, _, preparation)| {
6811 preparation.sheet_name.as_ref() != fallback.sheet_name
6812 }) {
6813 return Err(ExcelError::new(ExcelErrorKind::Value)
6814 .with_message("compressed source preparation sheet mismatch"));
6815 }
6816 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
6817 let initial_provider_revision = self.resolver.planning_semantic_revision();
6818 let mut fallback_batches = Vec::with_capacity(batches.len());
6819 let mut stale_fallback_batches = Vec::new();
6820 let mut pending_preparations = Vec::new();
6821 for (mut fallback, mut source, mut preparation) in batches {
6822 for formula in fallback.formulas.drain(..) {
6823 let source_order = formula.source_order.ok_or_else(|| {
6824 ExcelError::new(ExcelErrorKind::Value).with_message(
6825 "compressed source supplied formulas without source-order proof",
6826 )
6827 })?;
6828 let text = formula.formula_text.ok_or_else(|| {
6829 ExcelError::new(ExcelErrorKind::Value).with_message(
6830 "ordered compressed fallback formula has no exact source text",
6831 )
6832 })?;
6833 preparation
6834 .eager_replay
6835 .push(crate::engine::DeferredReplayFormula {
6836 source_order,
6837 row: formula.row,
6838 col: formula.col,
6839 text: text.to_string(),
6840 family: formula.source_family,
6841 partition_owner: formula.partition_owner,
6842 });
6843 }
6844 preparation
6845 .eager_replay
6846 .sort_by_key(|record| record.source_order);
6847 source.source_spool_replays = source
6848 .source_spool_replays
6849 .saturating_add(preparation.preparation_spool_replays);
6850 let stale_reason = preparation
6851 .function_semantics_used
6852 .then(|| {
6853 if preparation.function_provider_revision != initial_provider_revision {
6854 Some("FunctionProviderRevisionChanged")
6855 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
6856 {
6857 Some("FunctionSemanticEpochChanged")
6858 } else {
6859 None
6860 }
6861 })
6862 .flatten();
6863 let stale_semantics = stale_reason.is_some();
6864 for reason in preparation.rejected.values() {
6865 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
6866 }
6867 source.replay_families = source.families_seen;
6869 source.replay_cells = source.family_cells_seen;
6870 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
6871 fallback.sheet_name.clone(),
6872 source,
6873 );
6874 if stale_semantics {
6875 stale_fallback_batches.push((fallback, compressed));
6876 } else {
6877 fallback_batches.push((fallback, compressed));
6878 }
6879 pending_preparations.push((preparation, stale_semantics));
6880 }
6881 drop(initial_guard);
6882
6883 let configured_mode = self.config.formula_plane_mode;
6885 self.config.formula_plane_mode = FormulaPlaneMode::Off;
6886 let stale_result =
6887 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
6888 self.config.formula_plane_mode = configured_mode;
6889 let mut report = stale_result?;
6890 report.mode = configured_mode;
6891
6892 self.config.formula_plane_mode = FormulaPlaneMode::Off;
6893 let fallback_result =
6894 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
6895 self.config.formula_plane_mode = configured_mode;
6896 match fallback_result {
6897 Ok(fallback_report) => report.accumulate(&fallback_report),
6898 Err(error) => {
6899 self.record_formula_ingest_report(report);
6900 return Err(error);
6901 }
6902 }
6903
6904 let mut direct_report =
6905 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6906 for (preparation, _) in &pending_preparations {
6907 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
6908 preparation.clean_rejected_anchor_counts[0]
6909 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
6910 );
6911 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
6912 preparation.clean_rejected_anchor_counts[1]
6913 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
6914 );
6915 direct_report.source_anchor_analyses =
6916 direct_report.source_anchor_analyses.saturating_add(
6917 preparation.clean_rejected_anchor_counts[2]
6918 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
6919 );
6920 }
6921 loop {
6922 #[cfg(any(test, feature = "test-support"))]
6923 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
6924 hook();
6925 }
6926 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
6927 let commit_provider_revision = self.resolver.planning_semantic_revision();
6928 let mut newly_stale = Vec::new();
6929 let mut current = Vec::new();
6930 for pending in pending_preparations.drain(..) {
6931 let stale_reason = pending
6932 .0
6933 .function_semantics_used
6934 .then(|| {
6935 if pending.0.function_provider_revision != commit_provider_revision {
6936 Some("FunctionProviderRevisionChanged")
6937 } else if self
6938 .prepared_function_semantics_changed(&pending.0, &commit_guard)
6939 {
6940 Some("FunctionSemanticEpochChanged")
6941 } else {
6942 None
6943 }
6944 })
6945 .flatten();
6946 if let Some(reason) = stale_reason {
6947 newly_stale.push((pending, reason));
6948 } else {
6949 current.push(pending);
6950 }
6951 }
6952 if !newly_stale.is_empty() {
6953 drop(commit_guard);
6954 for ((mut preparation, was_initially_stale), reason) in newly_stale {
6955 preparation
6956 .eager_replay
6957 .sort_by_key(|record| record.source_order);
6958 direct_report.source_spool_replays =
6959 direct_report.source_spool_replays.saturating_add(1);
6960 if !was_initially_stale {
6961 direct_report
6962 .fallback_reasons
6963 .entry(reason.to_string())
6964 .or_default();
6965 }
6966 preparation.function_semantics_used = false;
6967 if !preparation.eager_replay.is_empty() {
6968 current.push((preparation, false));
6969 }
6970 }
6971 pending_preparations = current;
6972 continue;
6973 }
6974 drop(commit_guard);
6975
6976 enum SourceProposal {
6978 KnownFallback {
6979 source_order: crate::engine::SourceFormulaOrder,
6980 records: Vec<crate::engine::DeferredReplayFormula>,
6981 },
6982 }
6983
6984 impl SourceProposal {
6985 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
6986 match self {
6987 Self::KnownFallback { source_order, .. } => *source_order,
6988 }
6989 }
6990 }
6991
6992 for (mut preparation, _) in current {
6993 let mut proposals = Vec::with_capacity(preparation.eager_replay.len());
6994 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
6995 for record in preparation.eager_replay.drain(..) {
6996 if let Some(owner) = record.partition_owner.or(record.family) {
6997 family_fallbacks.entry(owner).or_default().push(record);
6998 } else {
6999 proposals.push(SourceProposal::KnownFallback {
7000 source_order: record.source_order,
7001 records: vec![record],
7002 });
7003 }
7004 }
7005 for (_, mut records) in family_fallbacks {
7006 records.sort_by_key(|record| record.source_order);
7007 if records
7008 .windows(2)
7009 .any(|window| window[0].source_order == window[1].source_order)
7010 {
7011 self.publish_compressed_partial_report(&report, &direct_report);
7012 return Err(ExcelError::new(ExcelErrorKind::Value)
7013 .with_message("duplicate exact-replay source-order proof"));
7014 }
7015 let Some(source_order) = records.first().map(|record| record.source_order)
7016 else {
7017 self.publish_compressed_partial_report(&report, &direct_report);
7018 return Err(ExcelError::new(ExcelErrorKind::Value)
7019 .with_message("empty exact-replay fallback family"));
7020 };
7021 proposals.push(SourceProposal::KnownFallback {
7022 source_order,
7023 records,
7024 });
7025 }
7026 proposals.sort_by_key(SourceProposal::source_order);
7027 if proposals
7028 .windows(2)
7029 .any(|window| window[0].source_order() == window[1].source_order())
7030 {
7031 self.publish_compressed_partial_report(&report, &direct_report);
7032 return Err(ExcelError::new(ExcelErrorKind::Value)
7033 .with_message("ambiguous compressed source-order proof"));
7034 }
7035
7036 for proposal in proposals {
7037 match proposal {
7038 SourceProposal::KnownFallback { records, .. } => {
7039 let batch = match self.formula_batch_from_exact_replay(
7040 preparation.sheet_name.as_ref(),
7041 records,
7042 ) {
7043 Ok(batch) => batch,
7044 Err(error) => {
7045 self.publish_compressed_partial_report(&report, &direct_report);
7046 return Err(error);
7047 }
7048 };
7049 if batch.is_empty() {
7050 continue;
7051 }
7052 let snapshot = match self.fallback_planning_snapshot(&batch) {
7053 Ok(snapshot) => snapshot,
7054 Err(error) => {
7055 self.publish_compressed_partial_report(&report, &direct_report);
7056 return Err(error);
7057 }
7058 };
7059 let commit_guard =
7060 crate::function_registry::semantic_epoch_read_guard();
7061 let provider_revision_initial =
7062 self.resolver.planning_semantic_revision();
7063 if (commit_guard.epoch() != snapshot.epoch()
7064 && snapshot.semantic_changes_affect_requests_since_guarded(
7065 &commit_guard,
7066 snapshot.epoch(),
7067 ))
7068 || snapshot.provider_revision().is_some_and(|revision| {
7069 Some(revision) != provider_revision_initial
7070 })
7071 {
7072 drop(commit_guard);
7073 self.publish_compressed_partial_report(&report, &direct_report);
7074 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7075 "ordered fallback planning snapshot became stale",
7076 ));
7077 }
7078 let (plan, formula_count) = match self
7079 .prepare_legacy_batch_fallback(batch, &snapshot)
7080 {
7081 Ok(plan) => plan,
7082 Err(error) => {
7083 drop(commit_guard);
7084 self.publish_compressed_partial_report(&report, &direct_report);
7085 return Err(error);
7086 }
7087 };
7088 let provider_revision_after =
7089 self.resolver.planning_semantic_revision();
7090 if provider_revision_after != provider_revision_initial {
7091 drop(commit_guard);
7092 self.publish_compressed_partial_report(&report, &direct_report);
7093 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7094 "function provider changed while preparing ordered fallback",
7095 ));
7096 }
7097 let graph_vertices = plan.new_vertex_count();
7098 let Some(graph_edges) = plan.planned_edge_count() else {
7099 drop(commit_guard);
7100 self.publish_compressed_partial_report(&report, &direct_report);
7101 return Err(ExcelError::new(ExcelErrorKind::Value)
7102 .with_message("prepared ordered fallback size overflow"));
7103 };
7104 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
7105 {
7106 drop(commit_guard);
7107 self.publish_compressed_partial_report(&report, &direct_report);
7108 return Err(error);
7109 }
7110 if let Err(error) =
7111 self.graph.validate_prepared_legacy_graph_plan(&plan)
7112 {
7113 drop(commit_guard);
7114 self.publish_compressed_partial_report(&report, &direct_report);
7115 return Err(ExcelError::new(ExcelErrorKind::Value)
7116 .with_message(error.to_string()));
7117 }
7118 #[cfg(test)]
7119 if let Some(hook) = self
7120 .before_legacy_fallback_final_provider_sample_hook
7121 .take()
7122 {
7123 hook();
7124 }
7125 let provider_revision_final =
7126 self.resolver.planning_semantic_revision();
7127 if provider_revision_final != provider_revision_initial {
7128 drop(commit_guard);
7129 self.publish_compressed_partial_report(&report, &direct_report);
7130 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7131 "function provider changed after ordered fallback validation",
7132 ));
7133 }
7134 let graph_formulas =
7135 self.graph.apply_prevalidated_legacy_graph_plan(plan);
7136 direct_report.formula_cells_seen = direct_report
7137 .formula_cells_seen
7138 .saturating_add(formula_count);
7139 direct_report.graph_formula_cells_materialized = direct_report
7140 .graph_formula_cells_materialized
7141 .saturating_add(graph_formulas as u64);
7142 direct_report.graph_vertices_created = direct_report
7143 .graph_vertices_created
7144 .saturating_add(graph_vertices as u64);
7145 direct_report.graph_edges_created = direct_report
7146 .graph_edges_created
7147 .saturating_add(graph_edges as u64);
7148 }
7149 }
7150 #[cfg(test)]
7151 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
7152 hook();
7153 }
7154 }
7155 }
7156 break;
7157 }
7158 report.accumulate(&direct_report);
7159 self.record_formula_ingest_report(report.clone());
7160 Ok(report)
7161 }
7162 pub(crate) fn ingest_compressed_formula_source_batches(
7164 &mut self,
7165 batches: Vec<(
7166 FormulaIngestBatch,
7167 crate::engine::FormulaCompressedSourceBatch,
7168 )>,
7169 ) -> Result<FormulaIngestReport, ExcelError> {
7170 self.ingest_compressed_formula_source_batches_inner(batches, true)
7171 }
7172
7173 fn ingest_compressed_formula_source_batches_inner(
7174 &mut self,
7175 batches: Vec<(
7176 FormulaIngestBatch,
7177 crate::engine::FormulaCompressedSourceBatch,
7178 )>,
7179 publish_report: bool,
7180 ) -> Result<FormulaIngestReport, ExcelError> {
7181 let mut source_counts = [0_u64; 11];
7182 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
7183 let mut formula_batches = Vec::with_capacity(batches.len());
7184 let mut compressed_families = Vec::new();
7185 let mut partitioned_families = Vec::new();
7186 for (batch, compressed_batch) in batches {
7187 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
7188 if sheet_name.as_ref() != batch.sheet_name {
7189 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7190 "compressed formula source sheet does not match its replay batch",
7191 ));
7192 }
7193 compressed_families.push((batch.sheet_name.clone(), families));
7194 partitioned_families.push((batch.sheet_name.clone(), partitions));
7195 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
7196 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
7197 source_counts[2] =
7198 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
7199 source_counts[3] =
7200 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
7201 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
7202 source_counts[5] =
7203 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
7204 source_counts[6] =
7205 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
7206 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
7207 source_counts[8] =
7208 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
7209 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
7210 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
7211 source_report.families_seen = source_report
7212 .families_seen
7213 .saturating_add(compressed.families_seen);
7214 source_report.family_cells_seen = source_report
7215 .family_cells_seen
7216 .saturating_add(compressed.family_cells_seen);
7217 source_report.source_clean_families = source_report
7218 .source_clean_families
7219 .saturating_add(compressed.source_clean_families);
7220 source_report.source_clean_cells = source_report
7221 .source_clean_cells
7222 .saturating_add(compressed.source_clean_cells);
7223 source_report.source_fragmentable_families = source_report
7224 .source_fragmentable_families
7225 .saturating_add(compressed.source_fragmentable_families);
7226 source_report.source_fragmentable_cells = source_report
7227 .source_fragmentable_cells
7228 .saturating_add(compressed.source_fragmentable_cells);
7229 source_report.source_fragment_count = source_report
7230 .source_fragment_count
7231 .saturating_add(compressed.source_fragment_count);
7232 source_report.source_isolated_fallback_cells = source_report
7233 .source_isolated_fallback_cells
7234 .saturating_add(compressed.source_isolated_fallback_cells);
7235 source_report.source_hole_exclusions = source_report
7236 .source_hole_exclusions
7237 .saturating_add(compressed.source_hole_exclusions);
7238 source_report.source_ordinary_exclusions = source_report
7239 .source_ordinary_exclusions
7240 .saturating_add(compressed.source_ordinary_exclusions);
7241 source_report.source_partition_failures = source_report
7242 .source_partition_failures
7243 .saturating_add(compressed.source_partition_failures);
7244 source_report.replay_families = source_report
7245 .replay_families
7246 .saturating_add(compressed.replay_families);
7247 source_report.replay_cells = source_report
7248 .replay_cells
7249 .saturating_add(compressed.replay_cells);
7250 source_report.forward_descendants = source_report
7251 .forward_descendants
7252 .saturating_add(compressed.forward_descendants);
7253 source_report.evidence_limit_fallbacks = source_report
7254 .evidence_limit_fallbacks
7255 .saturating_add(compressed.evidence_limit_fallbacks);
7256 source_report.evidence_peak_bytes = source_report
7257 .evidence_peak_bytes
7258 .max(compressed.evidence_peak_bytes);
7259 for (reason, count) in compressed.fallback_reasons {
7260 *source_report.fallback_reasons.entry(reason).or_default() += count;
7261 }
7262 formula_batches.push(batch);
7263 }
7264 self.ingest_formula_batches_inner(
7265 formula_batches,
7266 source_counts,
7267 Some(source_report),
7268 compressed_families,
7269 partitioned_families,
7270 publish_report,
7271 )
7272 }
7273
7274 pub fn ingest_formula_batches(
7275 &mut self,
7276 batches: Vec<FormulaIngestBatch>,
7277 ) -> Result<FormulaIngestReport, ExcelError> {
7278 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
7279 let report = self.ingest_formula_batches_inner(
7280 batches,
7281 [0; 11],
7282 None,
7283 Vec::new(),
7284 Vec::new(),
7285 true,
7286 )?;
7287 if has_formulas {
7288 self.mark_topology_edited();
7289 }
7290 Ok(report)
7291 }
7292
7293 fn ingest_formula_batches_unpublished(
7294 &mut self,
7295 batches: Vec<FormulaIngestBatch>,
7296 ) -> Result<FormulaIngestReport, ExcelError> {
7297 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
7298 }
7299
7300 fn ingest_formula_batches_inner(
7301 &mut self,
7302 batches: Vec<FormulaIngestBatch>,
7303 source_counts: [u64; 11],
7304 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
7305 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
7306 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
7307 publish_report: bool,
7308 ) -> Result<FormulaIngestReport, ExcelError> {
7309 self.observe_function_semantic_epoch()?;
7310 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7311 #[cfg(feature = "tracing")]
7312 let arena_nodes_before = self.graph.data_store().memory_usage().total_ast_nodes;
7313 #[cfg(feature = "tracing")]
7314 let route = if !partitioned_families.is_empty() {
7315 "partitioned"
7316 } else if !compressed_families.is_empty() {
7317 "compressed_source"
7318 } else if source_report.is_some() {
7319 "replay"
7320 } else {
7321 "ordinary"
7322 };
7323 let _ingest_span = crate::engine::trace::fz_span!(
7324 tracing::Level::INFO,
7325 "ingest",
7326 "ingest.batch",
7327 mode = ?FormulaPlaneMode::Off,
7328 route,
7329 formula_cells = formula_cells_seen
7330 );
7331 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
7332 let materialize_batches = batches;
7333 report.formula_cells_seen = formula_cells_seen;
7334 report.source_formula_events = source_counts[0];
7335 report.source_ordinary_events = source_counts[1];
7336 report.source_shared_anchor_events = source_counts[2];
7337 report.source_shared_descendant_events = source_counts[3];
7338 report.source_unknown_events = source_counts[4];
7339 report.source_formula_records_spooled = source_counts[5];
7340 report.source_spool_encoded_bytes = source_counts[6];
7341 report.source_spool_peak_memory_bytes = source_counts[7];
7342 report.source_spool_spilled_bytes = source_counts[8];
7343 report.source_spool_spill_files = source_counts[9];
7344 report.source_spool_replays = source_counts[10];
7345 if let Some(source) = source_report {
7346 report.source_families_seen = source.families_seen;
7347 report.source_family_cells_seen = source.family_cells_seen;
7348 report.source_family_shadow_eligible = source.source_clean_families;
7349 report.source_family_shadow_eligible_cells = source.source_clean_cells;
7350 report.source_partitioned_families_seen = source.source_fragmentable_families;
7351 report.source_partition_holes = source.source_hole_exclusions;
7352 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
7353 report.source_partition_failures = source.source_partition_failures;
7354 report.source_partition_surviving_cells = source.source_fragmentable_cells;
7355 report.source_family_fallback = report
7356 .source_family_fallback
7357 .saturating_add(source.replay_families);
7358 report.source_family_fallback_cells = report
7359 .source_family_fallback_cells
7360 .saturating_add(source.replay_cells);
7361 report.source_forward_descendants = source.forward_descendants;
7362 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
7363 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
7364 for (reason, count) in source.fallback_reasons {
7365 let total = report.fallback_reasons.entry(reason).or_default();
7366 *total = total.saturating_add(count);
7367 }
7368 }
7369
7370 if self.graph.first_load_assume_new()
7374 && !self.graph_admission_enabled()
7375 && !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
7376 {
7377 let mut builder =
7378 crate::engine::ingest_builder::BulkIngestBuilder::new(&mut self.graph);
7379 for batch in materialize_batches {
7380 if batch.is_empty() {
7381 continue;
7382 }
7383 let sheet_id = builder
7384 .add_sheet_checked(&batch.sheet_name)
7385 .ok_or_else(|| {
7386 ExcelError::new(ExcelErrorKind::Ref)
7387 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7388 })?;
7389 builder.add_formula_refs(
7390 sheet_id,
7391 batch.formulas.into_iter().map(|record| {
7392 (
7393 record.row,
7394 record.col,
7395 crate::engine::graph::FormulaRef::of_ingested(
7396 record.ast_id,
7397 record.member_anchor,
7398 ),
7399 )
7400 }),
7401 );
7402 }
7403 let summary = builder.finish_with_provider(&self.resolver)?;
7404 report.graph_formula_cells_materialized = summary.formulas as u64;
7405 report.graph_vertices_created = summary.vertices as u64;
7406 report.graph_edges_created = summary.edges as u64;
7407 } else if !materialize_batches.iter().all(FormulaIngestBatch::is_empty) {
7408 let mut prepared_by_sheet: BTreeMap<String, Vec<_>> = BTreeMap::new();
7409 for batch in materialize_batches {
7410 if batch.is_empty() {
7411 continue;
7412 }
7413 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
7414 ExcelError::new(ExcelErrorKind::Ref)
7415 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
7416 })?;
7417 let mut pipeline = self.ingest_pipeline();
7418 let prepared = prepared_by_sheet.entry(batch.sheet_name).or_default();
7421 prepared.reserve(batch.formulas.len());
7422 for record in batch.formulas {
7423 let placement = CellRef::new(
7424 sheet_id,
7425 Coord::from_excel(record.row, record.col, true, true),
7426 );
7427 let input = match record.member_anchor {
7428 Some(anchor) => FormulaAstInput::Member {
7429 template: record.ast_id,
7430 anchor,
7431 },
7432 None => FormulaAstInput::RawArena(record.ast_id),
7433 };
7434 let formula = pipeline.ingest_formula(input, placement, None)?;
7435 prepared.push((
7436 record.row,
7437 record.col,
7438 crate::engine::graph::FormulaRef::of_ingested(
7439 formula.ast_id,
7440 formula.member_anchor,
7441 ),
7442 formula.dep_plan,
7443 ));
7444 }
7445 }
7446 let admission_preflighted = self.graph_admission_enabled();
7447 if admission_preflighted {
7448 let mut preview = Vec::new();
7449 for (sheet_name, formulas) in &prepared_by_sheet {
7450 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
7451 ExcelError::new(ExcelErrorKind::Ref)
7452 .with_message(format!("unknown ingest sheet: {sheet_name}"))
7453 })?;
7454 preview.extend(
7455 formulas
7456 .iter()
7457 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
7458 );
7459 }
7460 let admission = self.graph.preview_formula_mutations(&preview)?;
7461 self.preflight_graph_admission(admission)?;
7462 }
7463
7464 let mut builder = self.begin_bulk_ingest();
7465 if admission_preflighted {
7466 builder.mark_admission_preflighted();
7467 }
7468 for (sheet_name, formulas) in prepared_by_sheet {
7469 if formulas.is_empty() {
7470 continue;
7471 }
7472 let sheet_id = builder.add_sheet(&sheet_name);
7473 builder.add_formula_plans(sheet_id, formulas);
7474 }
7475 let summary = builder.finish()?;
7476 report.graph_formula_cells_materialized = summary.formulas as u64;
7477 report.graph_vertices_created = summary.vertices as u64;
7478 report.graph_edges_created = summary.edges as u64;
7479 }
7480
7481 crate::engine::trace::fz_event!(
7482 tracing::Level::INFO,
7483 "ingest",
7484 "ingest.summary",
7485 candidate_cells = report.shadow_candidate_cells,
7486 accepted_span_cells = report.shadow_accepted_span_cells,
7487 fallback_cells = report.shadow_fallback_cells,
7488 spans_created = report.shadow_spans_created,
7489 templates_interned = report.shadow_templates_interned,
7490 graph_vertices_created = report.graph_vertices_created,
7491 graph_edges_created = report.graph_edges_created,
7492 arena_nodes_delta = self
7493 .graph
7494 .data_store()
7495 .memory_usage()
7496 .total_ast_nodes
7497 .saturating_sub(arena_nodes_before)
7498 );
7499 if publish_report {
7500 self.record_formula_ingest_report(report.clone());
7501 }
7502 Ok(report)
7503 }
7504
7505 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
7506 let mut unique = Vec::new();
7507 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
7508 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
7509 prior.sheet == diagnostic.sheet
7510 && prior.row == diagnostic.row
7511 && prior.col == diagnostic.col
7512 && prior.formula == diagnostic.formula
7513 && prior.policy == diagnostic.policy
7514 });
7515 if !duplicate {
7516 unique.push(diagnostic);
7517 }
7518 }
7519 self.formula_parse_diagnostics.extend(unique);
7520 }
7521
7522 pub fn handle_formula_parse_error(
7523 &mut self,
7524 sheet: &str,
7525 row: u32,
7526 col: u32,
7527 formula: &str,
7528 message: String,
7529 ) -> Result<Option<ASTNode>, ExcelError> {
7530 let policy = self.config.formula_parse_policy;
7531
7532 if policy == FormulaParsePolicy::Strict {
7533 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
7534 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7535 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
7536 )));
7537 }
7538
7539 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
7540 sheet: sheet.to_string(),
7541 row,
7542 col,
7543 formula: formula.to_string(),
7544 message: message.clone(),
7545 policy,
7546 });
7547
7548 match policy {
7549 FormulaParsePolicy::Strict => unreachable!(),
7550 FormulaParsePolicy::KeepCachedValue => Ok(None),
7551 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
7552 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
7553 None,
7554 ))),
7555 FormulaParsePolicy::CoerceToError => {
7556 let err = ExcelError::new(ExcelErrorKind::Error)
7557 .with_message(format!("Malformed formula: {message}"));
7558 Ok(Some(ASTNode::new(
7559 ASTNodeType::Literal(LiteralValue::Error(err)),
7560 None,
7561 )))
7562 }
7563 }
7564 }
7565
7566 #[cfg(test)]
7567 fn target_preparation_fault(
7568 &mut self,
7569 seam: crate::engine::target_preparation::TargetPreparationFault,
7570 ) -> Result<(), ExcelError> {
7571 if self.target_preparation_fault_for_test == Some(seam) {
7572 self.target_preparation_fault_for_test = None;
7573 Err(ExcelError::new(ExcelErrorKind::Value)
7574 .with_message(format!("injected target preparation fault: {seam:?}")))
7575 } else {
7576 Ok(())
7577 }
7578 }
7579
7580 fn preparation_stale(
7581 reason: formualizer_common::PreparationStaleReason,
7582 message: impl Into<String>,
7583 ) -> ExcelError {
7584 ExcelError::new(ExcelErrorKind::Value)
7585 .with_message(message)
7586 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
7587 }
7588
7589 fn preparation_revision_stale_reason(
7590 assumptions: &crate::engine::PreparationRevision,
7591 current: &crate::engine::PreparationRevision,
7592 planning_requests: &BTreeSet<(String, String, usize)>,
7593 staged_leases_match: bool,
7594 ) -> Option<formualizer_common::PreparationStaleReason> {
7595 if assumptions.graph != current.graph {
7596 Some(formualizer_common::PreparationStaleReason::Graph)
7597 } else if assumptions.staged != current.staged || !staged_leases_match {
7598 Some(formualizer_common::PreparationStaleReason::Staged)
7599 } else if assumptions.symbols != current.symbols {
7600 Some(formualizer_common::PreparationStaleReason::Symbols)
7601 } else if assumptions.provider != current.provider {
7602 Some(formualizer_common::PreparationStaleReason::Provider)
7603 } else if assumptions.semantic != current.semantic
7604 && crate::function_registry::semantic_changes_affect_requests_since(
7605 assumptions.semantic,
7606 planning_requests.iter().cloned(),
7607 )
7608 {
7609 Some(formualizer_common::PreparationStaleReason::Semantic)
7610 } else {
7611 None
7612 }
7613 }
7614
7615 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
7616 crate::engine::PreparationRevision {
7617 graph: self.graph.topology_revision(),
7618 authority: 0,
7619 authority_indexes: 0,
7620 authority_indexed_plane: 0,
7621 staged: self.staged_formula_index.revision(),
7622 symbols: self.graph.symbol_revision(),
7623 semantic: crate::function_registry::semantic_epoch(),
7624 provider: self.resolver.planning_semantic_revision(),
7625 }
7626 }
7627
7628 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
7629 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
7630 let provider = self.resolver.planning_semantic_revision();
7631 let semantic = registry_guard.epoch();
7632 PlanningRevisionSnapshot {
7633 engine_topology_epoch: self.topology_epoch,
7634 graph_topology_revision: self.graph.topology_revision(),
7635 staged: self.staged_formula_index.revision(),
7636 symbols: self.graph.symbol_revision(),
7637 semantic,
7638 provider,
7639 deterministic_mode: self.config.deterministic_mode.clone(),
7640 budgets: self.evaluation_resource_budgets.clone(),
7641 }
7642 }
7643
7644 fn recalc_plan_key(&self) -> RecalcPlanKey {
7645 RecalcPlanKey {
7646 engine_token: Arc::clone(&self.recalc_plan_token),
7647 revisions: self.planning_revision_snapshot(),
7648 }
7649 }
7650
7651 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
7652 ExcelError::new(ExcelErrorKind::Value)
7653 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
7654 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
7655 }
7656
7657 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
7658 use formualizer_common::PlanStaleReason;
7659
7660 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
7661 return Err(Self::plan_stale(PlanStaleReason::Engine));
7662 }
7663
7664 let current = self.planning_revision_snapshot();
7665 let expected = &key.revisions;
7666 let stale = if expected.provider != current.provider {
7667 Some(PlanStaleReason::Provider)
7668 } else if expected.semantic != current.semantic {
7669 Some(PlanStaleReason::Semantic)
7670 } else if expected.budgets != current.budgets
7671 || expected.deterministic_mode != current.deterministic_mode
7672 {
7673 Some(PlanStaleReason::Budget)
7674 } else if expected.staged != current.staged {
7675 Some(PlanStaleReason::Staged)
7676 } else if expected.symbols != current.symbols {
7677 Some(PlanStaleReason::Symbols)
7678 } else if expected.graph_topology_revision != current.graph_topology_revision
7679 || expected.engine_topology_epoch != current.engine_topology_epoch
7680 {
7681 Some(PlanStaleReason::Graph)
7682 } else {
7683 None
7684 };
7685 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
7686 }
7687
7688 fn target_preparation_checkpoint(
7689 &mut self,
7690 deadline: Option<std::time::Instant>,
7691 work: u64,
7692 ) -> Result<(), ExcelError> {
7693 if self
7694 .active_cancel_flag
7695 .as_ref()
7696 .is_some_and(|cancel| cancel.is_cancelled())
7697 {
7698 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
7699 .with_message("target graph preparation cancelled"));
7700 }
7701 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
7702 return Err(crate::engine::ResourceLedgerError::Exhausted(
7703 formualizer_common::ResourceExhaustionDetail {
7704 reason: formualizer_common::ResourceExhaustionReason::Deadline,
7705 limit: 0,
7706 observed: 1,
7707 request_id: self
7708 .active_evaluation_resource_request
7709 .as_ref()
7710 .map(|stats| stats.request_id),
7711 },
7712 )
7713 .into_excel_error());
7714 }
7715 self.charge_bounded_work(work)
7716 }
7717
7718 fn opaque_reason_in_ast(
7719 &self,
7720 ast: &ASTNode,
7721 provider: &dyn crate::traits::FunctionProvider,
7722 ) -> Option<crate::engine::OpaqueReason> {
7723 match &ast.node_type {
7724 ASTNodeType::Function { name, args } => {
7725 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
7726 if canonical == "INDIRECT" {
7727 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
7728 }
7729 let Some(function) = provider.get_function_for_planning("", name) else {
7730 return Some(crate::engine::OpaqueReason::UnknownFunction);
7731 };
7732 let caps = function.caps();
7733 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7734 || caps.contains(FnCaps::RETURNS_REFERENCE)
7735 {
7736 return Some(crate::engine::OpaqueReason::DynamicReference);
7737 }
7738 args.iter()
7739 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
7740 }
7741 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
7742 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
7743 ASTNodeType::BinaryOp { left, right, .. } => self
7744 .opaque_reason_in_ast(left, provider)
7745 .or_else(|| self.opaque_reason_in_ast(right, provider)),
7746 ASTNodeType::Array(rows) => rows
7747 .iter()
7748 .flat_map(|row| row.iter())
7749 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
7750 ASTNodeType::Reference {
7751 reference:
7752 ReferenceType::Cell {
7753 sheet: Some(sheet), ..
7754 }
7755 | ReferenceType::Range {
7756 sheet: Some(sheet), ..
7757 },
7758 ..
7759 } if self.graph.sheet_id(sheet).is_none() => {
7760 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
7761 }
7762 ASTNodeType::Reference {
7763 reference:
7764 ReferenceType::External(_)
7765 | ReferenceType::Cell3D { .. }
7766 | ReferenceType::Range3D { .. },
7767 ..
7768 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
7769 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
7770 }
7771 }
7772
7773 fn target_planning_snapshot(
7774 &mut self,
7775 ast: &ASTNode,
7776 planning_requests: &mut BTreeSet<(String, String, usize)>,
7777 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7778 #[cfg(test)]
7779 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
7780 return Err(Self::preparation_stale(
7781 formualizer_common::PreparationStaleReason::Semantic,
7782 "injected target semantic preparation movement",
7783 ));
7784 }
7785 #[cfg(test)]
7786 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
7787 hook();
7788 }
7789 let mut requests = Vec::new();
7790 Self::collect_planning_function_requests(ast, &mut requests);
7791 requests.sort();
7792 requests.dedup();
7793 planning_requests.extend(requests.iter().cloned());
7794 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7795 &self.resolver,
7796 requests,
7797 )
7798 .map_err(|error| {
7799 ExcelError::new(ExcelErrorKind::Value)
7800 .with_message(format!("target planning snapshot unavailable: {error:?}"))
7801 })
7802 }
7803
7804 fn target_planning_snapshot_stale_reason(
7805 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
7806 assumptions: &crate::engine::PreparationRevision,
7807 ) -> Option<formualizer_common::PreparationStaleReason> {
7808 if snapshot
7809 .provider_revision()
7810 .is_some_and(|revision| Some(revision) != assumptions.provider)
7811 {
7812 Some(formualizer_common::PreparationStaleReason::Provider)
7813 } else if snapshot.epoch() != assumptions.semantic
7814 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
7815 {
7816 Some(formualizer_common::PreparationStaleReason::Semantic)
7817 } else {
7818 None
7819 }
7820 }
7821
7822 fn widen_target_preparation(
7823 policy: crate::engine::OpaquePreparePolicy,
7824 scope: &mut crate::engine::PrepareScope,
7825 reasons: &mut Vec<crate::engine::OpaqueReason>,
7826 reason: crate::engine::OpaqueReason,
7827 ) -> Result<bool, ExcelError> {
7828 if policy == crate::engine::OpaquePreparePolicy::Error {
7829 return Err(ExcelError::new(ExcelErrorKind::NImpl)
7830 .with_message(format!("opaque target preparation semantics: {reason:?}")));
7831 }
7832 if !reasons.contains(&reason) {
7833 reasons.push(reason);
7834 }
7835 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
7836 *scope = crate::engine::PrepareScope::Workbook;
7837 Ok(true)
7838 } else {
7839 Ok(false)
7840 }
7841 }
7842
7843 fn widen_target_preparation_to_sheet(
7844 policy: crate::engine::OpaquePreparePolicy,
7845 scope: &mut crate::engine::PrepareScope,
7846 reasons: &mut Vec<crate::engine::OpaqueReason>,
7847 reason: crate::engine::OpaqueReason,
7848 sheet: &str,
7849 ) -> Result<bool, ExcelError> {
7850 if policy == crate::engine::OpaquePreparePolicy::Error {
7851 return Err(ExcelError::new(ExcelErrorKind::NImpl)
7852 .with_message(format!("opaque target preparation semantics: {reason:?}")));
7853 }
7854 if !reasons.contains(&reason) {
7855 reasons.push(reason);
7856 }
7857 match scope {
7858 crate::engine::PrepareScope::Exact => {
7859 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
7860 Ok(true)
7861 }
7862 crate::engine::PrepareScope::Sheets(sheets) => {
7863 if sheets.iter().any(|candidate| candidate == sheet) {
7864 Ok(false)
7865 } else {
7866 sheets.push(sheet.to_string());
7867 sheets.sort();
7868 Ok(true)
7869 }
7870 }
7871 crate::engine::PrepareScope::Workbook => Ok(false),
7872 }
7873 }
7874
7875 fn ast_has_proven_sheet_local_dynamic(
7876 ast: &ASTNode,
7877 provider: &dyn crate::traits::FunctionProvider,
7878 ) -> bool {
7879 fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
7880 match &ast.node_type {
7881 ASTNodeType::Function { name, args } => {
7882 let Some(function) = provider.get_function_for_planning("", name) else {
7883 return (false, false);
7884 };
7885 let caps = function.caps();
7886 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
7887 || caps.contains(FnCaps::RETURNS_REFERENCE);
7888 let canonical = name.rsplit('.').next().unwrap_or(name);
7889 if dynamic
7890 && !canonical.eq_ignore_ascii_case("OFFSET")
7891 && !canonical.eq_ignore_ascii_case("INDEX")
7892 {
7893 return (false, true);
7894 }
7895 let mut has_dynamic = dynamic;
7896 for arg in args {
7897 let (safe, child_dynamic) = classify(arg, provider);
7898 if !safe {
7899 return (false, has_dynamic || child_dynamic);
7900 }
7901 has_dynamic |= child_dynamic;
7902 }
7903 (true, has_dynamic)
7904 }
7905 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
7906 ASTNodeType::BinaryOp { left, right, .. } => {
7907 let (left_safe, left_dynamic) = classify(left, provider);
7908 let (right_safe, right_dynamic) = classify(right, provider);
7909 (left_safe && right_safe, left_dynamic || right_dynamic)
7910 }
7911 ASTNodeType::Array(rows) => {
7912 let mut has_dynamic = false;
7913 for item in rows.iter().flatten() {
7914 let (safe, child_dynamic) = classify(item, provider);
7915 if !safe {
7916 return (false, has_dynamic || child_dynamic);
7917 }
7918 has_dynamic |= child_dynamic;
7919 }
7920 (true, has_dynamic)
7921 }
7922 ASTNodeType::Reference {
7923 reference:
7924 ReferenceType::Cell { sheet: None, .. }
7925 | ReferenceType::Range { sheet: None, .. },
7926 ..
7927 }
7928 | ASTNodeType::Literal(_)
7929 | ASTNodeType::Omitted => (true, false),
7930 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
7931 }
7932 }
7933
7934 let (safe, dynamic) = classify(ast, provider);
7935 safe && dynamic
7936 }
7937
7938 fn table_selection_region(
7939 &self,
7940 entry: &crate::engine::graph::TableEntry,
7941 selection: &crate::engine::TableSelection,
7942 ) -> Result<PreparationRegion, ExcelError> {
7943 let mut start_row = entry.range.start.coord.row() + 1;
7944 let mut end_row = entry.range.end.coord.row() + 1;
7945 let mut start_col = entry.range.start.coord.col() + 1;
7946 let mut end_col = entry.range.end.coord.col() + 1;
7947 match selection {
7948 crate::engine::TableSelection::Whole => {}
7949 crate::engine::TableSelection::Headers => {
7950 if !entry.header_row {
7951 return Err(ExcelError::new(ExcelErrorKind::Value)
7952 .with_message(format!("table {} has no header row", entry.name)));
7953 }
7954 end_row = start_row;
7955 }
7956 crate::engine::TableSelection::Data => {
7957 if entry.header_row {
7958 start_row = start_row.saturating_add(1);
7959 }
7960 if entry.totals_row {
7961 end_row = end_row.saturating_sub(1);
7962 }
7963 }
7964 crate::engine::TableSelection::Totals => {
7965 if !entry.totals_row {
7966 return Err(ExcelError::new(ExcelErrorKind::Value)
7967 .with_message(format!("table {} has no totals row", entry.name)));
7968 }
7969 start_row = end_row;
7970 }
7971 crate::engine::TableSelection::Column(column) => {
7972 let index = entry.col_index(column).ok_or_else(|| {
7973 ExcelError::new(ExcelErrorKind::Name)
7974 .with_message(format!("unknown table column: {column}"))
7975 })?;
7976 start_col = start_col.saturating_add(index as u32);
7977 end_col = start_col;
7978 }
7979 crate::engine::TableSelection::Columns { start, end } => {
7980 let first = entry.col_index(start).ok_or_else(|| {
7981 ExcelError::new(ExcelErrorKind::Name)
7982 .with_message(format!("unknown table column: {start}"))
7983 })?;
7984 let last = entry.col_index(end).ok_or_else(|| {
7985 ExcelError::new(ExcelErrorKind::Name)
7986 .with_message(format!("unknown table column: {end}"))
7987 })?;
7988 if first > last {
7989 return Err(ExcelError::new(ExcelErrorKind::Value)
7990 .with_message("table column selection is reversed"));
7991 }
7992 start_col = start_col.saturating_add(first as u32);
7993 end_col = entry
7994 .range
7995 .start
7996 .coord
7997 .col()
7998 .saturating_add(last as u32)
7999 .saturating_add(1);
8000 }
8001 }
8002 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
8003 start_row = entry.range.start.coord.row() + 1;
8004 end_row = start_row;
8005 }
8006 if start_row > end_row || start_col > end_col {
8007 return Err(
8008 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
8009 );
8010 }
8011 Ok(PreparationRegion {
8012 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
8013 sheet_id: entry.sheet_id(),
8014 start_row,
8015 start_col,
8016 end_row,
8017 end_col,
8018 })
8019 }
8020
8021 #[cfg(test)]
8022 pub(crate) fn reset_target_root_dedup_probes_for_test() {
8023 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
8024 }
8025
8026 #[cfg(test)]
8027 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
8028 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
8029 }
8030
8031 pub(crate) fn resolve_target_producers(
8032 &mut self,
8033 targets: &[crate::engine::EvaluationTarget],
8034 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
8035 use crate::engine::target_preparation::TargetProducer;
8036
8037 let request_id = self
8038 .active_evaluation_resource_request
8039 .as_ref()
8040 .map(|request| request.request_id);
8041 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
8042 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
8043 let resolve_region = |engine: &mut Self,
8044 region: Region,
8045 value_only: Option<CellRef>,
8046 roots: &mut OrderedTargetProducers|
8047 -> Result<(), ExcelError> {
8048 let before = roots.len();
8049 for anchor in engine.graph.spill_anchors_in_region(
8050 region.sheet_id(),
8051 region.axis_ranges().0.query_bounds().0,
8052 region.axis_ranges().1.query_bounds().0,
8053 region.axis_ranges().0.query_bounds().1,
8054 region.axis_ranges().1.query_bounds().1,
8055 ) {
8056 roots
8057 .push(TargetProducer::Legacy(anchor))
8058 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8059 }
8060 for vertex in engine.graph.vertices_in_region(
8061 region.sheet_id(),
8062 region.axis_ranges().0.query_bounds().0,
8063 region.axis_ranges().0.query_bounds().1,
8064 region.axis_ranges().1.query_bounds().0,
8065 region.axis_ranges().1.query_bounds().1,
8066 ) {
8067 let vertex = engine
8068 .graph
8069 .get_cell_ref(vertex)
8070 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
8071 .unwrap_or(vertex);
8072 if matches!(
8077 engine.graph.get_vertex_kind(vertex),
8078 VertexKind::FormulaScalar | VertexKind::FormulaArray
8079 ) {
8080 roots
8081 .push(TargetProducer::Legacy(vertex))
8082 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8083 }
8084 }
8085 if roots.len() == before
8086 && let Some(cell) = value_only
8087 {
8088 roots
8089 .push(TargetProducer::ValueOnly(cell))
8090 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
8091 }
8092 Ok(())
8093 };
8094
8095 for target in targets {
8096 match target {
8097 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8098 if *row == 0 || *col == 0 {
8099 return Err(ExcelError::new(ExcelErrorKind::Ref)
8100 .with_message("target cell coordinates are one-based"));
8101 }
8102 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8103 ExcelError::new(ExcelErrorKind::Ref)
8104 .with_message(format!("target sheet not found: {sheet}"))
8105 })?;
8106 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
8107 resolve_region(
8108 self,
8109 Region::point(sheet_id, *row - 1, *col - 1),
8110 Some(cell),
8111 &mut roots,
8112 )?;
8113 }
8114 crate::engine::EvaluationTarget::Range(range) => {
8115 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8116 ExcelError::new(ExcelErrorKind::Ref)
8117 .with_message(format!("target sheet not found: {}", range.sheet))
8118 })?;
8119 resolve_region(
8120 self,
8121 Region::rect(
8122 sheet_id,
8123 range.start_row - 1,
8124 range.end_row - 1,
8125 range.start_col - 1,
8126 range.end_col - 1,
8127 ),
8128 None,
8129 &mut roots,
8130 )?;
8131 }
8132 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8133 let scope = self.name_query_scope(scope_sheet.as_deref())?;
8134 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
8135 roots
8136 .push(TargetProducer::Symbol(entry.vertex))
8137 .map_err(|_| {
8138 target_root_allocation_error(roots.len() + 1, request_id)
8139 })?;
8140 }
8141 }
8142 crate::engine::EvaluationTarget::Table { name, .. } => {
8143 if let Some(entry) = self.graph.resolve_table_entry(name) {
8144 roots
8145 .push(TargetProducer::Symbol(entry.vertex))
8146 .map_err(|_| {
8147 target_root_allocation_error(roots.len() + 1, request_id)
8148 })?;
8149 }
8150 }
8151 }
8152 }
8153 Ok(roots.into_vec())
8154 }
8155
8156 pub fn prepare_graph_for_targets(
8160 &mut self,
8161 targets: &[crate::engine::EvaluationTarget],
8162 options: crate::engine::TargetEvalOptions<'_>,
8163 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8164 let previous_budgets = self.evaluation_resource_budgets.clone();
8165 let diagnostics_len = self.formula_parse_diagnostics.len();
8166 let previous_report = self.last_formula_ingest_report.clone();
8167 if let Some(budgets) = options.budgets {
8168 self.evaluation_resource_budgets = budgets.clone();
8169 }
8170 let previous_graph_budget_override = self
8171 .graph
8172 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
8173 let previous_cancel = self.active_cancel_flag.take();
8177 self.active_cancel_flag = options.cancel.clone();
8178 let result = self.observe_evaluation_resource_request(
8179 EvaluationRequestKind::TargetPreparation,
8180 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
8181 );
8182 self.active_cancel_flag = previous_cancel;
8183 self.graph
8184 .set_admission_budget_override(previous_graph_budget_override);
8185 self.evaluation_resource_budgets = previous_budgets;
8186 if result.is_err() {
8187 self.formula_parse_diagnostics.truncate(diagnostics_len);
8188 self.last_formula_ingest_report = previous_report;
8189 }
8190 result
8191 }
8192
8193 fn prepare_graph_for_targets_unobserved(
8194 &mut self,
8195 targets: &[crate::engine::EvaluationTarget],
8196 options: &crate::engine::TargetEvalOptions<'_>,
8197 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8198 let scratch_checkpoint = self
8199 .active_resource_ledger
8200 .as_ref()
8201 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
8202 let result = self.prepare_graph_for_targets_transaction(targets, options);
8203 let release = self
8204 .active_resource_ledger
8205 .as_mut()
8206 .map_or(Ok(()), |ledger| {
8207 ledger.release_scratch_to(scratch_checkpoint)
8208 });
8209 match (result, release) {
8210 (result, Ok(())) => result,
8211 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
8212 }
8213 }
8214
8215 fn prepare_graph_for_targets_transaction(
8216 &mut self,
8217 targets: &[crate::engine::EvaluationTarget],
8218 options: &crate::engine::TargetEvalOptions<'_>,
8219 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
8220 use crate::engine::{
8221 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
8222 TableSelection,
8223 };
8224
8225 self.target_preparation_checkpoint(options.deadline, 0)?;
8226 self.observe_function_semantic_epoch()?;
8227 let assumptions = self.preparation_revisions();
8228 let ledger_at_start = self
8229 .active_resource_ledger
8230 .as_ref()
8231 .map(|ledger| ledger.snapshot());
8232 let diagnostics_len = self.formula_parse_diagnostics.len();
8233 let report_len = self.last_formula_ingest_report.clone();
8234 let request_id = options.request_id.or_else(|| {
8235 self.active_evaluation_resource_request
8236 .as_ref()
8237 .map(|stats| stats.request_id)
8238 });
8239
8240 let mut scope = PrepareScope::Exact;
8241 let mut reasons = Vec::new();
8242 let mut regions = VecDeque::new();
8243 let mut deferred_shared_regions = VecDeque::new();
8244 let mut normalized = Vec::with_capacity(targets.len());
8245 let mut symbol_vertices = VecDeque::new();
8246 for target in targets {
8247 self.target_preparation_checkpoint(options.deadline, 1)?;
8248 match target {
8249 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
8250 if *row == 0 || *col == 0 {
8251 return Err(ExcelError::new(ExcelErrorKind::Ref)
8252 .with_message("target cell coordinates are one-based"));
8253 }
8254 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8255 ExcelError::new(ExcelErrorKind::Ref)
8256 .with_message(format!("target sheet not found: {sheet}"))
8257 })?;
8258 regions.push_back(PreparationRegion {
8259 sheet: sheet.clone(),
8260 sheet_id,
8261 start_row: *row,
8262 start_col: *col,
8263 end_row: *row,
8264 end_col: *col,
8265 });
8266 normalized.push(target.clone());
8267 }
8268 crate::engine::EvaluationTarget::Range(range) => {
8269 if range.start_row == 0
8270 || range.start_col == 0
8271 || range.end_row < range.start_row
8272 || range.end_col < range.start_col
8273 {
8274 return Err(ExcelError::new(ExcelErrorKind::Ref)
8275 .with_message("invalid target range"));
8276 }
8277 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
8278 ExcelError::new(ExcelErrorKind::Ref)
8279 .with_message(format!("target sheet not found: {}", range.sheet))
8280 })?;
8281 regions.push_back(PreparationRegion {
8282 sheet: range.sheet.clone(),
8283 sheet_id,
8284 start_row: range.start_row,
8285 start_col: range.start_col,
8286 end_row: range.end_row,
8287 end_col: range.end_col,
8288 });
8289 normalized.push(target.clone());
8290 }
8291 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
8292 let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
8293 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
8294 symbol_vertices.push_back(entry.vertex);
8295 } else {
8296 Self::widen_target_preparation(
8297 options.opaque_policy,
8298 &mut scope,
8299 &mut reasons,
8300 OpaqueReason::UnresolvedName,
8301 )?;
8302 }
8303 normalized.push(target.clone());
8304 }
8305 crate::engine::EvaluationTarget::Table { name, selection } => {
8306 if let Some(entry) = self.graph.resolve_table_entry(name) {
8307 let region = self.table_selection_region(entry, selection)?;
8308 symbol_vertices.push_back(entry.vertex);
8309 regions.push_back(region);
8310 } else {
8311 Self::widen_target_preparation(
8312 options.opaque_policy,
8313 &mut scope,
8314 &mut reasons,
8315 OpaqueReason::UnresolvedTable,
8316 )?;
8317 }
8318 normalized.push(target.clone());
8319 }
8320 }
8321 }
8322
8323 let mut visited_regions = FxHashSet::default();
8324 let mut visited_vertices = FxHashSet::default();
8325 let mut selected = FxHashSet::default();
8326 let mut prepared = Vec::new();
8327 let mut pending_diagnostics = Vec::new();
8328 let mut planning_requests = BTreeSet::new();
8329 let mut selected_cells = Vec::new();
8330 let mut workbook_seeded = false;
8331 let mut sheet_scope_seeded = BTreeSet::new();
8332 let mut indexed_query_sheets = FxHashSet::default();
8333 let mut discovery_scratch_reserved = 0u64;
8334 let mut package_encountered = false;
8335 let mut selected_package_sheets = FxHashSet::default();
8336 let mut selected_package_points: BTreeMap<String, BTreeSet<(u32, u32)>> = BTreeMap::new();
8337 let mut prepared_packages: Vec<PreparedTargetSourcePackage> = Vec::new();
8338 let authoritative_with_ordinary = false;
8339 let has_unknown_package_sheet = self
8340 .staged_formula_index
8341 .package_sheets()
8342 .any(|sheet| self.graph.sheet_id(sheet).is_none());
8343
8344 loop {
8345 if let PrepareScope::Sheets(sheets) = &scope {
8346 for sheet in sheets.clone() {
8347 if !sheet_scope_seeded.insert(sheet.clone()) {
8348 continue;
8349 }
8350 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8351 package_encountered = true;
8352 Self::widen_target_preparation(
8353 options.opaque_policy,
8354 &mut scope,
8355 &mut reasons,
8356 OpaqueReason::UnsupportedSourceSemantics,
8357 )?;
8358 continue;
8359 };
8360 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
8361 self.target_preparation_checkpoint(options.deadline, 1)?;
8362 regions.push_back(PreparationRegion {
8363 sheet: sheet.clone(),
8364 sheet_id,
8365 start_row: lease.row,
8366 start_col: lease.col,
8367 end_row: lease.row,
8368 end_col: lease.col,
8369 });
8370 }
8371 if self
8372 .staged_formula_index
8373 .package_lease_for_sheet(&sheet)
8374 .is_some()
8375 {
8376 regions.push_back(PreparationRegion {
8377 sheet: sheet.clone(),
8378 sheet_id,
8379 start_row: 1,
8380 start_col: 1,
8381 end_row: self.workbook_load_limits.max_sheet_rows,
8382 end_col: self.workbook_load_limits.max_sheet_cols,
8383 });
8384 }
8385 }
8386 }
8387 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
8388 workbook_seeded = true;
8389 for (sheet, lease) in self.staged_formula_index.all_leases() {
8390 self.target_preparation_checkpoint(options.deadline, 1)?;
8391 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8392 package_encountered = true;
8393 Self::widen_target_preparation(
8394 options.opaque_policy,
8395 &mut scope,
8396 &mut reasons,
8397 OpaqueReason::UnsupportedSourceSemantics,
8398 )?;
8399 continue;
8400 };
8401 regions.push_back(PreparationRegion {
8402 sheet,
8403 sheet_id,
8404 start_row: lease.row,
8405 start_col: lease.col,
8406 end_row: lease.row,
8407 end_col: lease.col,
8408 });
8409 }
8410 let package_sheets = self
8411 .staged_formula_index
8412 .package_sheets()
8413 .map(str::to_string)
8414 .collect::<Vec<_>>();
8415 for sheet in package_sheets {
8416 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
8417 package_encountered = true;
8418 Self::widen_target_preparation(
8419 options.opaque_policy,
8420 &mut scope,
8421 &mut reasons,
8422 OpaqueReason::UnsupportedSourceSemantics,
8423 )?;
8424 continue;
8425 };
8426 regions.push_back(PreparationRegion {
8427 sheet,
8428 sheet_id,
8429 start_row: 1,
8430 start_col: 1,
8431 end_row: self.workbook_load_limits.max_sheet_rows,
8432 end_col: self.workbook_load_limits.max_sheet_cols,
8433 });
8434 }
8435 }
8436
8437 let allow_partial_shared = regions.is_empty() && symbol_vertices.is_empty();
8440 let next_region = if allow_partial_shared {
8441 deferred_shared_regions.pop_front()
8442 } else {
8443 regions.pop_front()
8444 };
8445 let Some(region) = next_region else {
8446 if let Some(vertex) = symbol_vertices.pop_front() {
8447 self.target_preparation_checkpoint(options.deadline, 1)?;
8448 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
8449 continue;
8450 }
8451 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
8452 if let Some(ast) = self.graph.get_formula(vertex) {
8453 let snapshot =
8454 self.target_planning_snapshot(&ast, &mut planning_requests)?;
8455 if let Some(reason) =
8456 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8457 {
8458 return Err(Self::preparation_stale(
8459 reason,
8460 "target planning snapshot became stale during discovery",
8461 ));
8462 }
8463 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
8464 if let Some(reason) =
8465 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
8466 {
8467 if reason == OpaqueReason::DynamicReference
8468 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
8469 {
8470 let sheet = self.graph.get_vertex_sheet_id(vertex);
8471 let sheet = self.graph.sheet_name(sheet).to_string();
8472 Self::widen_target_preparation_to_sheet(
8473 options.opaque_policy,
8474 &mut scope,
8475 &mut reasons,
8476 reason,
8477 &sheet,
8478 )?;
8479 } else {
8480 Self::widen_target_preparation(
8481 options.opaque_policy,
8482 &mut scope,
8483 &mut reasons,
8484 reason,
8485 )?;
8486 }
8487 }
8488 } else if vertex_is_dynamic {
8489 Self::widen_target_preparation(
8490 options.opaque_policy,
8491 &mut scope,
8492 &mut reasons,
8493 OpaqueReason::DynamicReference,
8494 )?;
8495 }
8496 if let Some(anchor) = self
8497 .graph
8498 .get_cell_ref(vertex)
8499 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
8500 {
8501 symbol_vertices.push_back(anchor);
8502 }
8503 if let Some(cell) = self.graph.get_cell_ref(vertex) {
8504 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
8505 regions.push_back(PreparationRegion {
8506 sheet,
8507 sheet_id: cell.sheet_id,
8508 start_row: cell.coord.row() + 1,
8509 start_col: cell.coord.col() + 1,
8510 end_row: cell.coord.row() + 1,
8511 end_col: cell.coord.col() + 1,
8512 });
8513 }
8514 match self.graph.authority_vertex_precedents(vertex) {
8518 Some(precedents) => {
8519 for (sheet_id, rect) in precedents {
8520 self.target_preparation_checkpoint(options.deadline, 1)?;
8521 if sheet_id == crate::engine::authority::geom::SYMBOL_SHEET {
8522 for slot in rect.r0..=rect.r1 {
8523 if let Some(symbol) =
8524 self.graph.authority_host().symbols().vertex(slot)
8525 {
8526 symbol_vertices.push_back(symbol);
8527 }
8528 }
8529 continue;
8530 }
8531 let sheet = self.graph.sheet_name(sheet_id).to_string();
8532 regions.push_back(PreparationRegion {
8533 sheet,
8534 sheet_id,
8535 start_row: rect.r0 + 1,
8536 start_col: rect.c0 + 1,
8537 end_row: (rect.r1 + 1)
8538 .min(self.workbook_load_limits.max_sheet_rows),
8539 end_col: (rect.c1 + 1)
8540 .min(self.workbook_load_limits.max_sheet_cols),
8541 });
8542 }
8543 }
8544 None => {
8546 if Self::widen_target_preparation(
8547 options.opaque_policy,
8548 &mut scope,
8549 &mut reasons,
8550 OpaqueReason::UnresolvedCrossSheetBinding,
8551 )? {
8552 continue;
8553 }
8554 }
8555 }
8556 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
8557 match &name.definition {
8558 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
8559 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
8560 sheet_id: cell.sheet_id,
8561 start_row: cell.coord.row() + 1,
8562 start_col: cell.coord.col() + 1,
8563 end_row: cell.coord.row() + 1,
8564 end_col: cell.coord.col() + 1,
8565 }),
8566 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
8567 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
8568 sheet_id: range.start.sheet_id,
8569 start_row: range.start.coord.row() + 1,
8570 start_col: range.start.coord.col() + 1,
8571 end_row: range.end.coord.row() + 1,
8572 end_col: range.end.coord.col() + 1,
8573 }),
8574 NamedDefinition::Formula {
8575 ast,
8576 dependencies,
8577 range_deps,
8578 } => {
8579 let snapshot =
8580 self.target_planning_snapshot(ast, &mut planning_requests)?;
8581 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
8582 &snapshot,
8583 &assumptions,
8584 ) {
8585 return Err(Self::preparation_stale(
8586 reason,
8587 "target planning snapshot became stale during discovery",
8588 ));
8589 }
8590 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
8591 Self::widen_target_preparation(
8592 options.opaque_policy,
8593 &mut scope,
8594 &mut reasons,
8595 reason,
8596 )?;
8597 }
8598 for dependency in dependencies {
8599 self.target_preparation_checkpoint(options.deadline, 1)?;
8600 symbol_vertices.push_back(*dependency);
8601 }
8602 for range in range_deps {
8603 self.target_preparation_checkpoint(options.deadline, 1)?;
8604 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
8612 let Ok(sheet_id) =
8613 self.resolve_sheet_locator(&range.sheet, context_sheet)
8614 else {
8615 if Self::widen_target_preparation(
8616 options.opaque_policy,
8617 &mut scope,
8618 &mut reasons,
8619 OpaqueReason::UnresolvedCrossSheetBinding,
8620 )? {
8621 break;
8622 }
8623 continue;
8624 };
8625 regions.push_back(PreparationRegion {
8626 sheet: self.graph.sheet_name(sheet_id).to_string(),
8627 sheet_id,
8628 start_row: range
8629 .start_row
8630 .map_or(1, |bound| bound.index + 1),
8631 start_col: range
8632 .start_col
8633 .map_or(1, |bound| bound.index + 1),
8634 end_row: range.end_row.map_or(
8635 self.workbook_load_limits.max_sheet_rows,
8636 |bound| bound.index + 1,
8637 ),
8638 end_col: range.end_col.map_or(
8639 self.workbook_load_limits.max_sheet_cols,
8640 |bound| bound.index + 1,
8641 ),
8642 });
8643 }
8644 }
8645 NamedDefinition::Literal(_) => {}
8646 }
8647 }
8648 if let Some(table) = self.graph.table_by_vertex(vertex) {
8649 regions
8650 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
8651 }
8652 continue;
8653 }
8654 break;
8655 };
8656
8657 self.target_preparation_checkpoint(options.deadline, 1)?;
8658 if !visited_regions.insert(region.clone()) && !allow_partial_shared {
8659 continue;
8660 }
8661 if let PrepareScope::Sheets(sheets) = &mut scope
8662 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
8663 {
8664 sheets.push(region.sheet.clone());
8665 sheets.sort();
8666 }
8667 let package_match = self.staged_formula_index.package_for_region(
8668 ®ion.sheet,
8669 region.start_row,
8670 region.start_col,
8671 region.end_row,
8672 region.end_col,
8673 );
8674 let package_lease = match package_match {
8675 Some(Ok(lease)) => Some(lease),
8676 Some(Err(()))
8677 if region.start_row == 1
8678 && region.start_col == 1
8679 && region.end_row == self.workbook_load_limits.max_sheet_rows
8680 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
8681 {
8682 self.staged_formula_index
8683 .package_lease_for_sheet(®ion.sheet)
8684 }
8685 Some(Err(())) => {
8686 Self::widen_target_preparation(
8687 options.opaque_policy,
8688 &mut scope,
8689 &mut reasons,
8690 OpaqueReason::DeferredSourcePackage,
8691 )?;
8692 None
8693 }
8694 None => None,
8695 };
8696 let compatibility_before_package_replay = package_lease.is_some()
8697 && (authoritative_with_ordinary || has_unknown_package_sheet);
8698 let package_lease = if compatibility_before_package_replay {
8699 package_encountered = true;
8700 Self::widen_target_preparation(
8701 options.opaque_policy,
8702 &mut scope,
8703 &mut reasons,
8704 OpaqueReason::UnsupportedSourceSemantics,
8705 )?;
8706 None
8707 } else {
8708 package_lease
8709 };
8710 if let Some(package_lease) = package_lease
8711 && !selected_package_sheets.contains(®ion.sheet)
8712 {
8713 self.target_preparation_checkpoint(options.deadline, 1)?;
8714 let mut points = self.staged_formula_index.package_points_in_region(
8715 ®ion.sheet,
8716 region.start_row,
8717 region.start_col,
8718 region.end_row,
8719 region.end_col,
8720 );
8721 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8722 points.retain(|point| !selected.contains(point));
8723 }
8724
8725 if let Some(selected) = selected_package_points.get(®ion.sheet) {
8726 points.retain(|point| !selected.contains(point));
8727 }
8728 let permit_partial = allow_partial_shared
8729 || (points.len() == 1
8730 && regions.is_empty()
8731 && symbol_vertices.is_empty()
8732 && deferred_shared_regions.is_empty());
8733 let partial = self.prepare_target_exact_source_selection(
8734 ®ion.sheet,
8735 package_lease,
8736 points,
8737 selected_package_points
8738 .get(®ion.sheet)
8739 .unwrap_or(&BTreeSet::new()),
8740 permit_partial,
8741 options.deadline,
8742 &mut discovery_scratch_reserved,
8743 )?;
8744 let mut package = if let Some(mut package) = partial {
8745 if package.deferred_shared {
8746 deferred_shared_regions.push_back(region.clone());
8747 }
8748 if !package.direct_domains.is_empty() {
8752 for prior in prepared_packages
8753 .iter_mut()
8754 .filter(|prior| prior.sheet == region.sheet)
8755 {
8756 prior
8757 .replay_records
8758 .retain(|record| !package.direct_contains(record.row, record.col));
8759 prior
8760 .legacy
8761 .retain(|(row, col, _, _)| !package.direct_contains(*row, *col));
8762 if let Some(points) = prior.selected_points.as_mut() {
8763 points.retain(|&(row, col)| !package.direct_contains(row, col));
8764 }
8765 }
8766 }
8767 if selected_package_points.contains_key(®ion.sheet) {
8768 package.source_report = Default::default();
8769 }
8770 let points = package.selected_points.as_ref().unwrap();
8771 if !points.is_empty() {
8772 selected_package_points
8773 .entry(region.sheet.clone())
8774 .or_default()
8775 .extend(points.iter().copied());
8776 }
8777 package
8778 } else {
8779 selected_package_sheets.insert(region.sheet.clone());
8780 self.prepare_target_source_package(
8781 ®ion.sheet,
8782 package_lease,
8783 options.deadline,
8784 )?
8785 };
8786 if package
8787 .selected_points
8788 .as_ref()
8789 .is_none_or(|points| !points.is_empty())
8790 {
8791 let mut final_fallback = BTreeMap::new();
8792 for record in package.fallback_records() {
8793 final_fallback.insert((record.row, record.col), record.clone());
8794 }
8795 let batch = self.formula_batch_from_exact_replay(
8796 ®ion.sheet,
8797 final_fallback.into_values(),
8798 )?;
8799 for record in batch.formulas {
8800 self.target_preparation_checkpoint(options.deadline, 1)?;
8801 let ast = self
8802 .graph
8803 .data_store()
8804 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8805 .ok_or_else(|| {
8806 ExcelError::new(ExcelErrorKind::Value)
8807 .with_message("target fallback AST is unavailable")
8808 })?;
8809 let snapshot =
8810 self.target_planning_snapshot(&ast, &mut planning_requests)?;
8811 if let Some(reason) =
8812 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
8813 {
8814 return Err(Self::preparation_stale(
8815 reason,
8816 "target fallback planning snapshot became stale during discovery",
8817 ));
8818 }
8819 let proven_sheet_local_dynamic =
8820 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
8821 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
8822 if reason == OpaqueReason::DynamicReference
8823 && proven_sheet_local_dynamic
8824 {
8825 Self::widen_target_preparation_to_sheet(
8826 options.opaque_policy,
8827 &mut scope,
8828 &mut reasons,
8829 reason,
8830 ®ion.sheet,
8831 )?;
8832 } else {
8833 Self::widen_target_preparation(
8834 options.opaque_policy,
8835 &mut scope,
8836 &mut reasons,
8837 reason,
8838 )?;
8839 }
8840 }
8841 let placement = CellRef::new(
8842 package.sheet_id,
8843 Coord::from_excel(record.row, record.col, true, true),
8844 );
8845 let ingested = self
8846 .graph
8847 .ingest_pipeline(&snapshot)
8848 .enable_function_semantics()
8849 .ingest_formula(
8850 FormulaAstInput::RawArena(record.ast_id),
8851 placement,
8852 record.formula_text,
8853 )?;
8854 if ingested.dep_plan.dynamic {
8855 if proven_sheet_local_dynamic {
8856 Self::widen_target_preparation_to_sheet(
8857 options.opaque_policy,
8858 &mut scope,
8859 &mut reasons,
8860 OpaqueReason::DynamicReference,
8861 ®ion.sheet,
8862 )?;
8863 } else {
8864 Self::widen_target_preparation(
8865 options.opaque_policy,
8866 &mut scope,
8867 &mut reasons,
8868 OpaqueReason::DynamicReference,
8869 )?;
8870 }
8871 }
8872 for dep in &ingested.dep_plan.direct_cell_deps {
8873 self.target_preparation_checkpoint(options.deadline, 1)?;
8874 regions.push_back(PreparationRegion {
8875 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
8876 sheet_id: dep.sheet_id,
8877 start_row: dep.coord.row().saturating_add(1),
8878 start_col: dep.coord.col().saturating_add(1),
8879 end_row: dep.coord.row().saturating_add(1),
8880 end_col: dep.coord.col().saturating_add(1),
8881 });
8882 }
8883 for range in &ingested.dep_plan.range_deps {
8884 self.target_preparation_checkpoint(options.deadline, 1)?;
8885 let Ok(dependency_sheet) =
8888 self.resolve_sheet_locator(&range.sheet, package.sheet_id)
8889 else {
8890 Self::widen_target_preparation(
8891 options.opaque_policy,
8892 &mut scope,
8893 &mut reasons,
8894 OpaqueReason::UnresolvedCrossSheetBinding,
8895 )?;
8896 continue;
8897 };
8898 regions.push_back(PreparationRegion {
8899 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
8900 sheet_id: dependency_sheet,
8901 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
8902 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
8903 end_row: range
8904 .end_row
8905 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
8906 bound.index + 1
8907 }),
8908 end_col: range
8909 .end_col
8910 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
8911 bound.index + 1
8912 }),
8913 });
8914 }
8915 for name in ingested
8916 .dep_plan
8917 .resolved_named_refs
8918 .iter()
8919 .chain(&ingested.dep_plan.named_refs)
8920 {
8921 self.target_preparation_checkpoint(options.deadline, 1)?;
8922 if let Some(entry) =
8923 self.graph.resolve_name_entry(name, package.sheet_id)
8924 {
8925 symbol_vertices.push_back(entry.vertex);
8926 } else if self.graph.resolve_source_scalar_entry(name).is_none()
8927 && self.graph.resolve_source_table_entry(name).is_none()
8928 {
8929 Self::widen_target_preparation(
8930 options.opaque_policy,
8931 &mut scope,
8932 &mut reasons,
8933 OpaqueReason::UnresolvedName,
8934 )?;
8935 }
8936 }
8937 for table in &ingested.dep_plan.table_refs {
8938 self.target_preparation_checkpoint(options.deadline, 1)?;
8939 if let Some(entry) = self.graph.resolve_table_entry(table) {
8940 symbol_vertices.push_back(entry.vertex);
8941 } else if self.graph.resolve_source_table_entry(table).is_none() {
8942 Self::widen_target_preparation(
8943 options.opaque_policy,
8944 &mut scope,
8945 &mut reasons,
8946 OpaqueReason::UnresolvedTable,
8947 )?;
8948 }
8949 }
8950 package.legacy.push((
8951 record.row,
8952 record.col,
8953 ingested.ast_id,
8954 ingested.dep_plan,
8955 ));
8956 }
8957 prepared_packages.push(package);
8958 }
8959 }
8960 let leases = self.staged_formula_index.leases_in_region(
8961 ®ion.sheet,
8962 region.start_row,
8963 region.start_col,
8964 region.end_row,
8965 region.end_col,
8966 );
8967 for lease in leases {
8968 self.target_preparation_checkpoint(options.deadline, 1)?;
8969 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
8970 ExcelError::new(ExcelErrorKind::Ref)
8971 .with_message(format!("staged formula sheet not found: {}", region.sheet))
8972 })?;
8973 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
8974 if !selected.insert(key) {
8975 continue;
8976 }
8977 let text = self
8978 .staged_formulas
8979 .get(®ion.sheet)
8980 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
8981 .ok_or_else(|| {
8982 ExcelError::new(ExcelErrorKind::Value)
8983 .with_message("staged formula index is stale")
8984 })?
8985 .to_string();
8986 let formula = if text.starts_with('=') {
8987 text.clone()
8988 } else {
8989 format!("={text}")
8990 };
8991 self.target_preparation_checkpoint(options.deadline, 1)?;
8992 let ast = match formualizer_parse::parser::parse(&formula) {
8993 Ok(ast) => ast,
8994 Err(error) => {
8995 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
8996 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8997 format!(
8998 "Formula parse error at {}!{}{}: {error}",
8999 region.sheet,
9000 col_letters_from_1based(lease.col)
9001 .unwrap_or_else(|_| "?".to_string()),
9002 lease.row
9003 ),
9004 ));
9005 }
9006 pending_diagnostics.push(FormulaParseDiagnostic {
9007 sheet: region.sheet.clone(),
9008 row: lease.row,
9009 col: lease.col,
9010 formula: formula.clone(),
9011 message: error.to_string(),
9012 policy: self.config.formula_parse_policy,
9013 });
9014 match self.config.formula_parse_policy {
9015 FormulaParsePolicy::KeepCachedValue => {
9016 selected_cells.push(
9017 formualizer_common::RangeAddress::new(
9018 region.sheet.clone(),
9019 lease.row,
9020 lease.col,
9021 lease.row,
9022 lease.col,
9023 )
9024 .expect("selected staged coordinates are valid"),
9025 );
9026 prepared.push(PreparedOrdinaryStagedFormula {
9027 sheet: region.sheet.clone(),
9028 sheet_id,
9029 lease,
9030 ast_id: None,
9031 plan: None,
9032 });
9033 continue;
9034 }
9035 FormulaParsePolicy::AsText => ASTNode::new(
9036 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
9037 None,
9038 ),
9039 FormulaParsePolicy::CoerceToError => ASTNode::new(
9040 ASTNodeType::Literal(LiteralValue::Error(
9041 ExcelError::new(ExcelErrorKind::Error)
9042 .with_message(format!("Malformed formula: {error}")),
9043 )),
9044 None,
9045 ),
9046 FormulaParsePolicy::Strict => unreachable!(),
9047 }
9048 }
9049 };
9050 self.target_preparation_checkpoint(options.deadline, 1)?;
9051 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
9052 self.target_preparation_checkpoint(options.deadline, 1)?;
9053 if let Some(reason) =
9054 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
9055 {
9056 return Err(Self::preparation_stale(
9057 reason,
9058 "target planning snapshot became stale during discovery",
9059 ));
9060 }
9061 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
9062 if reason == OpaqueReason::DynamicReference
9063 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
9064 {
9065 Self::widen_target_preparation_to_sheet(
9066 options.opaque_policy,
9067 &mut scope,
9068 &mut reasons,
9069 reason,
9070 ®ion.sheet,
9071 )?;
9072 } else {
9073 Self::widen_target_preparation(
9074 options.opaque_policy,
9075 &mut scope,
9076 &mut reasons,
9077 reason,
9078 )?;
9079 }
9080 }
9081 let proven_sheet_local_dynamic =
9082 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
9083 let placement = CellRef::new(
9084 sheet_id,
9085 Coord::from_excel(lease.row, lease.col, true, true),
9086 );
9087 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
9088 FormulaAstInput::Tree(ast),
9089 placement,
9090 Some(Arc::from(formula)),
9091 )?;
9092 self.target_preparation_checkpoint(options.deadline, 1)?;
9093 if ingested.dep_plan.dynamic {
9094 if proven_sheet_local_dynamic {
9095 Self::widen_target_preparation_to_sheet(
9096 options.opaque_policy,
9097 &mut scope,
9098 &mut reasons,
9099 OpaqueReason::DynamicReference,
9100 ®ion.sheet,
9101 )?;
9102 } else {
9103 Self::widen_target_preparation(
9104 options.opaque_policy,
9105 &mut scope,
9106 &mut reasons,
9107 OpaqueReason::DynamicReference,
9108 )?;
9109 }
9110 }
9111 for dep in &ingested.dep_plan.direct_cell_deps {
9112 self.target_preparation_checkpoint(options.deadline, 1)?;
9113 regions.push_back(PreparationRegion {
9114 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
9115 sheet_id: dep.sheet_id,
9116 start_row: dep.coord.row() + 1,
9117 start_col: dep.coord.col() + 1,
9118 end_row: dep.coord.row() + 1,
9119 end_col: dep.coord.col() + 1,
9120 });
9121 }
9122 for range in &ingested.dep_plan.range_deps {
9123 self.target_preparation_checkpoint(options.deadline, 1)?;
9124 let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
9126 else {
9127 Self::widen_target_preparation(
9128 options.opaque_policy,
9129 &mut scope,
9130 &mut reasons,
9131 OpaqueReason::UnresolvedCrossSheetBinding,
9132 )?;
9133 continue;
9134 };
9135 regions.push_back(PreparationRegion {
9136 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
9137 sheet_id: dependency_sheet,
9138 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
9139 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
9140 end_row: range
9141 .end_row
9142 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
9143 bound.index + 1
9144 }),
9145 end_col: range
9146 .end_col
9147 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
9148 bound.index + 1
9149 }),
9150 });
9151 }
9152 for name in ingested
9153 .dep_plan
9154 .resolved_named_refs
9155 .iter()
9156 .chain(&ingested.dep_plan.named_refs)
9157 {
9158 self.target_preparation_checkpoint(options.deadline, 1)?;
9159 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9160 symbol_vertices.push_back(entry.vertex);
9161 } else if self.graph.resolve_source_scalar_entry(name).is_none()
9162 && self.graph.resolve_source_table_entry(name).is_none()
9163 {
9164 Self::widen_target_preparation(
9165 options.opaque_policy,
9166 &mut scope,
9167 &mut reasons,
9168 OpaqueReason::UnresolvedName,
9169 )?;
9170 }
9171 }
9172 for table in &ingested.dep_plan.table_refs {
9173 self.target_preparation_checkpoint(options.deadline, 1)?;
9174 if let Some(entry) = self.graph.resolve_table_entry(table) {
9175 symbol_vertices.push_back(entry.vertex);
9176 } else if self.graph.resolve_source_table_entry(table).is_none() {
9177 Self::widen_target_preparation(
9178 options.opaque_policy,
9179 &mut scope,
9180 &mut reasons,
9181 OpaqueReason::UnresolvedTable,
9182 )?;
9183 }
9184 }
9185 selected_cells.push(
9186 formualizer_common::RangeAddress::new(
9187 region.sheet.clone(),
9188 lease.row,
9189 lease.col,
9190 lease.row,
9191 lease.col,
9192 )
9193 .expect("selected staged coordinates are valid"),
9194 );
9195 prepared.push(PreparedOrdinaryStagedFormula {
9196 sheet: region.sheet.clone(),
9197 sheet_id,
9198 lease,
9199 ast_id: Some(ingested.ast_id),
9200 plan: Some(ingested.dep_plan),
9201 });
9202 }
9203
9204 if indexed_query_sheets.insert(region.sheet_id) {
9205 self.graph.prepare_sheet_index_for_query(region.sheet_id);
9206 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
9207 .saturating_mul(32);
9208 self.reserve_graph_source_scratch(bytes)?;
9209 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
9210 }
9211 let spill_anchors = self.graph.spill_anchors_in_region(
9212 region.sheet_id,
9213 region.start_row - 1,
9214 region.start_col - 1,
9215 region.end_row - 1,
9216 region.end_col - 1,
9217 );
9218 for anchor in spill_anchors {
9219 self.target_preparation_checkpoint(options.deadline, 1)?;
9220 symbol_vertices.push_back(anchor);
9221 }
9222 let vertices = self.graph.vertices_in_region(
9223 region.sheet_id,
9224 region.start_row - 1,
9225 region.end_row - 1,
9226 region.start_col - 1,
9227 region.end_col - 1,
9228 );
9229 for vertex in vertices {
9230 self.target_preparation_checkpoint(options.deadline, 1)?;
9231 symbol_vertices.push_back(vertex);
9232 }
9233 }
9234
9235 #[cfg(test)]
9236 self.target_preparation_fault(
9237 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
9238 )?;
9239
9240 if package_encountered {
9241 Self::widen_target_preparation(
9242 options.opaque_policy,
9243 &mut scope,
9244 &mut reasons,
9245 OpaqueReason::UnsupportedSourceSemantics,
9246 )?;
9247 self.target_preparation_checkpoint(options.deadline, 0)?;
9248 let selected_count = self.staged_formula_count();
9249 let selected_packages = self
9250 .staged_formulas
9251 .values()
9252 .filter_map(|staged| staged.deferred_package.as_ref())
9253 .map(|package| package.families.len() + package.partitioned_families.len())
9254 .sum();
9255 self.formula_parse_diagnostics.truncate(diagnostics_len);
9256 self.last_formula_ingest_report = report_len;
9257 #[cfg(test)]
9258 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9259 hook();
9260 }
9261 self.target_preparation_checkpoint(options.deadline, 0)?;
9262 #[cfg(test)]
9263 self.target_preparation_fault(
9264 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9265 )?;
9266 let current_revisions = self.preparation_revisions();
9267 if let Some(reason) = Self::preparation_revision_stale_reason(
9268 &assumptions,
9269 ¤t_revisions,
9270 &planning_requests,
9271 true,
9272 ) {
9273 return Err(Self::preparation_stale(
9274 reason,
9275 "target compatibility preparation plan is stale",
9276 ));
9277 }
9278 #[cfg(test)]
9279 self.target_preparation_fault(
9280 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9281 )?;
9282 let commit_work_before = self
9283 .active_resource_ledger
9284 .as_ref()
9285 .map_or(0, |ledger| ledger.snapshot().work_charged);
9286 let commit_started = crate::instant::FzInstant::now();
9287 self.build_graph_all_unobserved()?;
9288 let commit_window = commit_started.elapsed();
9289 let ledger_after = self
9290 .active_resource_ledger
9291 .as_ref()
9292 .map(|ledger| ledger.snapshot());
9293 let actual_commit_work = ledger_after
9294 .map_or(0, |snapshot| snapshot.work_charged)
9295 .saturating_sub(commit_work_before);
9296 let observed_scratch_bytes = ledger_after
9297 .map_or(0, |snapshot| snapshot.scratch_peak)
9298 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9299 let revisions = assumptions.clone();
9300 let report = PreparedTargetGraphReport {
9301 request_id: request_id.unwrap_or_default(),
9302 requested_targets: targets.len(),
9303 normalized_regions: visited_regions.len(),
9304 normalized_target_list: normalized,
9305 selected_staged_cells: selected_count,
9306 selected_source_families: selected_packages,
9307 retained_staged_cells: self.staged_formula_count(),
9308 selected_cells,
9309 retained_cells: Vec::new(),
9310 widened_scope: PrepareScope::Workbook,
9311 widening_reasons: reasons,
9312 revisions,
9313 commit_window,
9314 estimated_scratch_bytes: discovery_scratch_reserved
9315 .saturating_add((selected_count as u64).saturating_mul(256)),
9316 observed_scratch_bytes,
9317 estimated_commit_work: selected_count as u64,
9318 actual_commit_work,
9319 outcome: PreparationOutcome::CompatibilityPrepared,
9320 };
9321 self.observe_target_preparation_report(&report);
9322 return Ok(report);
9323 }
9324
9325 prepared.sort_by_key(|formula| formula.lease.insertion_order);
9326 for package in &prepared_packages {
9327 if let Some(points) = &package.selected_points {
9328 selected_cells.extend(points.iter().filter_map(|&(row, col)| {
9329 formualizer_common::RangeAddress::new(&package.sheet, row, col, row, col).ok()
9330 }));
9331 continue;
9332 }
9333 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
9334 formualizer_common::RangeAddress::new(
9335 package.sheet.clone(),
9336 record.row,
9337 record.col,
9338 record.row,
9339 record.col,
9340 )
9341 .ok()
9342 }));
9343 }
9344 let (legacy_graph, planned_formula_count) =
9345 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
9346 let new_vertices = legacy_graph.new_vertex_count();
9347 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
9348 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9349 })?;
9350 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
9351 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
9352 })?;
9353 let current = self.graph.baseline_stats();
9354 let final_vertices = current
9355 .graph_vertex_count
9356 .checked_add(new_vertices)
9357 .ok_or_else(|| {
9358 crate::engine::ResourceLedgerError::Exhausted(
9359 formualizer_common::ResourceExhaustionDetail {
9360 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9361 limit: u64::MAX,
9362 observed: u64::MAX,
9363 request_id,
9364 },
9365 )
9366 .into_excel_error()
9367 })?;
9368 let final_edges = current
9369 .graph_edge_count
9370 .checked_sub(removed_edges)
9371 .and_then(|count| count.checked_add(new_edges))
9372 .ok_or_else(|| {
9373 crate::engine::ResourceLedgerError::Exhausted(
9374 formualizer_common::ResourceExhaustionDetail {
9375 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
9376 limit: u64::MAX,
9377 observed: u64::MAX,
9378 request_id,
9379 },
9380 )
9381 .into_excel_error()
9382 })?;
9383 #[cfg(test)]
9384 self.target_preparation_fault(
9385 crate::engine::target_preparation::TargetPreparationFault::Admission,
9386 )?;
9387 let resource = |reason, limit: u64, observed: u64| {
9388 crate::engine::ResourceLedgerError::Exhausted(
9389 formualizer_common::ResourceExhaustionDetail {
9390 reason,
9391 limit,
9392 observed,
9393 request_id,
9394 },
9395 )
9396 .into_excel_error()
9397 };
9398 let admission = crate::engine::resource_ledger::GraphAdmission {
9399 final_vertices,
9400 final_edges,
9401 materialization_cells: planned_formula_count as u64,
9402 added_vertices: new_vertices,
9403 added_edges: new_edges,
9404 };
9405 let materialized_bytes = admission
9406 .materialized_graph_bytes()
9407 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
9408 if let Err(error) = self.preflight_graph_admission(admission) {
9409 if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
9410 self.observe_target_admission_failure(detail.reason);
9411 }
9412 return Err(error);
9413 }
9414 let selected_package_records = prepared_packages
9415 .iter()
9416 .map(|package| package.replay_records.len() as u64)
9417 .sum::<u64>();
9418 let planned_working_bytes = (prepared.len() as u64)
9419 .saturating_add(selected_package_records)
9420 .saturating_mul(256)
9421 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
9422 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
9423 .saturating_add((selected.len() as u64).saturating_mul(96))
9424 .saturating_add(materialized_bytes);
9425 let residual_scratch = selected_package_points
9426 .iter()
9427 .map(|(sheet, points)| {
9428 let source = self
9429 .staged_formulas
9430 .get(sheet)
9431 .unwrap()
9432 .deferred_package
9433 .as_ref()
9434 .unwrap();
9435 (source.families.len() as u64)
9436 .saturating_mul(256)
9437 .saturating_add(
9438 source
9439 .partitioned_families
9440 .iter()
9441 .map(|family| {
9442 256u64
9443 .saturating_add(family.fragments.len() as u64 * 32)
9444 .saturating_add(family.legacy_members.len() as u64 * 32)
9445 })
9446 .sum::<u64>(),
9447 )
9448 .saturating_add(points.len() as u64 * 32)
9451 .saturating_add(
9452 prepared_packages
9453 .iter()
9454 .filter(|package| &package.sheet == sheet)
9455 .map(|package| package.replay_records.len() as u64 * 480)
9456 .sum::<u64>(),
9457 )
9458 })
9459 .sum::<u64>();
9460 let scratch_bytes = discovery_scratch_reserved
9461 .saturating_add(planned_working_bytes)
9462 .saturating_add(residual_scratch);
9463 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
9464 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
9465 self.observe_target_admission_failure(
9466 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9467 );
9468 return Err(error);
9469 }
9470
9471 let mut residual_sources = BTreeMap::new();
9472 let mut residual_owners = BTreeMap::new();
9473 for (sheet, points) in &selected_package_points {
9474 let selected: BTreeMap<_, BTreeSet<_>> = prepared_packages
9475 .iter()
9476 .filter(|package| &package.sheet == sheet)
9477 .flat_map(|package| &package.replay_records)
9478 .filter_map(|record| {
9479 record.partition_owner.or(record.family).map(|owner| {
9480 (
9481 owner,
9482 crate::engine::SourceCoord {
9483 row: record.row - 1,
9484 col: record.col - 1,
9485 },
9486 )
9487 })
9488 })
9489 .fold(BTreeMap::new(), |mut map, (owner, coord)| {
9490 map.entry(owner).or_default().insert(coord);
9491 map
9492 });
9493 let metadata_work = self
9494 .staged_formulas
9495 .get(sheet)
9496 .unwrap()
9497 .deferred_package
9498 .as_ref()
9499 .map_or(0, |source| {
9500 source.families.len() as u64
9501 + source
9502 .partitioned_families
9503 .iter()
9504 .map(|family| {
9505 1 + family.fragments.len() as u64 * family.fragments.len() as u64
9506 + family.legacy_members.len() as u64
9507 })
9508 .sum::<u64>()
9509 });
9510 let split_work = selected
9511 .values()
9512 .map(|points| points.len() as u64 * 128)
9513 .sum::<u64>();
9514 self.target_preparation_checkpoint(
9515 options.deadline,
9516 metadata_work
9517 .saturating_add(split_work)
9518 .saturating_add(points.len() as u64),
9519 )?;
9520 let source = self
9521 .staged_formulas
9522 .get(sheet)
9523 .unwrap()
9524 .deferred_package
9525 .as_ref()
9526 .unwrap();
9527 let complete: BTreeSet<_> = prepared_packages
9528 .iter()
9529 .filter(|package| &package.sheet == sheet)
9530 .flat_map(|package| package.complete_selections.iter().copied())
9531 .collect();
9532 let residual = source
9533 .residual_sources(&selected, &complete, &self.workbook_load_limits)
9534 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
9535 residual_owners.insert(
9536 sheet.clone(),
9537 residual
9538 .1
9539 .iter()
9540 .map(|family| family.source_id)
9541 .collect::<BTreeSet<_>>(),
9542 );
9543 residual_sources.insert(sheet.clone(), residual);
9544 }
9545
9546 for (sheet, points) in &selected_package_points {
9549 let package = self
9550 .staged_formulas
9551 .get_mut(sheet)
9552 .unwrap()
9553 .deferred_package
9554 .as_mut()
9555 .unwrap();
9556 let owner_count = prepared_packages
9557 .iter()
9558 .filter(|p| &p.sheet == sheet)
9559 .flat_map(|p| &p.replay_records)
9560 .filter_map(|record| record.partition_owner.or(record.family))
9561 .filter(|owner| residual_owners[sheet].contains(owner))
9562 .count();
9563 package
9564 .consumed_members
9565 .try_reserve(owner_count)
9566 .map_err(|_| {
9567 resource(
9568 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9569 0,
9570 owner_count as u64 * 24,
9571 )
9572 })?;
9573 package.suppressed.try_reserve(points.len()).map_err(|_| {
9574 resource(
9575 formualizer_common::ResourceExhaustionReason::ScratchMemory,
9576 0,
9577 points.len() as u64 * 16,
9578 )
9579 })?;
9580 }
9581
9582 let estimated_commit_duration = std::time::Duration::from_nanos(
9583 (new_vertices as u64)
9584 .saturating_add(new_edges as u64)
9585 .saturating_add(prepared.len() as u64)
9586 .saturating_add(selected_package_records)
9587 .max(1)
9588 .saturating_mul(100),
9589 );
9590 if options.deadline.is_some_and(|deadline| {
9591 std::time::Instant::now()
9592 .checked_add(estimated_commit_duration)
9593 .is_none_or(|finish| finish > deadline)
9594 }) {
9595 self.observe_target_admission_failure(
9596 formualizer_common::ResourceExhaustionReason::Deadline,
9597 );
9598 return Err(resource(
9599 formualizer_common::ResourceExhaustionReason::Deadline,
9600 0,
9601 1,
9602 ));
9603 }
9604 #[cfg(test)]
9605 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
9606 hook();
9607 }
9608 self.target_preparation_checkpoint(options.deadline, 0)?;
9609 #[cfg(test)]
9610 self.target_preparation_fault(
9611 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
9612 )?;
9613 let current_revisions = self.preparation_revisions();
9614 let staged_leases_match = prepared.iter().all(|formula| {
9615 self.staged_formula_index
9616 .lease_matches(&formula.sheet, formula.lease)
9617 }) && prepared_packages.iter().all(|package| {
9618 self.staged_formula_index
9619 .package_lease_matches(&package.sheet, package.lease)
9620 });
9621 let stale_reason = Self::preparation_revision_stale_reason(
9622 &assumptions,
9623 ¤t_revisions,
9624 &planning_requests,
9625 staged_leases_match,
9626 );
9627 if let Some(reason) = stale_reason {
9628 return Err(Self::preparation_stale(
9629 reason,
9630 "target graph preparation plan is stale",
9631 ));
9632 }
9633 #[cfg(test)]
9634 self.target_preparation_fault(
9635 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
9636 )?;
9637 self.graph
9638 .validate_prepared_legacy_graph_plan(&legacy_graph)
9639 .map_err(|error| {
9640 Self::preparation_stale(
9641 formualizer_common::PreparationStaleReason::Graph,
9642 format!("target graph preparation plan is stale: {error}"),
9643 )
9644 })?;
9645 #[cfg(test)]
9646 self.target_preparation_fault(
9647 crate::engine::target_preparation::TargetPreparationFault::Reservation,
9648 )?;
9649 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
9650 self.formula_parse_diagnostics
9651 .try_reserve(pending_diagnostics.len())
9652 .map_err(|_| {
9653 resource(
9654 formualizer_common::ResourceExhaustionReason::Admission,
9655 pending_diagnostics.len() as u64,
9656 pending_diagnostics.len() as u64,
9657 )
9658 })?;
9659 self.target_preparation_checkpoint(options.deadline, 0)?;
9660 #[cfg(test)]
9661 self.target_preparation_fault(
9662 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
9663 )?;
9664
9665 let commit_started = crate::instant::FzInstant::now();
9666 let committed = self
9667 .graph
9668 .apply_prevalidated_legacy_graph_plan(legacy_graph);
9669 for formula in &prepared {
9670 let removed = self
9671 .staged_formulas
9672 .get_mut(&formula.sheet)
9673 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
9674 debug_assert!(removed.is_some());
9675 let index_removed = self.staged_formula_index.remove(
9676 &formula.sheet,
9677 formula.lease.row,
9678 formula.lease.col,
9679 );
9680 debug_assert!(index_removed);
9681 }
9682 for package in &prepared_packages {
9683 if let Some(points) = &package.selected_points {
9684 let staged = self.staged_formulas.get_mut(&package.sheet).unwrap();
9685 let source = staged.deferred_package.as_mut().unwrap();
9686 source.consumed_engine = Some(Arc::clone(&self.source_formula_token));
9687 source
9690 .consumed_members
9691 .extend(package.replay_records.iter().filter_map(|record| {
9692 record
9693 .partition_owner
9694 .or(record.family)
9695 .filter(|owner| residual_owners[&package.sheet].contains(owner))
9696 .map(|owner| {
9697 (
9698 owner,
9699 crate::engine::SourceCoord {
9700 row: record.row - 1,
9701 col: record.col - 1,
9702 },
9703 )
9704 })
9705 }));
9706 source.suppressed.extend(points.iter().copied());
9707 source.source_accounted = true;
9708 self.staged_formula_index
9709 .consume_package_points(&package.sheet, points);
9710 if source.suppressed.len() >= source.source_coordinates.len() {
9711 staged.deferred_package = None;
9712 self.staged_formula_index.set_package(&package.sheet, None);
9713 }
9714 } else {
9715 let removed = self
9716 .staged_formulas
9717 .get_mut(&package.sheet)
9718 .and_then(|staged| staged.deferred_package.take());
9719 debug_assert!(removed.is_some());
9720 self.staged_formula_index.set_package(&package.sheet, None);
9721 }
9722 }
9723 for (sheet, (families, partitions)) in residual_sources {
9724 if let Some(source) = self
9725 .staged_formulas
9726 .get_mut(&sheet)
9727 .and_then(|staged| staged.deferred_package.as_mut())
9728 {
9729 source.families = families;
9730 source.partitioned_families = partitions;
9731 source
9732 .consumed_members
9733 .retain(|(owner, _)| residual_owners[&sheet].contains(owner));
9734 self.staged_formula_index.update_package_family_count(
9735 &sheet,
9736 source.families.len() + source.partitioned_families.len(),
9737 );
9738 }
9739 }
9740 let empty_sheets = self
9741 .staged_formulas
9742 .iter()
9743 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
9744 .collect::<Vec<_>>();
9745 for sheet in empty_sheets {
9746 self.staged_formulas.remove(&sheet);
9747 }
9748 if committed > 0 {
9749 self.mark_topology_edited();
9750 }
9751 self.formula_parse_diagnostics.extend(pending_diagnostics);
9752 if !prepared.is_empty() || !prepared_packages.is_empty() {
9753 let mut ingest_delta = FormulaIngestReport::with_mode(FormulaPlaneMode::Off);
9754 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
9755 prepared_packages
9756 .iter()
9757 .map(|package| {
9758 (package.replay_records.len() as u64).saturating_add(
9759 if package.selected_points.is_some() {
9760 package.direct_cells
9761 } else {
9762 0
9763 },
9764 )
9765 })
9766 .sum::<u64>(),
9767 );
9768 ingest_delta.graph_formula_cells_materialized = committed as u64;
9769 ingest_delta.graph_vertices_created = new_vertices as u64;
9770 ingest_delta.graph_edges_created = new_edges as u64;
9771 for package in &prepared_packages {
9772 let source = &package.source_report;
9773 ingest_delta.source_formula_events = ingest_delta
9774 .source_formula_events
9775 .saturating_add(source.source_formula_events);
9776 ingest_delta.source_formula_records_spooled = ingest_delta
9777 .source_formula_records_spooled
9778 .saturating_add(source.source_formula_records_spooled);
9779 ingest_delta.source_spool_encoded_bytes = ingest_delta
9780 .source_spool_encoded_bytes
9781 .saturating_add(source.source_spool_encoded_bytes);
9782 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
9783 .source_spool_peak_memory_bytes
9784 .max(source.source_spool_peak_memory_bytes);
9785 ingest_delta.source_spool_spilled_bytes = ingest_delta
9786 .source_spool_spilled_bytes
9787 .saturating_add(source.source_spool_spilled_bytes);
9788 ingest_delta.source_spool_spill_files = ingest_delta
9789 .source_spool_spill_files
9790 .saturating_add(source.source_spool_spill_files);
9791 ingest_delta.source_spool_replays = ingest_delta
9792 .source_spool_replays
9793 .saturating_add(source.source_spool_replays)
9794 .saturating_add(package.spool_replays);
9795 ingest_delta.source_families_seen = ingest_delta
9796 .source_families_seen
9797 .saturating_add(source.families_seen);
9798 ingest_delta.source_family_cells_seen = ingest_delta
9799 .source_family_cells_seen
9800 .saturating_add(source.family_cells_seen);
9801 ingest_delta.source_family_shadow_eligible = ingest_delta
9802 .source_family_shadow_eligible
9803 .saturating_add(source.source_clean_families);
9804 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
9805 .source_family_shadow_eligible_cells
9806 .saturating_add(source.source_clean_cells);
9807 ingest_delta.source_partitioned_families_seen = ingest_delta
9808 .source_partitioned_families_seen
9809 .saturating_add(source.source_fragmentable_families);
9810 ingest_delta.source_partition_holes = ingest_delta
9811 .source_partition_holes
9812 .saturating_add(source.source_hole_exclusions);
9813 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
9814 .source_partition_ordinary_exceptions
9815 .saturating_add(source.source_ordinary_exclusions);
9816 ingest_delta.source_partition_surviving_cells = ingest_delta
9817 .source_partition_surviving_cells
9818 .saturating_add(source.source_fragmentable_cells);
9819 for (reason, count) in &source.fallback_reasons {
9820 let total = ingest_delta
9821 .fallback_reasons
9822 .entry(reason.clone())
9823 .or_default();
9824 *total = total.saturating_add(*count);
9825 }
9826
9827 ingest_delta.source_family_fallback = ingest_delta
9828 .source_family_fallback
9829 .saturating_add(source.families_seen);
9830 ingest_delta.source_family_fallback_cells = ingest_delta
9831 .source_family_fallback_cells
9832 .saturating_add(source.family_cells_seen);
9833 }
9834 self.record_formula_ingest_report(ingest_delta);
9835 }
9836 let commit_window = commit_started.elapsed();
9837 let retained_cells = self
9838 .staged_formula_index
9839 .all_leases()
9840 .into_iter()
9841 .filter_map(|(sheet, lease)| {
9842 formualizer_common::RangeAddress::new(
9843 sheet, lease.row, lease.col, lease.row, lease.col,
9844 )
9845 .ok()
9846 })
9847 .collect::<Vec<_>>();
9848 let observed_scratch_bytes = self
9849 .active_resource_ledger
9850 .as_ref()
9851 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
9852 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
9853 let committed_spans = 0u64;
9854 let selected_source_families = prepared_packages
9855 .iter()
9856 .filter(|package| package.selected_points.is_none())
9857 .map(|package| package.lease.family_count)
9858 .sum::<usize>()
9859 + prepared_packages
9860 .iter()
9861 .filter(|package| package.selected_points.is_some())
9862 .flat_map(|package| package.replay_records.iter())
9863 .filter_map(|record| record.partition_owner.or(record.family))
9864 .chain(
9865 prepared_packages
9866 .iter()
9867 .flat_map(|package| package.complete_selections.iter().copied()),
9868 )
9869 .collect::<BTreeSet<_>>()
9870 .len();
9871 let selected_staged_cells = prepared.len().saturating_add(
9872 prepared_packages
9873 .iter()
9874 .map(|package| {
9875 package
9876 .selected_points
9877 .as_ref()
9878 .map_or(package.replay_records.len(), BTreeSet::len)
9879 })
9880 .sum::<usize>(),
9881 );
9882 let actual_commit_work = (new_vertices as u64)
9883 .saturating_add(new_edges as u64)
9884 .saturating_add(committed as u64)
9885 .saturating_add(committed_spans)
9886 .saturating_add(prepared.len() as u64)
9887 .saturating_add(
9888 prepared_packages
9889 .iter()
9890 .map(|package| {
9891 package
9892 .selected_points
9893 .as_ref()
9894 .map_or(1, |points| points.len() as u64)
9895 })
9896 .sum::<u64>(),
9897 );
9898 let report = PreparedTargetGraphReport {
9899 request_id: request_id.unwrap_or_default(),
9900 requested_targets: targets.len(),
9901 normalized_regions: visited_regions.len(),
9902 normalized_target_list: normalized,
9903 selected_staged_cells,
9904 selected_source_families,
9905 retained_staged_cells: self.staged_formula_count(),
9906 selected_cells,
9907 retained_cells,
9908 widened_scope: scope,
9909 widening_reasons: reasons,
9910 revisions: assumptions,
9911 commit_window,
9912 estimated_scratch_bytes: scratch_bytes,
9913 observed_scratch_bytes,
9914 estimated_commit_work: (new_vertices as u64)
9915 .saturating_add(new_edges as u64)
9916 .saturating_add(prepared.len() as u64)
9917 .saturating_add(selected_package_records),
9918 actual_commit_work,
9919 outcome: PreparationOutcome::Prepared,
9920 };
9921 self.observe_target_preparation_report(&report);
9922 Ok(report)
9923 }
9924
9925 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
9927 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
9928 engine.build_graph_all_unobserved()
9929 })
9930 }
9931
9932 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
9933 let selected = self.staged_formula_count();
9934 let started = crate::instant::FzInstant::now();
9935 self.resource_checkpoint(selected as u64)?;
9936 let scratch_bytes = (selected as u64).saturating_mul(256);
9937 let result = self.with_request_scratch(scratch_bytes, |engine| {
9938 let index_snapshot = engine.staged_formula_index.clone();
9939 let collected = std::mem::take(&mut engine.staged_formulas)
9940 .into_iter()
9941 .collect();
9942 engine.staged_formula_index.clear_all();
9943 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
9944 });
9945 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
9946 result
9947 }
9948
9949 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
9951 &mut self,
9952 sheets: I,
9953 ) -> Result<(), formualizer_parse::ExcelError> {
9954 let mut sheets = sheets.into_iter();
9955 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
9956 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
9957 engine.with_request_scratch(name_scratch, |engine| {
9958 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
9961 engine.charge_bounded_work(names.len() as u64)?;
9962 engine.build_graph_for_sheet_names_unobserved(names)
9963 })
9964 })
9965 }
9966
9967 fn build_graph_for_sheet_names_unobserved(
9968 &mut self,
9969 sheets: Vec<String>,
9970 ) -> Result<(), formualizer_parse::ExcelError> {
9971 let started = crate::instant::FzInstant::now();
9972 let selected = sheets
9973 .iter()
9974 .filter_map(|sheet| self.staged_formulas.get(sheet))
9975 .map(StagedSheet::len)
9976 .sum::<usize>();
9977 self.resource_checkpoint(selected as u64)?;
9978 let scratch_bytes = (selected as u64).saturating_mul(256);
9979 self.reserve_request_scratch(scratch_bytes)?;
9980 let index_snapshot = self.staged_formula_index.clone();
9981 let mut collected = Vec::new();
9982 for sheet in sheets {
9983 if let Some(staged) = self.staged_formulas.remove(&sheet) {
9984 self.index_removed_staged_sheet(&sheet, &staged);
9985 collected.push((sheet, staged));
9986 }
9987 }
9988 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
9989 self.release_request_scratch(scratch_bytes);
9990 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
9991 result
9992 }
9993
9994 fn build_graph_from_staged_batches(
9995 &mut self,
9996 collected: StagedFormulaBatches,
9997 share_parse_cache_across_sheets: bool,
9998 staged_index_snapshot: StagedFormulaIndex,
9999 ) -> Result<(), formualizer_parse::ExcelError> {
10000 if collected.is_empty() {
10001 return Ok(());
10002 }
10003 for (sheet, _) in &collected {
10004 let _ = self.add_sheet(sheet);
10005 }
10006
10007 let diagnostics_len = self.formula_parse_diagnostics.len();
10008 let mut collected = collected;
10009 let prepared = match self
10010 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
10011 {
10012 Ok(prepared) => prepared,
10013 Err(error) => {
10014 self.formula_parse_diagnostics.truncate(diagnostics_len);
10015 for (sheet, staged) in collected {
10016 self.restore_staged_sheet(sheet, staged);
10017 }
10018 self.staged_formula_index = staged_index_snapshot;
10019 return Err(error);
10020 }
10021 };
10022 let (ordinary, compressed, direct, may_fail) = prepared;
10023
10024 let first_build = if self.deferred_build_can_be_first_load() {
10034 if let Err(error) = self.check_staged_formula_plans(
10035 ordinary
10036 .iter()
10037 .chain(compressed.iter().map(|(batch, _)| batch)),
10038 &may_fail,
10039 ) {
10040 self.formula_parse_diagnostics.truncate(diagnostics_len);
10041 for (sheet, staged) in collected {
10042 self.restore_staged_sheet(sheet, staged);
10043 }
10044 self.staged_formula_index = staged_index_snapshot;
10045 return Err(error);
10046 }
10047 self.deferred_build_as_first_load()
10048 } else {
10049 None
10050 };
10051 let built_sheets: Vec<String> = if first_build.is_some() {
10052 collected.iter().map(|(sheet, _)| sheet.clone()).collect()
10053 } else {
10054 Vec::new()
10055 };
10056
10057 let result = (|| {
10061 if !ordinary.is_empty() {
10062 self.ingest_formula_batches(ordinary)?;
10063 }
10064 if !compressed.is_empty() {
10065 self.ingest_compressed_formula_source_batches(compressed)?;
10066 }
10067 if !direct.is_empty() {
10068 self.finish_compressed_formula_sources(direct)?;
10069 }
10070 Ok(())
10071 })();
10072 if let Some(saved) = first_build {
10073 self.leave_deferred_first_load(saved, &built_sheets);
10074 }
10075 if let Err(error) = result {
10076 self.formula_parse_diagnostics.truncate(diagnostics_len);
10077 for (sheet, staged) in collected {
10078 self.restore_staged_sheet(sheet, staged);
10079 }
10080 self.staged_formula_index = staged_index_snapshot;
10081 return Err(error);
10082 }
10083 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
10084 Ok(())
10085 }
10086
10087 fn deferred_build_can_be_first_load(&self) -> bool {
10093 !self.graph.first_load_assume_new()
10094 && !self.graph_admission_enabled()
10095 && self.graph.formula_vertex_count() == 0
10096 }
10097
10098 fn check_staged_formula_plans<'b>(
10103 &mut self,
10104 batches: impl Iterator<Item = &'b FormulaIngestBatch>,
10105 may_fail: &FxHashSet<crate::engine::arena::AstNodeId>,
10106 ) -> Result<(), ExcelError> {
10107 if may_fail.is_empty() {
10108 return Ok(());
10109 }
10110 let mut seen: FxHashSet<(SheetId, crate::engine::arena::AstNodeId)> = FxHashSet::default();
10111 for batch in batches {
10112 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
10113 ExcelError::new(ExcelErrorKind::Ref)
10114 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
10115 })?;
10116 let mut pipeline = self.ingest_pipeline();
10117 for record in &batch.formulas {
10118 if record.member_anchor.is_some()
10119 || !may_fail.contains(&record.ast_id)
10120 || !seen.insert((sheet_id, record.ast_id))
10121 {
10122 continue;
10123 }
10124 let placement = CellRef::new(
10125 sheet_id,
10126 Coord::from_excel(record.row, record.col, true, true),
10127 );
10128 pipeline.ingest_formula(
10129 FormulaAstInput::RawArena(record.ast_id),
10130 placement,
10131 None,
10132 )?;
10133 }
10134 }
10135 Ok(())
10136 }
10137
10138 fn deferred_build_as_first_load(&mut self) -> Option<(crate::engine::SheetIndexMode, usize)> {
10139 if !self.deferred_build_can_be_first_load() {
10140 return None;
10141 }
10142 let saved = (
10143 self.graph.get_config().sheet_index_mode,
10144 self.config.range_expansion_limit,
10145 );
10146 self.graph
10147 .set_sheet_index_mode(crate::engine::SheetIndexMode::Lazy);
10148 self.config.range_expansion_limit = 0;
10149 self.graph.set_first_load_assume_new(true);
10150 self.graph.reset_ensure_touched();
10151 Some(saved)
10152 }
10153
10154 fn leave_deferred_first_load(
10157 &mut self,
10158 (index_mode, range_limit): (crate::engine::SheetIndexMode, usize),
10159 sheets: &[String],
10160 ) {
10161 self.graph.set_first_load_assume_new(false);
10162 self.graph.reset_ensure_touched();
10163 self.graph.set_sheet_index_mode(index_mode);
10164 self.config.range_expansion_limit = range_limit;
10165 for sheet in sheets {
10166 self.graph.finalize_sheet_index(sheet);
10167 }
10168 }
10169
10170 fn prepare_staged_formula_batches(
10171 &mut self,
10172 collected: &mut StagedFormulaBatches,
10173 share_parse_cache_across_sheets: bool,
10174 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
10175 let mut ordinary = Vec::new();
10176 let mut compressed = Vec::new();
10177 let mut direct = Vec::new();
10178 let mut may_fail = FxHashSet::default();
10179 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
10180 rustc_hash::FxHashMap::default();
10181 cache.reserve(4096);
10182
10183 for (sheet, staged) in collected {
10184 if !share_parse_cache_across_sheets {
10185 cache.clear();
10186 }
10187 let mut grouper = crate::engine::FormulaFamilyGrouper::new();
10191 let mut replayed_records = Vec::new();
10195 let deferred_source = None;
10196 let mut deferred_fallback = None;
10197 if let Some(package) = staged.deferred_package.as_mut() {
10198 if package.sheet_name != *sheet {
10199 return Err(ExcelError::new(ExcelErrorKind::Value)
10200 .with_message("deferred formula package sheet mismatch"));
10201 }
10202 let eligible: Vec<_> = package
10203 .families
10204 .iter()
10205 .filter(|family| !package.invalidated.contains(&family.source_id))
10206 .cloned()
10207 .collect();
10208 let eligible_partitions: Vec<_> = package
10209 .partitioned_families
10210 .iter()
10211 .filter(|family| !package.invalidated.contains(&family.source_id))
10212 .cloned()
10213 .collect();
10214
10215 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
10216 for partition in &eligible_partitions {
10217 replay_disposition
10218 .register_partition(partition, false)
10219 .map_err(|reason| {
10220 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
10221 })?;
10222 }
10223 replay_disposition
10224 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
10225 let replayed = package
10226 .replay
10227 .lock()
10228 .map_err(|_| {
10229 ExcelError::new(ExcelErrorKind::Value)
10230 .with_message("deferred formula spool lock poisoned")
10231 })?
10232 .replay(&replay_disposition)
10233 .map_err(|message| {
10234 ExcelError::new(ExcelErrorKind::Value).with_message(message)
10235 })?;
10236 replayed_records = replayed;
10237 let mut report = package.accounting_report();
10238 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
10239 deferred_fallback = Some((report, package.families.clone(), eligible_partitions));
10240 }
10241
10242 let n_entries = staged.entries.len() + replayed_records.len();
10243 let entries = staged
10244 .entries
10245 .iter()
10246 .cloned()
10247 .map(|(row, col, text)| (row, col, text, None))
10248 .chain(replayed_records.into_iter().map(|record| {
10249 (
10250 record.row,
10251 record.col,
10252 record.text,
10253 Some((record.source_order, record.family, record.partition_owner)),
10254 )
10255 }));
10256 let mut formulas = Vec::with_capacity(n_entries);
10257 let staged_order_base = u64::MAX.saturating_sub(n_entries as u64);
10258 for (entry_index, (row, col, txt, source_proof)) in entries.enumerate() {
10259 let key = if txt.starts_with('=') {
10260 txt
10261 } else {
10262 format!("={txt}")
10263 };
10264 let staged_record = if let Some(cached) = cache.get(&key) {
10265 cached.map(|ast_id| {
10266 self.note_staged_formula(&mut grouper, row, col, ast_id);
10267 FormulaIngestRecord::new(row, col, ast_id, Some(Arc::<str>::from(key)))
10268 })
10269 } else {
10270 let parsed = match formualizer_parse::parser::parse(&key) {
10271 Ok(parsed) => Some(parsed),
10272 Err(error) => self.handle_formula_parse_error(
10273 sheet,
10274 row,
10275 col,
10276 &key,
10277 error.to_string(),
10278 )?,
10279 };
10280 match parsed {
10281 Some(ast) => {
10282 let record = self.stage_formula_ast(&mut grouper, row, col, &ast, None);
10283 if !record.is_family_member() && self.formula_may_fail_planning(&ast) {
10284 may_fail.insert(record.ast_id);
10285 }
10286 if record.is_family_member() {
10289 Some(record)
10290 } else {
10291 let ast_id = record.ast_id;
10292 cache.insert(key.clone(), Some(ast_id));
10293 Some(FormulaIngestRecord::new(
10294 row,
10295 col,
10296 ast_id,
10297 Some(Arc::<str>::from(key)),
10298 ))
10299 }
10300 }
10301 None => {
10302 cache.insert(key, None);
10303 None
10304 }
10305 }
10306 };
10307
10308 if let Some(mut formula) = staged_record {
10309 if let Some((order, family, owner)) = source_proof {
10310 formula = formula.with_source_proof(order, family, owner);
10311 } else if deferred_source.is_some() {
10312 formula = formula.with_source_proof(
10313 crate::engine::SourceFormulaOrder::new(
10314 staged_order_base.saturating_add(entry_index as u64),
10315 ),
10316 None,
10317 None,
10318 );
10319 }
10320 formulas.push(formula);
10321 }
10322 }
10323
10324 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
10325 if let Some((report, preparation)) = deferred_source {
10326 direct.push((batch, report, preparation));
10327 } else if let Some((report, families, partitions)) = deferred_fallback {
10328 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
10329 batch.sheet_name.clone(),
10330 report,
10331 families,
10332 partitions,
10333 );
10334 compressed.push((batch, source_batch));
10335 } else if !batch.is_empty() {
10336 ordinary.push(batch);
10337 }
10338 }
10339 Ok((ordinary, compressed, direct, may_fail))
10340 }
10341
10342 fn formula_may_fail_planning(&self, ast: &formualizer_parse::parser::ASTNode) -> bool {
10347 use formualizer_parse::parser::{ASTNodeType, ReferenceType};
10348 let sheet_missing = |sheet: &Option<String>| {
10349 sheet
10350 .as_deref()
10351 .is_some_and(|s| self.graph.sheet_id(s).is_none())
10352 };
10353 match &ast.node_type {
10354 ASTNodeType::Literal(_) | ASTNodeType::Omitted => false,
10355 ASTNodeType::Reference { reference, .. } => match reference {
10356 ReferenceType::Cell { sheet, .. } => sheet_missing(sheet),
10357 ReferenceType::Range {
10358 sheet,
10359 start_row,
10360 start_col,
10361 end_row,
10362 end_col,
10363 ..
10364 } => {
10365 sheet_missing(sheet)
10366 || matches!((start_row, end_row), (Some(a), Some(b)) if a > b)
10367 || matches!((start_col, end_col), (Some(a), Some(b)) if a > b)
10368 }
10369 ReferenceType::NamedRange(_) => false,
10370 _ => true,
10371 },
10372 ASTNodeType::UnaryOp { expr, .. } => self.formula_may_fail_planning(expr),
10373 ASTNodeType::BinaryOp { left, right, .. } => {
10374 self.formula_may_fail_planning(left) || self.formula_may_fail_planning(right)
10375 }
10376 ASTNodeType::Function { args, .. } => {
10377 args.iter().any(|a| self.formula_may_fail_planning(a))
10378 }
10379 ASTNodeType::Call { callee, args } => {
10380 self.formula_may_fail_planning(callee)
10381 || args.iter().any(|a| self.formula_may_fail_planning(a))
10382 }
10383 ASTNodeType::Array(rows) => rows
10384 .iter()
10385 .flatten()
10386 .any(|a| self.formula_may_fail_planning(a)),
10387 }
10388 }
10389
10390 pub fn begin_bulk_ingest_arrow(
10392 &mut self,
10393 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
10394 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
10395 }
10396
10397 pub fn begin_bulk_update_arrow(
10399 &mut self,
10400 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
10401 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
10402 }
10403
10404 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
10405 self.graph.sheet_id(sheet).ok_or(
10406 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
10407 name: sheet.to_string(),
10408 reason: "Unknown sheet".to_string(),
10409 },
10410 )
10411 }
10412
10413 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
10414 if row_1based == 0 {
10415 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10416 }
10417 Ok(row_1based - 1)
10418 }
10419
10420 fn normalize_row_range_1based(
10421 start_row_1based: u32,
10422 end_row_1based: u32,
10423 ) -> Result<(u32, u32), crate::engine::EditorError> {
10424 if start_row_1based == 0 || end_row_1based == 0 {
10425 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
10426 }
10427 if start_row_1based > end_row_1based {
10428 return Err(crate::engine::EditorError::TransactionFailed {
10429 reason: "Row range start is greater than end".to_string(),
10430 });
10431 }
10432 Ok((start_row_1based - 1, end_row_1based - 1))
10433 }
10434
10435 fn invalidate_row_visibility_mask_cache(&self) {
10436 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10437 cache.clear();
10438 }
10439 }
10440
10441 fn row_visibility_changed(&mut self) {
10444 self.invalidate_row_visibility_mask_cache();
10445 self.graph.mark_row_visibility_readers_dirty();
10446 }
10447
10448 fn set_row_hidden_by_sheet_id(
10449 &mut self,
10450 sheet_id: SheetId,
10451 row0: u32,
10452 hidden: bool,
10453 source: RowVisibilitySource,
10454 ) -> bool {
10455 let changed = {
10456 let state = self.row_visibility.entry(sheet_id).or_default();
10457 state.set_row_hidden(row0, hidden, source)
10458 };
10459
10460 let remove_entry = self
10461 .row_visibility
10462 .get(&sheet_id)
10463 .map(|state| state.is_empty())
10464 .unwrap_or(false);
10465 if remove_entry {
10466 self.row_visibility.remove(&sheet_id);
10467 }
10468
10469 if changed {
10470 self.row_visibility_changed();
10471 }
10472
10473 changed
10474 }
10475
10476 fn set_rows_hidden_by_sheet_id(
10477 &mut self,
10478 sheet_id: SheetId,
10479 start_row0: u32,
10480 end_row0: u32,
10481 hidden: bool,
10482 source: RowVisibilitySource,
10483 ) -> bool {
10484 let changed = {
10485 let state = self.row_visibility.entry(sheet_id).or_default();
10486 state.set_rows_hidden(start_row0, end_row0, hidden, source)
10487 };
10488
10489 let remove_entry = self
10490 .row_visibility
10491 .get(&sheet_id)
10492 .map(|state| state.is_empty())
10493 .unwrap_or(false);
10494 if remove_entry {
10495 self.row_visibility.remove(&sheet_id);
10496 }
10497
10498 if changed {
10499 self.row_visibility_changed();
10500 }
10501
10502 changed
10503 }
10504
10505 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
10506 if count == 0 {
10507 return;
10508 }
10509 let mut changed = false;
10510 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10511 changed = state.insert_rows(before0, count);
10512 state.is_empty()
10513 } else {
10514 false
10515 };
10516 if remove_entry {
10517 self.row_visibility.remove(&sheet_id);
10518 }
10519 if changed {
10520 self.row_visibility_changed();
10521 }
10522 }
10523
10524 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
10525 if count == 0 {
10526 return;
10527 }
10528 let mut changed = false;
10529 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
10530 changed = state.delete_rows(start0, count);
10531 state.is_empty()
10532 } else {
10533 false
10534 };
10535 if remove_entry {
10536 self.row_visibility.remove(&sheet_id);
10537 }
10538 if changed {
10539 self.row_visibility_changed();
10540 }
10541 }
10542
10543 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10544 if let crate::engine::ChangeEvent::SetRowVisibility {
10545 sheet_id,
10546 row0,
10547 source,
10548 old_hidden,
10549 ..
10550 } = event
10551 {
10552 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
10553 }
10554 }
10555
10556 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
10557 if let crate::engine::ChangeEvent::SetRowVisibility {
10558 sheet_id,
10559 row0,
10560 source,
10561 new_hidden,
10562 ..
10563 } = event
10564 {
10565 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
10566 }
10567 }
10568
10569 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10570 for event in events.iter().rev() {
10571 self.apply_inverse_row_visibility_event(event);
10572 }
10573 }
10574
10575 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
10576 for event in events {
10577 self.apply_forward_row_visibility_event(event);
10578 }
10579 }
10580
10581 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10582 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10583 sheet,
10584 row,
10585 col,
10586 old,
10587 ..
10588 } = event
10589 {
10590 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
10591 }
10592 }
10593
10594 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
10595 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
10596 sheet,
10597 row,
10598 col,
10599 new,
10600 ..
10601 } = event
10602 {
10603 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
10604 }
10605 }
10606
10607 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
10610 match target {
10611 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
10612 None => {
10613 self.clear_staged_formula_text(sheet, row, col);
10614 }
10615 }
10616 }
10617
10618 pub fn set_row_hidden(
10619 &mut self,
10620 sheet: &str,
10621 row_1based: u32,
10622 hidden: bool,
10623 source: RowVisibilitySource,
10624 ) -> Result<(), crate::engine::EditorError> {
10625 self.observe_function_semantic_epoch()
10626 .map_err(crate::engine::EditorError::Excel)?;
10627 self.observe_function_semantic_epoch()
10628 .map_err(crate::engine::EditorError::Excel)?;
10629 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10630 let row0 = Self::normalize_row_1based(row_1based)?;
10631 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
10632 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10633 sheet_id, row0,
10634 )));
10635 self.mark_data_edited();
10636 }
10637 Ok(())
10638 }
10639
10640 pub fn set_rows_hidden(
10641 &mut self,
10642 sheet: &str,
10643 start_row_1based: u32,
10644 end_row_1based: u32,
10645 hidden: bool,
10646 source: RowVisibilitySource,
10647 ) -> Result<(), crate::engine::EditorError> {
10648 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10649 let (start_row0, end_row0) =
10650 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
10651 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
10652 if start_row0 == end_row0 {
10653 self.record_structural_change(StructuralScope::Region(Region::whole_row(
10654 sheet_id, start_row0,
10655 )));
10656 } else {
10657 self.record_structural_change(StructuralScope::Sheet(sheet_id));
10658 }
10659 self.mark_data_edited();
10660 }
10661 Ok(())
10662 }
10663
10664 pub fn is_row_hidden(
10665 &self,
10666 sheet: &str,
10667 row_1based: u32,
10668 source: Option<RowVisibilitySource>,
10669 ) -> Option<bool> {
10670 let sheet_id = self.graph.sheet_id(sheet)?;
10671 let row0 = row_1based.checked_sub(1)?;
10672 Some(
10673 self.row_visibility
10674 .get(&sheet_id)
10675 .map(|state| state.is_row_hidden(row0, source))
10676 .unwrap_or(false),
10677 )
10678 }
10679
10680 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
10681 let sheet_id = self.graph.sheet_id(sheet)?;
10682 Some(
10683 self.row_visibility
10684 .get(&sheet_id)
10685 .map(|state| state.version())
10686 .unwrap_or(0),
10687 )
10688 }
10689
10690 fn nested_subtotal_cells_for_view(
10696 &self,
10697 view: &RangeView<'_>,
10698 include_aggregate: bool,
10699 ) -> Option<Vec<(usize, usize, usize)>> {
10700 if !view.is_sheet_backed() {
10701 return None;
10702 }
10703 let (rows, cols) = view.dims();
10704 if rows == 0 || cols == 0 {
10705 return Some(Vec::new());
10706 }
10707 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10708 let (r0, c0) = (view.start_row(), view.start_col());
10709 let mask = if include_aggregate {
10710 crate::engine::arena::SUBTOTAL_CALL | crate::engine::arena::AGGREGATE_CALL
10711 } else {
10712 crate::engine::arena::SUBTOTAL_CALL
10713 };
10714 let ds = self.graph.data_store();
10715 let mut found: Vec<(usize, usize, usize)> = self
10716 .graph
10717 .formula_intervals_in_region(
10718 sheet_id,
10719 r0 as u32,
10720 (r0 + rows - 1) as u32,
10721 c0 as u32,
10722 (c0 + cols - 1) as u32,
10723 |template| ds.ast_subtotal_calls(template) & mask != 0,
10724 )
10725 .into_iter()
10726 .map(|(col, lo, hi)| (col as usize - c0, lo as usize - r0, hi as usize - r0))
10727 .collect();
10728 found.sort_unstable();
10729 let mut out: Vec<(usize, usize, usize)> = Vec::with_capacity(found.len());
10732 for (col, lo, hi) in found {
10733 match out.last_mut() {
10734 Some(last) if last.0 == col && lo <= last.2 + 1 => last.2 = last.2.max(hi),
10735 _ => out.push((col, lo, hi)),
10736 }
10737 }
10738 Some(out)
10739 }
10740
10741 fn build_row_visibility_mask_for_view(
10742 &self,
10743 view: &RangeView<'_>,
10744 mode: VisibilityMaskMode,
10745 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
10746 let sheet_rows = view.sheet().nrows as usize;
10747 if sheet_rows == 0 || view.start_row() >= sheet_rows {
10748 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
10749 }
10750
10751 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
10752 let start_row0 = view.start_row() as u32;
10753 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
10754 let version = self
10755 .row_visibility
10756 .get(&sheet_id)
10757 .map(|state| state.version())
10758 .unwrap_or(0);
10759 let key = VisibilityMaskCacheKey {
10760 sheet_id,
10761 start_row0,
10762 end_row0,
10763 mode,
10764 version,
10765 };
10766
10767 if let Ok(cache) = self.row_visibility_mask_cache.read()
10768 && let Some(mask) = cache.get(&key)
10769 {
10770 #[cfg(test)]
10771 visibility_mask_test_hooks::inc_hit();
10772 return Some(mask.clone());
10773 }
10774
10775 #[cfg(test)]
10776 visibility_mask_test_hooks::inc_miss();
10777
10778 let state = self.row_visibility.get(&sheet_id);
10779 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
10780 for row0 in start_row0..=end_row0 {
10781 let manual_hidden = state
10782 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
10783 .unwrap_or(false);
10784 let filter_hidden = state
10785 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
10786 .unwrap_or(false);
10787
10788 let include = match mode {
10789 VisibilityMaskMode::IncludeAll => true,
10790 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
10791 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
10792 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
10793 !(manual_hidden || filter_hidden)
10794 }
10795 };
10796 out.push(include);
10797 }
10798
10799 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
10800 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
10801 const MAX_CACHE_ENTRIES: usize = 4096;
10802 if cache.len() >= MAX_CACHE_ENTRIES {
10803 cache.clear();
10804 #[cfg(test)]
10805 visibility_mask_test_hooks::inc_eviction();
10806 }
10807 cache.insert(key, mask.clone());
10808 }
10809
10810 Some(mask)
10811 }
10812
10813 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
10814 let changes =
10815 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
10816 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
10817 let provider_revision = self.resolver.planning_semantic_revision();
10818 let provider_changed = provider_revision != self.function_provider_revision_seen;
10819 if !global_changed && !provider_changed {
10820 return Ok(false);
10821 }
10822
10823 let changed = !changes.keys.is_empty();
10824 if global_changed && changed || provider_changed {
10825 self.cached_static_schedule = None;
10826 self.recent_schedules.clear();
10827 self.base_schedule = None;
10828 }
10829 self.function_semantic_epoch_seen = changes.epoch;
10830 self.function_provider_revision_seen = provider_revision;
10831 Ok(false)
10832 }
10833
10834 pub(crate) fn ast_uses_changed_function(
10835 ast: &ASTNode,
10836 changed: &BTreeSet<(String, String)>,
10837 ) -> bool {
10838 match &ast.node_type {
10839 ASTNodeType::Function { name, args } => {
10840 let normalized = name.to_uppercase();
10841 let mut spellings = vec![(String::new(), normalized.clone())];
10842 let mut stripped = normalized.as_str();
10843 while let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
10844 .iter()
10845 .find_map(|prefix| stripped.strip_prefix(prefix))
10846 {
10847 stripped = rest;
10848 spellings.push((String::new(), stripped.to_string()));
10849 }
10850 let resolved = crate::function_registry::resolve("", name);
10851 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
10852 || resolved.as_ref().is_some_and(|resolved| {
10853 changed.contains(&(
10854 resolved.namespace.clone(),
10855 resolved.canonical_name.clone(),
10856 ))
10857 });
10858 directly_changed
10859 || resolved.is_none()
10860 || args
10861 .iter()
10862 .any(|arg| Self::ast_uses_changed_function(arg, changed))
10863 }
10864 ASTNodeType::Call { callee, args } => {
10865 Self::ast_uses_changed_function(callee, changed)
10866 || args
10867 .iter()
10868 .any(|arg| Self::ast_uses_changed_function(arg, changed))
10869 }
10870 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
10871 ASTNodeType::BinaryOp { left, right, .. } => {
10872 Self::ast_uses_changed_function(left, changed)
10873 || Self::ast_uses_changed_function(right, changed)
10874 }
10875 ASTNodeType::Array(rows) => rows
10876 .iter()
10877 .flatten()
10878 .any(|node| Self::ast_uses_changed_function(node, changed)),
10879 _ => false,
10880 }
10881 }
10882
10883 fn ast_contains_function(ast: &ASTNode) -> bool {
10884 match &ast.node_type {
10885 ASTNodeType::Function { .. } => true,
10886 ASTNodeType::Call { callee, args } => {
10887 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
10888 }
10889 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
10890 ASTNodeType::BinaryOp { left, right, .. } => {
10891 Self::ast_contains_function(left) || Self::ast_contains_function(right)
10892 }
10893 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
10894 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
10895 }
10896 }
10897
10898 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
10899 Region::rows_from(sheet_id, start_row0)
10900 }
10901
10902 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
10903 Region::cols_from(sheet_id, start_col0)
10904 }
10905
10906 #[cfg(test)]
10907 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
10908 self.force_non_cycle_schedule_fallback_for_test = true;
10909 }
10910
10911 fn materialize_deferred_sheet_before_structural_edit(
10912 &mut self,
10913 sheet: &str,
10914 ) -> Result<(), crate::engine::EditorError> {
10915 if self.staged_formulas.contains_key(sheet) {
10916 self.build_graph_for_sheets([sheet])?;
10917 }
10918 Ok(())
10919 }
10920
10921 fn structural_row_occupancy(
10922 &self,
10923 sheet: &str,
10924 sheet_id: SheetId,
10925 ) -> crate::engine::graph::StructuralOccupancy {
10926 if !self.graph.has_compressed_range_readers() {
10927 return crate::engine::graph::StructuralOccupancy::default();
10928 }
10929 let mut occupancy = self.graph.structural_occupancy(sheet_id);
10930 if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
10931 occupancy.include_arrow_sheet(arrow_sheet);
10932 occupancy
10933 } else {
10934 crate::engine::graph::StructuralOccupancy::conservative()
10936 }
10937 }
10938
10939 fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
10940 crate::engine::graph::StructuralOccupancy::conservative()
10944 }
10945
10946 pub fn insert_rows(
10948 &mut self,
10949 sheet: &str,
10950 before: u32,
10951 count: u32,
10952 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
10953 {
10954 if count == 0 {
10955 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
10956 }
10957 self.observe_function_semantic_epoch()
10958 .map_err(crate::engine::EditorError::Excel)?;
10959 use crate::engine::graph::editor::vertex_editor::VertexEditor;
10960 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
10961 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10962 let before0 = before.saturating_sub(1);
10963 let affected_region = Self::structural_row_region(sheet_id, before0);
10964 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
10965 let summary = {
10966 let mut editor =
10967 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
10968 editor.insert_rows(sheet_id, before0, count)?
10969 };
10970 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
10971 let before0 = before0 as usize;
10972 asheet.insert_rows(before0, count as usize);
10973 }
10974 self.purge_derived_formats_after_row(sheet_id, before0);
10975 self.mark_moved_formula_vertices_dirty(&summary);
10976 self.clear_computed_overlay_after_row(sheet, before0 as usize);
10977 self.shift_row_visibility_insert(sheet_id, before0, count);
10978 self.record_structural_change(StructuralScope::Region(affected_region));
10979 self.mark_topology_edited();
10980 Ok(summary)
10981 }
10982
10983 pub fn delete_rows(
10985 &mut self,
10986 sheet: &str,
10987 start: u32,
10988 count: u32,
10989 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
10990 {
10991 if count == 0 {
10992 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
10993 }
10994 self.observe_function_semantic_epoch()
10995 .map_err(crate::engine::EditorError::Excel)?;
10996 use crate::engine::graph::editor::vertex_editor::VertexEditor;
10997 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
10998 let sheet_id = self.ensure_known_sheet_id(sheet)?;
10999 let start0 = start.saturating_sub(1);
11000 let affected_region = Self::structural_row_region(sheet_id, start0);
11001 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
11002 let summary = {
11003 let mut editor =
11004 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11005 editor.delete_rows(sheet_id, start0, count)?
11006 };
11007 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11008 let start0 = start0 as usize;
11009 asheet.delete_rows(start0, count as usize);
11010 }
11011 self.purge_derived_formats_after_row(sheet_id, start0);
11012 self.mark_moved_formula_vertices_dirty(&summary);
11013 self.clear_computed_overlay_after_row(sheet, start0 as usize);
11014 self.shift_row_visibility_delete(sheet_id, start0, count);
11015 self.record_structural_change(StructuralScope::Region(affected_region));
11016 self.mark_topology_edited();
11017 Ok(summary)
11018 }
11019
11020 pub fn insert_columns(
11022 &mut self,
11023 sheet: &str,
11024 before: u32,
11025 count: u32,
11026 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11027 {
11028 if count == 0 {
11029 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11030 }
11031 self.observe_function_semantic_epoch()
11032 .map_err(crate::engine::EditorError::Excel)?;
11033 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11034 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11035 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11036 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11037 name: sheet.to_string(),
11038 reason: "Unknown sheet".to_string(),
11039 },
11040 )?;
11041 let before0 = before.saturating_sub(1);
11042 let affected_region = Self::structural_col_region(sheet_id, before0);
11043 let occupancy = self.structural_column_occupancy();
11044 let summary = {
11045 let mut editor =
11046 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11047 editor.insert_columns(sheet_id, before0, count)?
11048 };
11049 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11050 let before0 = before0 as usize;
11051 asheet.insert_columns(before0, count as usize);
11052 }
11053 self.purge_derived_formats_after_col(sheet_id, before0);
11054 self.mark_moved_formula_vertices_dirty(&summary);
11055 self.clear_computed_overlay_after_col(sheet, before0 as usize);
11056 self.record_structural_change(StructuralScope::Region(affected_region));
11057 self.mark_topology_edited();
11058 Ok(summary)
11059 }
11060
11061 pub fn delete_columns(
11063 &mut self,
11064 sheet: &str,
11065 start: u32,
11066 count: u32,
11067 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
11068 {
11069 if count == 0 {
11070 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
11071 }
11072 self.observe_function_semantic_epoch()
11073 .map_err(crate::engine::EditorError::Excel)?;
11074 use crate::engine::graph::editor::vertex_editor::VertexEditor;
11075 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
11076 let sheet_id = self.graph.sheet_id(sheet).ok_or(
11077 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11078 name: sheet.to_string(),
11079 reason: "Unknown sheet".to_string(),
11080 },
11081 )?;
11082 let start0 = start.saturating_sub(1);
11083 let affected_region = Self::structural_col_region(sheet_id, start0);
11084 let occupancy = self.structural_column_occupancy();
11085 let summary = {
11086 let mut editor =
11087 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
11088 editor.delete_columns(sheet_id, start0, count)?
11089 };
11090 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
11091 let start0 = start0 as usize;
11092 asheet.delete_columns(start0, count as usize);
11093 }
11094 self.purge_derived_formats_after_col(sheet_id, start0);
11095 self.mark_moved_formula_vertices_dirty(&summary);
11096 self.clear_computed_overlay_after_col(sheet, start0 as usize);
11097 self.record_structural_change(StructuralScope::Region(affected_region));
11098 self.mark_topology_edited();
11099 Ok(summary)
11100 }
11101 fn arrow_used_row_bounds(
11103 &self,
11104 sheet: &str,
11105 start_col: u32,
11106 end_col: u32,
11107 ) -> Option<(u32, u32)> {
11108 let a = self.sheet_store().sheet(sheet)?;
11109 if a.columns.is_empty() {
11110 return None;
11111 }
11112 let sc0 = start_col.saturating_sub(1) as usize;
11113 let ec0 = end_col.saturating_sub(1) as usize;
11114 let col_hi = a.columns.len().saturating_sub(1);
11115 if sc0 > col_hi {
11116 return None;
11117 }
11118 let ec0 = ec0.min(col_hi);
11119 let snap = self.data_snapshot_id();
11121 let mut min_r0: Option<usize> = None;
11122 for ci in sc0..=ec0 {
11123 let sheet_id = self.graph.sheet_id(sheet)?;
11124 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
11125 g.as_ref()
11126 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11127 }) {
11128 let mv = mv as usize;
11129 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
11130 continue;
11131 }
11132 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11134 if let Ok(mut g) = self.row_bounds_cache.write() {
11135 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11136 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
11137 }
11138 if let Some(m) = min_c {
11139 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
11140 }
11141 }
11142 min_r0?;
11143 let mut max_r0: Option<usize> = None;
11144 for ci in sc0..=ec0 {
11145 let sheet_id = self.graph.sheet_id(sheet)?;
11146 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
11147 g.as_ref()
11148 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
11149 }) {
11150 let mv = mv as usize;
11151 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
11152 continue;
11153 }
11154 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
11155 if let Ok(mut g) = self.row_bounds_cache.write() {
11156 g.get_or_insert_with(|| RowBoundsCache::new(snap))
11157 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
11158 }
11159 if let Some(m) = max_c {
11160 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
11161 }
11162 }
11163 match (min_r0, max_r0) {
11164 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11165 _ => None,
11166 }
11167 }
11168
11169 fn scan_column_used_bounds(
11170 a: &crate::arrow_store::ArrowSheet,
11171 ci: usize,
11172 ) -> (Option<u32>, Option<u32>) {
11173 let col = &a.columns[ci];
11174
11175 let mut min_r0: Option<u32> = None;
11177 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11178 let tags = chunk.type_tag.values();
11179 for (off, &t) in tags.iter().enumerate() {
11180 let overlay_non_empty = chunk
11181 .overlay
11182 .get(off)
11183 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11184 .unwrap_or(false)
11185 || chunk
11186 .computed_overlay
11187 .get(off)
11188 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11189 .unwrap_or(false);
11190 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11191 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11192 break;
11193 };
11194 let row0 = chunk_start + off;
11195 min_r0 = Some(row0 as u32);
11196 break;
11197 }
11198 }
11199 if min_r0.is_some() {
11200 break;
11201 }
11202 }
11203 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
11204 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11205 sparse_idxs.sort_unstable();
11206 for chunk_idx in sparse_idxs {
11207 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11208 continue;
11209 };
11210 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11211 continue;
11212 };
11213 let tags = chunk.type_tag.values();
11214 for (off, &t) in tags.iter().enumerate() {
11215 let overlay_non_empty = chunk
11216 .overlay
11217 .get(off)
11218 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11219 .unwrap_or(false)
11220 || chunk
11221 .computed_overlay
11222 .get(off)
11223 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11224 .unwrap_or(false);
11225 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11226 let row0 = chunk_start + off;
11227 min_r0 = Some(row0 as u32);
11228 break;
11229 }
11230 }
11231 if min_r0.is_some() {
11232 break;
11233 }
11234 }
11235 }
11236
11237 let mut max_r0: Option<u32> = None;
11239 if !col.sparse_chunks.is_empty() {
11240 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
11241 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
11242 for chunk_idx in sparse_idxs {
11243 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
11244 continue;
11245 };
11246 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11247 continue;
11248 };
11249 let tags = chunk.type_tag.values();
11250 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11251 let overlay_non_empty = chunk
11252 .overlay
11253 .get(rev_idx)
11254 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11255 .unwrap_or(false)
11256 || chunk
11257 .computed_overlay
11258 .get(rev_idx)
11259 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11260 .unwrap_or(false);
11261 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11262 let row0 = chunk_start + rev_idx;
11263 max_r0 = Some(row0 as u32);
11264 break;
11265 }
11266 }
11267 if max_r0.is_some() {
11268 break;
11269 }
11270 }
11271 }
11272 if max_r0.is_none() {
11273 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
11274 let tags = chunk.type_tag.values();
11275 for (rev_idx, &t) in tags.iter().enumerate().rev() {
11276 let overlay_non_empty = chunk
11277 .overlay
11278 .get(rev_idx)
11279 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11280 .unwrap_or(false)
11281 || chunk
11282 .computed_overlay
11283 .get(rev_idx)
11284 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11285 .unwrap_or(false);
11286 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
11287 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11288 break;
11289 };
11290 let row0 = chunk_start + rev_idx;
11291 max_r0 = Some(row0 as u32);
11292 break;
11293 }
11294 }
11295 if max_r0.is_some() {
11296 break;
11297 }
11298 }
11299 }
11300
11301 (min_r0, max_r0)
11302 }
11303
11304 fn arrow_used_col_bounds(
11306 &self,
11307 sheet: &str,
11308 start_row: u32,
11309 end_row: u32,
11310 ) -> Option<(u32, u32)> {
11311 let a = self.sheet_store().sheet(sheet)?;
11312 if a.columns.is_empty() {
11313 return None;
11314 }
11315 let sr0 = start_row.saturating_sub(1) as usize;
11316 let er0 = end_row.saturating_sub(1) as usize;
11317 if sr0 > er0 {
11318 return None;
11319 }
11320 let mut min_c0: Option<usize> = None;
11323 let mut max_c0: Option<usize> = None;
11324 for (ci, col) in a.columns.iter().enumerate() {
11326 let mut any_in_range = false;
11327
11328 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
11329 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
11330 return false;
11331 };
11332 let chunk_len = chunk.type_tag.len();
11333 if chunk_len == 0 {
11334 return false;
11335 }
11336 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
11337 if sr0 > chunk_end || er0 < chunk_start {
11339 return false;
11340 }
11341 let start_off = sr0.max(chunk_start) - chunk_start;
11342 let end_off = er0.min(chunk_end) - chunk_start;
11343 let tags = chunk.type_tag.values();
11344 for off in start_off..=end_off {
11345 let overlay_non_empty = chunk
11346 .overlay
11347 .get(off)
11348 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11349 .unwrap_or(false)
11350 || chunk
11351 .computed_overlay
11352 .get(off)
11353 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
11354 .unwrap_or(false);
11355 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
11356 return true;
11357 }
11358 }
11359 false
11360 };
11361
11362 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
11363 if scan_chunk(chunk_idx, chunk) {
11364 any_in_range = true;
11365 break;
11366 }
11367 }
11368
11369 if !any_in_range && !col.sparse_chunks.is_empty() {
11370 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
11371 if scan_chunk(chunk_idx, chunk) {
11372 any_in_range = true;
11373 break;
11374 }
11375 }
11376 }
11377
11378 if any_in_range {
11379 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
11380 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
11381 }
11382 }
11383 match (min_c0, max_c0) {
11384 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
11385 _ => None,
11386 }
11387 }
11388
11389 fn formula_row_bounds_for_columns(
11390 &self,
11391 sheet: &str,
11392 start_col: u32,
11393 end_col: u32,
11394 ) -> Option<(u32, u32)> {
11395 let sheet_id = self.graph.sheet_id(sheet)?;
11396 let sc0 = start_col.saturating_sub(1);
11397 let ec0 = end_col.saturating_sub(1);
11398 let mut min_r0: Option<u32> = None;
11399 let mut max_r0: Option<u32> = None;
11400
11401 if self.graph.sheet_index(sheet_id).is_some() {
11402 for vid in self.graph.vertices_in_cols(sheet_id, sc0, ec0) {
11403 if !matches!(
11404 self.graph.get_vertex_kind(vid),
11405 VertexKind::FormulaScalar | VertexKind::FormulaArray
11406 ) {
11407 continue;
11408 }
11409 let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
11410 continue;
11411 };
11412 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11413 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11414 }
11415 } else {
11416 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11417 if !matches!(
11418 self.graph.get_vertex_kind(vid),
11419 VertexKind::FormulaScalar | VertexKind::FormulaArray
11420 ) {
11421 continue;
11422 }
11423 let col0 = coord.col();
11424 if col0 < sc0 || col0 > ec0 {
11425 continue;
11426 }
11427 let row0 = coord.row();
11428 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
11429 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
11430 }
11431 }
11432
11433 match (min_r0, max_r0) {
11434 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11435 _ => None,
11436 }
11437 }
11438
11439 fn formula_col_bounds_for_rows(
11440 &self,
11441 sheet: &str,
11442 start_row: u32,
11443 end_row: u32,
11444 ) -> Option<(u32, u32)> {
11445 let sheet_id = self.graph.sheet_id(sheet)?;
11446 let sr0 = start_row.saturating_sub(1);
11447 let er0 = end_row.saturating_sub(1);
11448 let mut min_c0: Option<u32> = None;
11449 let mut max_c0: Option<u32> = None;
11450
11451 if self.graph.sheet_index(sheet_id).is_some() {
11452 for vid in self.graph.vertices_in_rows(sheet_id, sr0, er0) {
11453 if !matches!(
11454 self.graph.get_vertex_kind(vid),
11455 VertexKind::FormulaScalar | VertexKind::FormulaArray
11456 ) {
11457 continue;
11458 }
11459 let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
11460 continue;
11461 };
11462 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11463 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11464 }
11465 } else {
11466 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
11467 if !matches!(
11468 self.graph.get_vertex_kind(vid),
11469 VertexKind::FormulaScalar | VertexKind::FormulaArray
11470 ) {
11471 continue;
11472 }
11473 let row0 = coord.row();
11474 if row0 < sr0 || row0 > er0 {
11475 continue;
11476 }
11477 let col0 = coord.col();
11478 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
11479 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
11480 }
11481 }
11482
11483 match (min_c0, max_c0) {
11484 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
11485 _ => None,
11486 }
11487 }
11488
11489 fn union_used_bounds(
11490 first: Option<(u32, u32)>,
11491 second: Option<(u32, u32)>,
11492 ) -> Option<(u32, u32)> {
11493 match (first, second) {
11494 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
11495 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
11496 (None, None) => None,
11497 }
11498 }
11499
11500 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
11503 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11504 return;
11505 }
11506 if self.arrow_sheets.sheet(sheet).is_none() {
11507 self.arrow_sheets
11508 .sheets
11509 .push(crate::arrow_store::ArrowSheet {
11510 name: std::sync::Arc::<str>::from(sheet),
11511 date_system: self.config.date_system,
11512 columns: Vec::new(),
11513 nrows: 0,
11514 chunk_starts: Vec::new(),
11515 chunk_rows: 32 * 1024,
11516 });
11517 }
11518
11519 let row0 = row.saturating_sub(1) as usize;
11520 let col0 = col.saturating_sub(1) as usize;
11521
11522 let asheet = self
11523 .arrow_sheets
11524 .sheet_mut(sheet)
11525 .expect("ArrowSheet must exist");
11526
11527 let cur_cols = asheet.columns.len();
11528 if col0 >= cur_cols {
11529 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11530 }
11531
11532 if row0 >= asheet.nrows as usize {
11533 if asheet.columns.is_empty() {
11534 asheet.insert_columns(0, 1);
11535 }
11536 asheet.ensure_row_capacity(row0 + 1);
11537 }
11538 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
11539 let ov =
11540 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
11541 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
11542 let _ = ch.overlay.set(in_off, ov);
11543 let format = match value {
11544 LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
11545 LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
11546 LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
11547 LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
11548 _ => None,
11549 };
11550 ch.overlay.set_format(in_off, format);
11551 ch.computed_overlay.remove(in_off)
11556 } else {
11557 return;
11558 };
11559 let abs_threshold = 1024usize;
11561 let frac_den = 50usize;
11562 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
11563 if freed > 0 {
11564 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
11565 }
11566 self.adjust_computed_overlay_bytes(computed_delta);
11567 }
11568 }
11569
11570 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
11575 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11576 return;
11577 }
11578 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11579 return;
11580 };
11581 let row0 = row.saturating_sub(1) as usize;
11582 let col0 = col.saturating_sub(1) as usize;
11583 if row0 >= asheet.nrows as usize {
11584 return;
11585 }
11586 if col0 >= asheet.columns.len() {
11587 return;
11588 }
11589 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11590 return;
11591 };
11592 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
11593 let _ = ch.overlay.remove(in_off);
11594 }
11595 }
11596
11597 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
11598 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11599 return;
11600 }
11601
11602 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11603 return;
11604 };
11605 if start_row0 >= asheet.nrows as usize {
11606 return;
11607 }
11608
11609 let starts = asheet.chunk_starts.clone();
11610 let nrows = asheet.nrows as usize;
11611 let mut delta = 0isize;
11612 for col in &mut asheet.columns {
11613 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
11614 let Some(&chunk_start) = starts.get(chunk_idx) else {
11615 continue;
11616 };
11617 let chunk_end = starts
11618 .get(chunk_idx + 1)
11619 .copied()
11620 .unwrap_or(nrows)
11621 .min(chunk_start.saturating_add(ch.len()));
11622 if chunk_end <= start_row0 {
11623 continue;
11624 }
11625 if chunk_start >= start_row0 {
11626 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11627 } else {
11628 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11629 delta = delta
11630 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11631 }
11632 }
11633
11634 for (chunk_idx, ch) in &mut col.sparse_chunks {
11635 let Some(&chunk_start) = starts.get(*chunk_idx) else {
11636 continue;
11637 };
11638 let chunk_end = starts
11639 .get(*chunk_idx + 1)
11640 .copied()
11641 .unwrap_or(nrows)
11642 .min(chunk_start.saturating_add(ch.len()));
11643 if chunk_end <= start_row0 {
11644 continue;
11645 }
11646 if chunk_start >= start_row0 {
11647 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11648 } else {
11649 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
11650 delta = delta
11651 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
11652 }
11653 }
11654 }
11655 self.adjust_computed_overlay_bytes(delta);
11656 }
11657
11658 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
11659 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
11660 return;
11661 }
11662
11663 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
11664 return;
11665 };
11666 if start_col0 >= asheet.columns.len() {
11667 return;
11668 }
11669
11670 let mut delta = 0isize;
11671 for col in asheet.columns.iter_mut().skip(start_col0) {
11672 for ch in &mut col.chunks {
11673 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11674 }
11675 for ch in col.sparse_chunks.values_mut() {
11676 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
11677 }
11678 }
11679 self.adjust_computed_overlay_bytes(delta);
11680 }
11681
11682 #[inline]
11683 fn literal_to_overlay_value(
11684 value: &LiteralValue,
11685 date_system: crate::engine::DateSystem,
11686 ) -> crate::arrow_store::OverlayValue {
11687 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
11688 }
11689
11690 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
11691 self.arrow_sheets
11692 .sheet(sheet)
11693 .map(|sheet| sheet.date_system)
11694 .unwrap_or(self.config.date_system)
11695 }
11696
11697 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11700 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11701 return None;
11702 }
11703 let asheet = self.arrow_sheets.sheet(sheet)?;
11704 let row0 = row.saturating_sub(1) as usize;
11705 let col0 = col.saturating_sub(1) as usize;
11706 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11707 return None;
11708 }
11709 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11710 let ch = asheet.columns[col0].chunk(ch_idx)?;
11711 ch.overlay
11712 .get_scalar(in_off)
11713 .map(|ov| ov.to_literal_for(asheet.date_system))
11714 }
11715
11716 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
11719 if !(self.config.arrow_storage_enabled
11720 && self.config.delta_overlay_enabled
11721 && self.config.write_formula_overlay_enabled)
11722 {
11723 return None;
11724 }
11725 let asheet = self.arrow_sheets.sheet(sheet)?;
11726 let row0 = row.saturating_sub(1) as usize;
11727 let col0 = col.saturating_sub(1) as usize;
11728 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
11729 return None;
11730 }
11731 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
11732 let ch = asheet.columns[col0].chunk(ch_idx)?;
11733 ch.computed_overlay
11734 .get_scalar(in_off)
11735 .map(|ov| ov.to_literal_for(asheet.date_system))
11736 }
11737
11738 fn set_delta_overlay_cell_raw(
11739 &mut self,
11740 sheet: &str,
11741 row: u32,
11742 col: u32,
11743 value: Option<LiteralValue>,
11744 ) {
11745 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
11746 return;
11747 }
11748
11749 self.ensure_arrow_sheet(sheet);
11750 let date_system = self.arrow_sheet_date_system(sheet);
11751 let ov_opt = value
11752 .as_ref()
11753 .map(|value| Self::literal_to_overlay_value(value, date_system));
11754 let row0 = row.saturating_sub(1) as usize;
11755 let col0 = col.saturating_sub(1) as usize;
11756 let asheet = self
11757 .arrow_sheets
11758 .sheet_mut(sheet)
11759 .expect("ArrowSheet must exist");
11760
11761 let cur_cols = asheet.columns.len();
11762 if col0 >= cur_cols {
11763 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11764 }
11765 if row0 >= asheet.nrows as usize {
11766 if asheet.columns.is_empty() {
11767 asheet.insert_columns(0, 1);
11768 }
11769 asheet.ensure_row_capacity(row0 + 1);
11770 }
11771
11772 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11773 return;
11774 };
11775 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
11776 return;
11777 };
11778
11779 if let Some(ov) = ov_opt {
11780 let _ = ch.overlay.set(in_off, ov);
11781 } else {
11782 let _ = ch.overlay.remove(in_off);
11783 }
11784 }
11785
11786 fn set_computed_overlay_cell_raw(
11787 &mut self,
11788 sheet: &str,
11789 row: u32,
11790 col: u32,
11791 value: Option<LiteralValue>,
11792 ) {
11793 if !(self.config.arrow_storage_enabled
11794 && self.config.delta_overlay_enabled
11795 && self.config.write_formula_overlay_enabled)
11796 {
11797 return;
11798 }
11799
11800 self.ensure_arrow_sheet(sheet);
11801 let date_system = self.arrow_sheet_date_system(sheet);
11802 let ov_opt = value
11803 .as_ref()
11804 .map(|value| Self::literal_to_overlay_value(value, date_system));
11805 let row0 = row.saturating_sub(1) as usize;
11806 let col0 = col.saturating_sub(1) as usize;
11807 let asheet = self
11808 .arrow_sheets
11809 .sheet_mut(sheet)
11810 .expect("ArrowSheet must exist");
11811
11812 let cur_cols = asheet.columns.len();
11813 if col0 >= cur_cols {
11814 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
11815 }
11816 if row0 >= asheet.nrows as usize {
11817 if asheet.columns.is_empty() {
11818 asheet.insert_columns(0, 1);
11819 }
11820 asheet.ensure_row_capacity(row0 + 1);
11821 }
11822
11823 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
11824 return;
11825 };
11826 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
11827 return;
11828 };
11829
11830 let delta = if let Some(ov) = ov_opt {
11831 ch.computed_overlay.set(in_off, ov)
11832 } else {
11833 ch.computed_overlay.remove(in_off)
11834 };
11835 self.adjust_computed_overlay_bytes(delta);
11836 }
11837
11838 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
11839 use crate::engine::ArrowOp;
11840
11841 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
11842 Box::new(batch.ops.iter().rev())
11843 } else {
11844 Box::new(batch.ops.iter())
11845 };
11846
11847 for op in iter {
11848 match op {
11849 ArrowOp::SetDeltaCell {
11850 sheet_id,
11851 row0,
11852 col0,
11853 old,
11854 new,
11855 } => {
11856 let sheet = self.graph.sheet_name(*sheet_id).to_string();
11857 let v = if undo { old.clone() } else { new.clone() };
11858 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
11859 }
11860 ArrowOp::SetComputedCell {
11861 sheet_id,
11862 row0,
11863 col0,
11864 old,
11865 new,
11866 } => {
11867 let sheet = self.graph.sheet_name(*sheet_id).to_string();
11868 let v = if undo { old.clone() } else { new.clone() };
11869 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
11870 }
11871 ArrowOp::RestoreComputedRect {
11872 sheet_id,
11873 sr0,
11874 sc0,
11875 er0,
11876 ec0,
11877 old,
11878 new,
11879 } => {
11880 let sheet = self.graph.sheet_name(*sheet_id).to_string();
11881 let vals = if undo { old } else { new };
11882 let height = (*er0).saturating_sub(*sr0) as usize + 1;
11883 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
11884 for r in 0..height {
11885 for c in 0..width {
11886 let v = vals
11887 .get(r)
11888 .and_then(|row| row.get(c))
11889 .cloned()
11890 .unwrap_or(LiteralValue::Empty);
11891 self.set_computed_overlay_cell_raw(
11892 &sheet,
11893 *sr0 + 1 + r as u32,
11894 *sc0 + 1 + c as u32,
11895 Some(v),
11896 );
11897 }
11898 }
11899 }
11900 ArrowOp::InsertRows {
11901 sheet_id,
11902 before0,
11903 count,
11904 } => {
11905 let sheet = self.graph.sheet_name(*sheet_id).to_string();
11906 self.ensure_arrow_sheet(&sheet);
11907 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
11908 if undo {
11909 asheet.delete_rows(*before0 as usize, *count as usize);
11910 } else {
11911 asheet.insert_rows(*before0 as usize, *count as usize);
11912 }
11913 }
11914 self.purge_derived_formats_after_row(*sheet_id, *before0);
11915 }
11916 ArrowOp::InsertCols {
11917 sheet_id,
11918 before0,
11919 count,
11920 } => {
11921 let sheet = self.graph.sheet_name(*sheet_id).to_string();
11922 self.ensure_arrow_sheet(&sheet);
11923 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
11924 if undo {
11925 asheet.delete_columns(*before0 as usize, *count as usize);
11926 } else {
11927 asheet.insert_columns(*before0 as usize, *count as usize);
11928 }
11929 }
11930 self.purge_derived_formats_after_col(*sheet_id, *before0);
11931 }
11932 }
11933 }
11934 }
11935
11936 fn record_spill_ops_into_arrow_undo(
11937 &mut self,
11938 undo: &mut crate::engine::ArrowUndoBatch,
11939 events: &[crate::engine::ChangeEvent],
11940 ) {
11941 use crate::engine::ChangeEvent;
11942 use formualizer_common::LiteralValue;
11943
11944 #[allow(clippy::type_complexity)]
11945 let rect_from_snapshot =
11946 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
11947 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
11948 if snap.target_cells.is_empty() {
11949 return None;
11950 }
11951 let sheet_id = snap.target_cells[0].sheet_id;
11952 let sr0 = snap.target_cells[0].coord.row();
11953 let sc0 = snap.target_cells[0].coord.col();
11954 if snap.values.is_empty() || snap.values[0].is_empty() {
11955 return None;
11956 }
11957 let h = snap.values.len() as u32;
11958 let w = snap.values[0].len() as u32;
11959 let er0 = sr0.saturating_add(h.saturating_sub(1));
11960 let ec0 = sc0.saturating_add(w.saturating_sub(1));
11961 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
11962 };
11963
11964 for ev in events {
11965 match ev {
11966 ChangeEvent::SpillCommitted { old, new, .. } => {
11967 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
11968 let old_vals = if let Some(old_snap) = old {
11969 rect_from_snapshot(old_snap)
11970 .map(|(_, _, _, _, _, v)| v)
11971 .unwrap_or_else(|| {
11972 vec![
11973 vec![LiteralValue::Empty; new_vals[0].len()];
11974 new_vals.len()
11975 ]
11976 })
11977 } else {
11978 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
11979 };
11980 undo.record_restore_computed_rect(
11981 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
11982 );
11983 }
11984 }
11985 ChangeEvent::SpillCleared { old, .. } => {
11986 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
11987 let new_vals =
11988 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
11989 undo.record_restore_computed_rect(
11990 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
11991 );
11992 }
11993 }
11994 _ => {}
11995 }
11996 }
11997 }
11998
11999 fn mirror_value_to_computed_overlay(
12004 &mut self,
12005 sheet: &str,
12006 row: u32,
12007 col: u32,
12008 value: &LiteralValue,
12009 ) {
12010 if !(self.config.arrow_storage_enabled
12011 && self.config.delta_overlay_enabled
12012 && self.config.write_formula_overlay_enabled)
12013 {
12014 return;
12015 }
12016 if self.computed_overlay_mirroring_disabled {
12017 return;
12018 }
12019
12020 let date_system = self.arrow_sheet_date_system(sheet);
12021 let ov = Self::literal_to_overlay_value(value, date_system);
12022 self.write_computed_overlay_value_0based(
12023 sheet,
12024 row.saturating_sub(1),
12025 col.saturating_sub(1),
12026 ov,
12027 );
12028 }
12029
12030 fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
12031 if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
12032 self.record_derived_format_at(cell, format);
12033 }
12034 }
12035
12036 fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
12037 #[cfg(test)]
12038 self.derived_format_operations_for_test
12039 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12040 let format = format.filter(|id| *id != crate::format::FormatId::GENERAL);
12041 self.derived_formats.set(cell, format);
12042 }
12043
12044 fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
12045 self.derived_formats.set(cell, None);
12046 if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
12047 arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
12048 }
12049 }
12050
12051 fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
12052 let cells = events
12053 .iter()
12054 .filter_map(|event| match event {
12055 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
12056 Some(*addr)
12057 }
12058 _ => None,
12059 })
12060 .collect::<FxHashSet<_>>();
12061 for cell in cells {
12062 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
12063 self.clear_cell_format_state(&sheet, cell);
12064 }
12065 }
12066
12067 fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
12068 self.derived_formats
12069 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.row() < start0);
12070 }
12071
12072 fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
12073 self.derived_formats
12074 .retain(|cell| cell.sheet_id != sheet_id || cell.coord.col() < start0);
12075 }
12076
12077 fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
12078 self.derived_formats
12079 .retain(|cell| cell.sheet_id != sheet_id);
12080 }
12081
12082 #[cfg(test)]
12083 pub(crate) fn debug_computed_overlay_format_0based(
12084 &self,
12085 sheet: &str,
12086 row0: u32,
12087 col0: u32,
12088 ) -> Option<crate::format::FormatId> {
12089 let sheet = self.arrow_sheets.sheet(sheet)?;
12090 let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
12091 sheet
12092 .columns
12093 .get(col0 as usize)?
12094 .chunk(chunk_idx)?
12095 .computed_overlay
12096 .get_format(row_in_chunk)
12097 }
12098
12099 #[cfg(test)]
12100 pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
12101 &self,
12102 sheet: &str,
12103 row0: u32,
12104 col0: u32,
12105 ) -> bool {
12106 let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
12107 return false;
12108 };
12109 let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
12110 return false;
12111 };
12112 sheet
12113 .columns
12114 .get(col0 as usize)
12115 .and_then(|column| column.chunk(chunk_idx))
12116 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12117 }
12118
12119 #[cfg(test)]
12120 pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
12121 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12122 self.derived_formats
12123 .set(CellRef::new_absolute(sheet_id, row0, col0), None);
12124 }
12125 }
12126
12127 #[cfg(test)]
12128 pub(crate) fn debug_record_derived_format_0based(
12129 &self,
12130 sheet: &str,
12131 row0: u32,
12132 col0: u32,
12133 format: Option<crate::format::FormatId>,
12134 ) {
12135 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
12136 self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
12137 }
12138 }
12139
12140 #[cfg(test)]
12141 pub(crate) fn debug_derived_format_0based(
12142 &self,
12143 sheet: &str,
12144 row0: u32,
12145 col0: u32,
12146 ) -> Option<crate::format::FormatId> {
12147 let sheet_id = self.graph.sheet_id(sheet)?;
12148 self.derived_formats
12149 .get(&CellRef::new_absolute(sheet_id, row0, col0))
12150 }
12151
12152 #[cfg(test)]
12153 pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
12154 self.derived_format_operations_for_test
12155 .store(0, std::sync::atomic::Ordering::Relaxed);
12156 self.computed_overlay_set_explicit_entry_operations_for_test = 0;
12157 self.computed_overlay_stale_clear_range_effects_for_test = 0;
12158 self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
12159 self.computed_format_vector_allocations_for_test
12160 .store(0, std::sync::atomic::Ordering::Relaxed);
12161 }
12162
12163 #[cfg(test)]
12164 pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
12165 (
12166 self.derived_format_operations_for_test
12167 .load(std::sync::atomic::Ordering::Relaxed),
12168 self.computed_overlay_set_explicit_entry_operations_for_test,
12169 self.computed_format_vector_allocations_for_test
12170 .load(std::sync::atomic::Ordering::Relaxed),
12171 self.computed_overlay_stale_clear_range_effects_for_test,
12172 self.computed_overlay_stale_clear_offset_attempts_for_test,
12173 )
12174 }
12175
12176 fn write_computed_overlay_format_0based(
12177 &mut self,
12178 sheet: &str,
12179 row0: u32,
12180 col0: u32,
12181 format: Option<crate::format::FormatId>,
12182 ) {
12183 self.ensure_arrow_sheet(sheet);
12184 let (row0, col0) = (row0 as usize, col0 as usize);
12185 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
12186 return;
12187 };
12188 if col0 >= asheet.columns.len() {
12189 asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
12190 }
12191 if row0 >= asheet.nrows as usize {
12192 asheet.ensure_row_capacity(row0 + 1);
12193 }
12194 let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
12195 return;
12196 };
12197 if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
12198 chunk.computed_overlay.set_format(offset, format);
12199 }
12200 }
12201
12202 fn write_computed_cell_0based(
12207 &mut self,
12208 cell: CellRef,
12209 value: &LiteralValue,
12210 format: Option<crate::format::FormatId>,
12211 ) {
12212 if !(self.config.arrow_storage_enabled
12213 && self.config.delta_overlay_enabled
12214 && self.config.write_formula_overlay_enabled)
12215 || self.computed_overlay_mirroring_disabled
12216 {
12217 return;
12218 }
12219 let sheet = self.graph.sheet_name(cell.sheet_id);
12220 let index = match self
12221 .arrow_sheets
12222 .sheets
12223 .iter()
12224 .position(|s| s.name.as_ref() == sheet)
12225 {
12226 Some(index) => index,
12227 None => {
12228 let sheet = sheet.to_string();
12229 self.ensure_arrow_sheet(&sheet);
12230 self.arrow_sheets.sheets.len() - 1
12231 }
12232 };
12233 let (row0, col0) = (cell.coord.row() as usize, cell.coord.col() as usize);
12234 let asheet = &mut self.arrow_sheets.sheets[index];
12235 let ov = Self::literal_to_overlay_value(value, asheet.date_system);
12236 let cur_cols = asheet.columns.len();
12237 if col0 >= cur_cols {
12238 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12239 }
12240 if row0 >= asheet.nrows as usize {
12241 asheet.ensure_row_capacity(row0 + 1);
12242 }
12243 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12244 return;
12245 };
12246 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12247 return;
12248 };
12249 let delta = ch.computed_overlay.set_scalar(in_off, ov);
12250 ch.computed_overlay.set_format(in_off, format);
12251 self.adjust_computed_overlay_bytes(delta);
12252 if let Some(cap) = self.config.max_overlay_memory_bytes
12253 && self.computed_overlay_bytes_estimate > cap
12254 {
12255 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12256 }
12257 }
12258
12259 fn write_computed_overlay_value_0based(
12260 &mut self,
12261 sheet: &str,
12262 row0: u32,
12263 col0: u32,
12264 value: OverlayValue,
12265 ) {
12266 if !(self.config.arrow_storage_enabled
12267 && self.config.delta_overlay_enabled
12268 && self.config.write_formula_overlay_enabled)
12269 {
12270 return;
12271 }
12272 if self.computed_overlay_mirroring_disabled {
12273 return;
12274 }
12275
12276 self.ensure_arrow_sheet(sheet);
12277
12278 let row0 = row0 as usize;
12279 let col0 = col0 as usize;
12280 let asheet = self
12281 .arrow_sheets
12282 .sheet_mut(sheet)
12283 .expect("ArrowSheet must exist");
12284
12285 let cur_cols = asheet.columns.len();
12286 if col0 >= cur_cols {
12287 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12288 }
12289
12290 if row0 >= asheet.nrows as usize {
12291 if asheet.columns.is_empty() {
12292 asheet.insert_columns(0, 1);
12293 }
12294 asheet.ensure_row_capacity(row0 + 1);
12295 }
12296
12297 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
12298 return;
12299 };
12300 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
12301 return;
12302 };
12303
12304 let delta = ch.computed_overlay.set_scalar(in_off, value);
12305 self.adjust_computed_overlay_bytes(delta);
12306
12307 if let Some(cap) = self.config.max_overlay_memory_bytes
12308 && self.computed_overlay_bytes_estimate > cap
12309 {
12310 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12311 }
12312 }
12313
12314 pub(crate) fn plan_computed_write_coalescing(
12315 &self,
12316 buffer: &ComputedWriteBuffer,
12317 ) -> ComputedWriteCoalescingPlan {
12318 self.plan_computed_write_coalescing_from_writes(
12319 buffer.writes().iter().cloned(),
12320 buffer.formats_present,
12321 )
12322 }
12323
12324 fn plan_owned_computed_write_coalescing(
12325 &self,
12326 writes: Vec<ComputedWrite>,
12327 formats_present: bool,
12328 ) -> ComputedWriteCoalescingPlan {
12329 self.plan_computed_write_coalescing_from_writes(writes, formats_present)
12330 }
12331
12332 fn plan_computed_write_coalescing_from_writes(
12333 &self,
12334 writes: impl IntoIterator<Item = ComputedWrite>,
12335 formats_present: bool,
12336 ) -> ComputedWriteCoalescingPlan {
12337 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
12338 BTreeMap::new();
12339 let mut input_cells = 0usize;
12340 let mut located: Option<(SheetId, Option<&crate::arrow_store::ArrowSheet>)> = None;
12342
12343 for write in writes {
12344 match write {
12345 ComputedWrite::Cell {
12346 seq,
12347 sheet_id,
12348 row0,
12349 col0,
12350 value,
12351 format_id,
12352 } => {
12353 input_cells = input_cells.saturating_add(1);
12354 let sheet = match located {
12355 Some((id, sheet)) if id == sheet_id => sheet,
12356 _ => {
12357 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12358 located = Some((sheet_id, sheet));
12359 sheet
12360 }
12361 };
12362 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12363 Some(sheet) => {
12364 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize)
12365 }
12366 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12367 row0 as usize,
12368 32 * 1024,
12369 ),
12370 };
12371 groups
12372 .entry(ComputedWriteChunkKey {
12373 sheet_id,
12374 col0,
12375 chunk_idx,
12376 chunk_start_row0,
12377 })
12378 .or_default()
12379 .push(ComputedWriteChunkEntryPlan {
12380 row_in_chunk,
12381 seq,
12382 value,
12383 format_id,
12384 });
12385 }
12386 ComputedWrite::Run {
12387 seq,
12388 sheet_id,
12389 row0,
12390 col0,
12391 entries,
12392 } => {
12393 input_cells = input_cells.saturating_add(entries.len());
12394 let sheet = match located {
12395 Some((id, sheet)) if id == sheet_id => sheet,
12396 _ => {
12397 let sheet = self.arrow_sheets.sheet(self.graph.sheet_name(sheet_id));
12398 located = Some((sheet_id, sheet));
12399 sheet
12400 }
12401 };
12402 let mut segment: Vec<ComputedWriteChunkEntryPlan> = Vec::new();
12406 let mut segment_key: Option<ComputedWriteChunkKey> = None;
12407 for (k, (value, format_id)) in entries.into_iter().enumerate() {
12408 let row = row0.saturating_add(k as u32) as usize;
12409 let (chunk_idx, chunk_start_row0, row_in_chunk) = match sheet {
12410 Some(sheet) => {
12411 Self::locate_row_in_sheet_for_computed_write_plan(sheet, row)
12412 }
12413 None => Self::locate_row_in_empty_sheet_for_computed_write_plan(
12414 row,
12415 32 * 1024,
12416 ),
12417 };
12418 let key = ComputedWriteChunkKey {
12419 sheet_id,
12420 col0,
12421 chunk_idx,
12422 chunk_start_row0,
12423 };
12424 if segment_key != Some(key)
12425 && let Some(done) = segment_key.replace(key)
12426 {
12427 groups.entry(done).or_default().append(&mut segment);
12428 }
12429 segment.push(ComputedWriteChunkEntryPlan {
12430 row_in_chunk,
12431 seq,
12432 value,
12433 format_id,
12434 });
12435 }
12436 if let Some(done) = segment_key {
12437 groups.entry(done).or_default().append(&mut segment);
12438 }
12439 }
12440 ComputedWrite::Rect {
12441 seq,
12442 sheet_id,
12443 sr0,
12444 sc0,
12445 values,
12446 } => {
12447 for (r_off, row) in values.into_iter().enumerate() {
12448 for (c_off, value) in row.into_iter().enumerate() {
12449 input_cells = input_cells.saturating_add(1);
12450 self.push_computed_write_plan_entry(
12451 &mut groups,
12452 seq,
12453 sheet_id,
12454 sr0.saturating_add(r_off as u32),
12455 sc0.saturating_add(c_off as u32),
12456 value,
12457 None,
12458 );
12459 }
12460 }
12461 }
12462 }
12463 }
12464
12465 let mut plan = ComputedWriteCoalescingPlan {
12466 chunks: Vec::with_capacity(groups.len()),
12467 input_cells,
12468 coalesced_cells: 0,
12469 overwritten_cells: 0,
12470 };
12471 for (key, entries) in groups {
12472 let computed_lane_has_formats =
12473 self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
12474 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
12475 key,
12476 entries,
12477 formats_present,
12478 computed_lane_has_formats,
12479 );
12480 #[cfg(test)]
12481 if matches!(
12482 &chunk_plan.format_effect,
12483 ComputedWriteChunkFormatEffect::SetExplicit(_)
12484 ) {
12485 self.computed_format_vector_allocations_for_test
12486 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
12487 }
12488 plan.coalesced_cells = plan
12489 .coalesced_cells
12490 .saturating_add(chunk_plan.entries.len());
12491 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
12492 plan.chunks.push(chunk_plan);
12493 }
12494 debug_assert_eq!(
12495 plan.input_cells,
12496 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
12497 );
12498 plan
12499 }
12500
12501 fn push_computed_write_plan_entry(
12502 &self,
12503 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
12504 seq: u64,
12505 sheet_id: SheetId,
12506 row0: u32,
12507 col0: u32,
12508 value: OverlayValue,
12509 format_id: Option<crate::format::FormatId>,
12510 ) {
12511 let (chunk_idx, chunk_start_row0, row_in_chunk) =
12512 self.locate_computed_write_chunk(sheet_id, row0);
12513 let key = ComputedWriteChunkKey {
12514 sheet_id,
12515 col0,
12516 chunk_idx,
12517 chunk_start_row0,
12518 };
12519 groups
12520 .entry(key)
12521 .or_default()
12522 .push(ComputedWriteChunkEntryPlan {
12523 row_in_chunk,
12524 seq,
12525 value,
12526 format_id,
12527 });
12528 }
12529
12530 fn computed_overlay_chunk_has_formats(
12531 &self,
12532 sheet_id: SheetId,
12533 col0: u32,
12534 chunk_idx: usize,
12535 ) -> bool {
12536 let sheet_name = self.graph.sheet_name(sheet_id);
12537 self.arrow_sheets
12538 .sheet(sheet_name)
12539 .and_then(|sheet| sheet.columns.get(col0 as usize))
12540 .and_then(|column| column.chunk(chunk_idx))
12541 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
12542 }
12543
12544 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
12545 let sheet_name = self.graph.sheet_name(sheet_id);
12546 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
12547 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
12548 }
12549 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
12550 }
12551
12552 fn locate_row_in_sheet_for_computed_write_plan(
12553 sheet: &crate::arrow_store::ArrowSheet,
12554 row0: usize,
12555 ) -> (usize, u32, usize) {
12556 if row0 < sheet.nrows as usize
12557 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
12558 {
12559 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
12560 return (chunk_idx, chunk_start as u32, row_in_chunk);
12561 }
12562
12563 let chunk_rows = sheet.chunk_rows.max(1);
12564 if sheet.chunk_starts.is_empty() {
12565 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
12566 }
12567
12568 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
12569 let mut chunk_start = sheet.chunk_starts[chunk_idx];
12570 while chunk_start.saturating_add(chunk_rows) <= row0 {
12571 chunk_idx = chunk_idx.saturating_add(1);
12572 chunk_start = chunk_start.saturating_add(chunk_rows);
12573 }
12574 (
12575 chunk_idx,
12576 chunk_start as u32,
12577 row0.saturating_sub(chunk_start),
12578 )
12579 }
12580
12581 fn locate_row_in_empty_sheet_for_computed_write_plan(
12582 row0: usize,
12583 chunk_rows: usize,
12584 ) -> (usize, u32, usize) {
12585 let chunk_rows = chunk_rows.max(1);
12586 let chunk_idx = row0 / chunk_rows;
12587 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
12588 (
12589 chunk_idx,
12590 chunk_start as u32,
12591 row0.saturating_sub(chunk_start),
12592 )
12593 }
12594
12595 #[cfg(test)]
12596 pub(crate) fn debug_plan_computed_write_coalescing(
12597 &self,
12598 buffer: &ComputedWriteBuffer,
12599 ) -> ComputedWriteCoalescingPlan {
12600 self.plan_computed_write_coalescing(buffer)
12601 }
12602
12603 pub(crate) fn flush_computed_write_buffer(
12604 &mut self,
12605 buffer: &mut ComputedWriteBuffer,
12606 ) -> Result<(), ExcelError> {
12607 if buffer.is_empty() {
12608 return Ok(());
12609 }
12610
12611 self.resource_checkpoint(0)?;
12616 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
12617 let (writes, formats_present) = buffer.take_writes();
12618 let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
12619 self.flush_computed_write_plan(plan);
12620 self.observe_evaluation_commit_window(commit_started);
12621
12622 Ok(())
12623 }
12624
12625 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
12626 for chunk in plan.chunks {
12627 self.flush_computed_write_chunk_plan(chunk);
12628 }
12629 }
12630
12631 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
12632 match &chunk.shape {
12633 ComputedWriteChunkPlanShape::Point => {
12634 self.flush_computed_write_chunk_plan_as_points(chunk);
12635 }
12636 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
12637 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
12638 }
12639 ComputedWriteChunkPlanShape::DenseRange { .. } => {
12640 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12641 }
12642 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
12643 if Self::should_emit_computed_run_fragment(*len, *runs) {
12644 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
12645 } else {
12646 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
12647 }
12648 }
12649 }
12650 }
12651
12652 #[inline]
12653 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
12654 runs <= len / 2
12655 }
12656
12657 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
12658 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
12659 for entry in chunk.entries {
12660 let row0 = chunk
12661 .chunk_start_row0
12662 .saturating_add(entry.row_in_chunk as u32);
12663 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
12664 }
12665 self.apply_computed_overlay_format_effect(
12666 chunk.sheet_id,
12667 chunk.col0,
12668 chunk.chunk_idx,
12669 chunk.format_effect,
12670 );
12671 }
12672
12673 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
12674 &mut self,
12675 chunk: ComputedWriteChunkPlan,
12676 ) {
12677 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
12678 let sheet_id = chunk.sheet_id;
12679 let col0 = chunk.col0;
12680 let chunk_idx = chunk.chunk_idx;
12681 let chunk_start_row0 = chunk.chunk_start_row0;
12682 let format_effect = chunk.format_effect;
12683 let items: Vec<(usize, OverlayValue)> = chunk
12684 .entries
12685 .into_iter()
12686 .map(|entry| (entry.row_in_chunk, entry.value))
12687 .collect();
12688 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
12689 items,
12690 point_estimate,
12691 ) {
12692 Some(Ok(fragment)) => {
12693 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
12694 }
12695 Some(Err(cells)) => {
12696 self.flush_computed_overlay_cells_as_points(
12697 sheet_id,
12698 col0,
12699 chunk_start_row0,
12700 cells,
12701 );
12702 }
12703 None => {}
12704 }
12705 self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
12706 }
12707
12708 #[inline]
12709 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
12710 chunk
12711 .entries
12712 .iter()
12713 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
12714 .fold(0usize, usize::saturating_add)
12715 }
12716
12717 fn flush_computed_overlay_cells_as_points(
12718 &mut self,
12719 sheet_id: SheetId,
12720 col0: u32,
12721 chunk_start_row0: u32,
12722 cells: Vec<(usize, OverlayValue)>,
12723 ) {
12724 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12725 for (row_in_chunk, value) in cells {
12726 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
12727 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
12728 }
12729 }
12730
12731 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12732 if chunk.entries.is_empty() {
12733 return;
12734 }
12735 let start = chunk.entries[0].row_in_chunk;
12736 let values: Vec<OverlayValue> =
12737 chunk.entries.into_iter().map(|entry| entry.value).collect();
12738 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
12739 self.apply_computed_overlay_fragment(
12740 chunk.sheet_id,
12741 chunk.col0,
12742 chunk.chunk_idx,
12743 fragment,
12744 );
12745 }
12746 self.apply_computed_overlay_format_effect(
12747 chunk.sheet_id,
12748 chunk.col0,
12749 chunk.chunk_idx,
12750 chunk.format_effect,
12751 );
12752 }
12753
12754 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
12755 if chunk.entries.is_empty() {
12756 return;
12757 }
12758 let start = chunk.entries[0].row_in_chunk;
12759 let values: Vec<OverlayValue> =
12760 chunk.entries.into_iter().map(|entry| entry.value).collect();
12761 if let Some(fragment) = OverlayFragment::run_range(start, values) {
12762 self.apply_computed_overlay_fragment(
12763 chunk.sheet_id,
12764 chunk.col0,
12765 chunk.chunk_idx,
12766 fragment,
12767 );
12768 }
12769 self.apply_computed_overlay_format_effect(
12770 chunk.sheet_id,
12771 chunk.col0,
12772 chunk.chunk_idx,
12773 chunk.format_effect,
12774 );
12775 }
12776
12777 fn apply_computed_overlay_format_effect(
12778 &mut self,
12779 sheet_id: SheetId,
12780 col0: u32,
12781 chunk_idx: usize,
12782 effect: ComputedWriteChunkFormatEffect,
12783 ) {
12784 if !(self.config.arrow_storage_enabled
12785 && self.config.delta_overlay_enabled
12786 && self.config.write_formula_overlay_enabled)
12787 || self.computed_overlay_mirroring_disabled
12788 {
12789 return;
12790 }
12791
12792 let sheet_name = self.graph.sheet_name(sheet_id);
12793 let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
12794 return;
12795 };
12796 let Some(chunk) = sheet
12797 .columns
12798 .get_mut(col0 as usize)
12799 .and_then(|column| column.chunk_mut(chunk_idx))
12800 else {
12801 return;
12802 };
12803 match effect {
12804 ComputedWriteChunkFormatEffect::NoFormatWork => {}
12805 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
12806 start,
12807 end,
12808 }) => {
12809 #[cfg(test)]
12810 {
12811 self.computed_overlay_stale_clear_range_effects_for_test = self
12812 .computed_overlay_stale_clear_range_effects_for_test
12813 .saturating_add(1);
12814 }
12815 chunk.computed_overlay.clear_format_range(start, end);
12816 }
12817 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
12818 offsets,
12819 )) => {
12820 #[cfg(test)]
12821 {
12822 self.computed_overlay_stale_clear_offset_attempts_for_test = self
12823 .computed_overlay_stale_clear_offset_attempts_for_test
12824 .saturating_add(offsets.len() as u64);
12825 }
12826 chunk.computed_overlay.clear_format_offsets(&offsets);
12827 }
12828 ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
12829 #[cfg(test)]
12830 {
12831 self.computed_overlay_set_explicit_entry_operations_for_test = self
12832 .computed_overlay_set_explicit_entry_operations_for_test
12833 .saturating_add(formats.len() as u64);
12834 }
12835 for (row_in_chunk, format_id) in formats {
12836 chunk.computed_overlay.set_format(row_in_chunk, format_id);
12837 }
12838 }
12839 }
12840 }
12841
12842 fn apply_computed_overlay_fragment(
12843 &mut self,
12844 sheet_id: SheetId,
12845 col0: u32,
12846 chunk_idx: usize,
12847 fragment: OverlayFragment,
12848 ) {
12849 if !(self.config.arrow_storage_enabled
12850 && self.config.delta_overlay_enabled
12851 && self.config.write_formula_overlay_enabled)
12852 {
12853 return;
12854 }
12855 if self.computed_overlay_mirroring_disabled {
12856 return;
12857 }
12858
12859 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
12860 self.ensure_arrow_sheet(&sheet_name);
12861
12862 let col0 = col0 as usize;
12863 let asheet = self
12864 .arrow_sheets
12865 .sheet_mut(&sheet_name)
12866 .expect("ArrowSheet must exist");
12867
12868 let cur_cols = asheet.columns.len();
12869 if col0 >= cur_cols {
12870 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
12871 }
12872
12873 let start_row0 = asheet
12874 .chunk_starts
12875 .get(chunk_idx)
12876 .copied()
12877 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
12878 let required_rows =
12879 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
12880 if required_rows > asheet.nrows as usize {
12881 if asheet.columns.is_empty() {
12882 asheet.insert_columns(0, 1);
12883 }
12884 asheet.ensure_row_capacity(required_rows);
12885 }
12886
12887 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
12888 return;
12889 };
12890 let delta = ch.computed_overlay.apply_fragment(fragment);
12891 self.adjust_computed_overlay_bytes(delta);
12892
12893 if let Some(cap) = self.config.max_overlay_memory_bytes
12894 && self.computed_overlay_bytes_estimate > cap
12895 {
12896 self.disable_computed_overlay_mirroring_due_to_budget(cap);
12897 }
12898 }
12899
12900 #[inline]
12901 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
12902 if delta >= 0 {
12903 self.computed_overlay_bytes_estimate = self
12904 .computed_overlay_bytes_estimate
12905 .saturating_add(delta as usize);
12906 } else {
12907 self.computed_overlay_bytes_estimate = self
12908 .computed_overlay_bytes_estimate
12909 .saturating_sub((-delta) as usize);
12910 }
12911 }
12912
12913 fn clear_all_computed_overlays(&mut self) {
12914 let mut freed_total = 0usize;
12915 for sh in self.arrow_sheets.sheets.iter_mut() {
12916 for col in sh.columns.iter_mut() {
12917 for ch in col.chunks.iter_mut() {
12918 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
12919 }
12920 for ch in col.sparse_chunks.values_mut() {
12921 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
12922 }
12923 }
12924 }
12925 self.computed_overlay_bytes_estimate = self
12926 .computed_overlay_bytes_estimate
12927 .saturating_sub(freed_total);
12928 }
12929
12930 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
12931 self.compact_all_computed_overlays();
12934 }
12935
12936 fn compact_all_computed_overlays(&mut self) {
12939 let mut freed_total = 0usize;
12940 for sheet in self.arrow_sheets.sheets.iter_mut() {
12941 for col_idx in 0..sheet.columns.len() {
12942 let num_dense = sheet.columns[col_idx].chunks.len();
12944 for ch_idx in 0..num_dense {
12945 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
12946 }
12947 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
12949 .sparse_chunks
12950 .keys()
12951 .copied()
12952 .collect();
12953 for ch_idx in sparse_keys {
12954 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
12955 }
12956 }
12957 }
12958 self.computed_overlay_bytes_estimate = self
12959 .computed_overlay_bytes_estimate
12960 .saturating_sub(freed_total);
12961 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
12962 }
12963
12964 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
12965 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
12966 }
12967
12968 fn record_vertex_value_to_overlay(
12969 &mut self,
12970 vertex_id: VertexId,
12971 value: &LiteralValue,
12972 computed_writes: Option<&mut ComputedWriteBuffer>,
12973 ) -> Result<(), ExcelError> {
12974 if !(self.config.arrow_storage_enabled
12975 && self.config.delta_overlay_enabled
12976 && self.config.write_formula_overlay_enabled)
12977 {
12978 return Ok(());
12979 }
12980 if self.computed_overlay_mirroring_disabled {
12981 return Ok(());
12982 }
12983 if !matches!(
12984 self.graph.get_vertex_kind(vertex_id),
12985 VertexKind::FormulaScalar | VertexKind::FormulaArray
12986 ) {
12987 return Ok(());
12988 }
12989 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
12990 return Ok(());
12991 };
12992 let Some(buffer) = computed_writes else {
12993 let format_id = self.derived_formats.get(&cell);
12995 self.write_computed_cell_0based(cell, value, format_id);
12996 return Ok(());
12997 };
12998 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
12999 let date_system = self.arrow_sheet_date_system(&sheet_name);
13000 let ov = Self::literal_to_overlay_value(value, date_system);
13001 {
13002 let format_id = self.derived_formats.get(&cell);
13003 buffer.push_cell_with_format(
13004 cell.sheet_id,
13005 cell.coord.row(),
13006 cell.coord.col(),
13007 ov,
13008 format_id,
13009 );
13010 if self.should_flush_computed_write_buffer(buffer) {
13011 self.flush_computed_write_buffer(buffer)?;
13012 }
13013 }
13014 Ok(())
13015 }
13016
13017 #[inline]
13018 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
13019 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
13020 if cap == 0 {
13021 return false;
13022 }
13023 self.computed_overlay_bytes_estimate
13024 .saturating_add(buffer.estimated_bytes())
13025 > cap
13026 })
13027 }
13028
13029 pub fn overlay_memory_usage(&self) -> usize {
13031 self.computed_overlay_bytes_estimate
13032 }
13033
13034 #[cfg(test)]
13035 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
13036 self.overlay_compactions
13037 }
13038
13039 #[cfg(test)]
13040 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
13041 let mut total = 0usize;
13042 for sheet in &self.arrow_sheets.sheets {
13043 for column in &sheet.columns {
13044 for chunk in &column.chunks {
13045 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13046 }
13047 for chunk in column.sparse_chunks.values() {
13048 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
13049 }
13050 }
13051 }
13052 self.computed_overlay_bytes_estimate = total;
13053 total
13054 }
13055
13056 fn resolve_sheet_locator_for_write(
13057 &mut self,
13058 loc: formualizer_common::SheetLocator<'_>,
13059 current_sheet: &str,
13060 ) -> Result<SheetId, ExcelError> {
13061 Ok(match loc {
13062 formualizer_common::SheetLocator::Id(id) => id,
13063 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
13064 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
13065 })
13066 }
13067
13068 fn resolve_sheet_locator_for_read(
13069 &self,
13070 loc: formualizer_common::SheetLocator<'_>,
13071 current_sheet: &str,
13072 ) -> Result<SheetId, ExcelError> {
13073 match loc {
13074 formualizer_common::SheetLocator::Id(id) => Ok(id),
13075 formualizer_common::SheetLocator::Name(name) => self
13076 .graph
13077 .sheet_id(name.as_ref())
13078 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13079 formualizer_common::SheetLocator::Current => self
13080 .graph
13081 .sheet_id(current_sheet)
13082 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
13083 }
13084 }
13085
13086 pub fn set_cell_value(
13088 &mut self,
13089 sheet: &str,
13090 row: u32,
13091 col: u32,
13092 value: LiteralValue,
13093 ) -> Result<(), ExcelError> {
13094 self.observe_function_semantic_epoch()?;
13095 let sheet_existed = self.graph.sheet_id(sheet).is_some();
13096 let sheet_id = self.graph.sheet_id_mut(sheet);
13097 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13098 let replaced = self
13099 .graph
13100 .get_vertex_id_for_address(&cell_ref)
13101 .filter(|&vertex| {
13102 matches!(
13103 self.graph.get_vertex_kind(vertex),
13104 VertexKind::FormulaScalar | VertexKind::FormulaArray
13105 )
13106 });
13107 let replaced_formula = replaced.is_some();
13108 if let Some(anchor) = replaced
13112 && self.graph.spill_registry_has_anchor(anchor)
13113 {
13114 self.clear_spill_projection_and_mirror(anchor, None);
13115 }
13116 self.graph.set_cell_value(sheet, row, col, value.clone())?;
13117 self.clear_cell_format_state(sheet, cell_ref);
13118 self.record_changed_cell(sheet, row, col);
13119 if !sheet_existed || replaced_formula {
13120 self.mark_topology_edited();
13121 }
13122 self.mirror_value_to_overlay(sheet, row, col, &value);
13124 self.mark_data_edited();
13126 Ok(())
13127 }
13128
13129 fn record_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
13131 let sheet_id = self.graph.sheet_id_mut(sheet);
13132 self.record_structural_change(StructuralScope::Cell {
13133 sheet: sheet_id,
13134 row: row.saturating_sub(1),
13135 col: col.saturating_sub(1),
13136 });
13137 }
13138
13139 fn record_change_for_event(&mut self, event: &ChangeEvent) {
13140 match event {
13141 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
13142 self.record_structural_change(StructuralScope::Cell {
13143 sheet: addr.sheet_id,
13144 row: addr.coord.row(),
13145 col: addr.coord.col(),
13146 });
13147 }
13148 ChangeEvent::SpillCommitted { new, .. } => {
13149 if let Some(scope) = Self::structural_scope_from_cells(&new.target_cells) {
13150 self.record_structural_change(scope);
13151 }
13152 }
13153 ChangeEvent::SpillCleared { old, .. } => {
13154 if let Some(scope) = Self::structural_scope_from_cells(&old.target_cells) {
13155 self.record_structural_change(scope);
13156 }
13157 }
13158 ChangeEvent::DefineName { .. }
13159 | ChangeEvent::UpdateName { .. }
13160 | ChangeEvent::DeleteName { .. }
13161 | ChangeEvent::NamedRangeAdjusted { .. } => {
13162 self.record_structural_change(StructuralScope::AllSheets);
13166 }
13167 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
13168 }
13171 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
13172 self.record_structural_change(StructuralScope::Region(Region::whole_row(
13173 *sheet_id, *row0,
13174 )));
13175 }
13176 ChangeEvent::AddVertex { .. }
13177 | ChangeEvent::RemoveVertex { .. }
13178 | ChangeEvent::EdgeAdded { .. }
13179 | ChangeEvent::EdgeRemoved { .. }
13180 | ChangeEvent::CompoundStart { .. }
13181 | ChangeEvent::CompoundEnd { .. }
13182 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
13183 }
13184 }
13185
13186 fn record_structural_change(&mut self, scope: StructuralScope) {
13187 if let StructuralScope::Cell { sheet, row, col } = scope {
13188 self.wake_spill_owner_of_entered_formula(sheet, row, col);
13189 }
13190 self.wake_spill_blocker_waiters(scope);
13191 self.invalidate_pending_spills(scope);
13192 }
13193
13194 fn wake_spill_blocker_waiters(&mut self, scope: StructuralScope) {
13198 if self.spill_blocker_waiters.is_empty() {
13199 return;
13200 }
13201 let woken = self.spill_blocker_waiters.take_affected(scope);
13202 let woken: Vec<VertexId> = woken
13203 .into_iter()
13204 .filter(|&vertex| {
13205 self.graph.vertex_exists(vertex)
13206 && matches!(
13207 self.graph.get_vertex_kind(vertex),
13208 VertexKind::FormulaScalar | VertexKind::FormulaArray
13209 )
13210 })
13211 .collect();
13212 if !woken.is_empty() {
13213 self.graph.mark_dirty_many(&woken);
13214 }
13215 }
13216
13217 fn wake_spill_owner_of_entered_formula(&mut self, sheet: SheetId, row: u32, col: u32) {
13221 let cell = self.graph.make_cell_ref_internal(sheet, row, col);
13222 if let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
13223 && self.graph.is_foreign_formula_cell(&cell, owner)
13224 {
13225 self.graph.mark_dirty_many(&[owner]);
13226 }
13227 }
13228
13229 fn structural_scope_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
13230 let first = cells.first()?;
13231 let sheet_id = first.sheet_id;
13232 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
13233 return Some(StructuralScope::OpaqueGlobal);
13234 }
13235 let mut row_start = first.coord.row();
13236 let mut row_end = row_start;
13237 let mut col_start = first.coord.col();
13238 let mut col_end = col_start;
13239 for cell in cells.iter().skip(1) {
13240 row_start = row_start.min(cell.coord.row());
13241 row_end = row_end.max(cell.coord.row());
13242 col_start = col_start.min(cell.coord.col());
13243 col_end = col_end.max(cell.coord.col());
13244 }
13245 Some(StructuralScope::Region(Region::rect(
13246 sheet_id, row_start, row_end, col_start, col_end,
13247 )))
13248 }
13249
13250 pub fn set_cell_value_ref(
13251 &mut self,
13252 cell: formualizer_common::SheetCellRef<'_>,
13253 current_sheet: &str,
13254 value: LiteralValue,
13255 ) -> Result<(), ExcelError> {
13256 let owned = cell.into_owned();
13257 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13258 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13259 self.set_cell_value(
13260 &sheet_name,
13261 owned.coord.row() + 1,
13262 owned.coord.col() + 1,
13263 value,
13264 )
13265 }
13266
13267 pub fn set_cell_formula_ref(
13268 &mut self,
13269 cell: formualizer_common::SheetCellRef<'_>,
13270 current_sheet: &str,
13271 ast: ASTNode,
13272 ) -> Result<(), ExcelError> {
13273 let owned = cell.into_owned();
13274 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
13275 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
13276 self.set_cell_formula(
13277 &sheet_name,
13278 owned.coord.row() + 1,
13279 owned.coord.col() + 1,
13280 ast,
13281 )
13282 }
13283
13284 pub fn get_cell_value_ref(
13285 &self,
13286 cell: formualizer_common::SheetCellRef<'_>,
13287 current_sheet: &str,
13288 ) -> Result<Option<LiteralValue>, ExcelError> {
13289 let owned = cell.into_owned();
13290 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
13291 let sheet_name = self.graph.sheet_name(sheet_id);
13292 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
13293 }
13294
13295 pub fn resolve_range_view_sheet_ref<'c>(
13296 &'c self,
13297 r: &formualizer_common::SheetRef<'_>,
13298 current_sheet: &str,
13299 ) -> Result<RangeView<'c>, ExcelError> {
13300 use formualizer_common::SheetLocator;
13301
13302 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
13303 match loc {
13304 SheetLocator::Current => Ok(None),
13305 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
13306 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
13307 }
13308 };
13309
13310 let rt = match r {
13311 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
13312 sheet: sheet_to_opt_name(cell.sheet.clone())?,
13313 row: cell.coord.row() + 1,
13314 col: cell.coord.col() + 1,
13315 row_abs: cell.coord.row_abs(),
13316 col_abs: cell.coord.col_abs(),
13317 },
13318 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
13319 sheet: sheet_to_opt_name(range.sheet.clone())?,
13320 start_row: range.start_row.map(|b| b.index + 1),
13321 start_col: range.start_col.map(|b| b.index + 1),
13322 end_row: range.end_row.map(|b| b.index + 1),
13323 end_col: range.end_col.map(|b| b.index + 1),
13324 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
13325 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
13326 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
13327 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
13328 },
13329 };
13330
13331 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
13332 }
13333
13334 pub fn set_cell_formula(
13336 &mut self,
13337 sheet: &str,
13338 row: u32,
13339 col: u32,
13340 ast: ASTNode,
13341 ) -> Result<(), ExcelError> {
13342 self.observe_function_semantic_epoch()?;
13343 let sheet_id = self.graph.sheet_id_mut(sheet);
13344 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13345 let ingested = {
13346 let mut pipeline = self.ingest_pipeline();
13347 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
13348 };
13349 self.graph.set_cell_formula_with_plan(
13350 sheet,
13351 row,
13352 col,
13353 ingested.ast_id,
13354 &ingested.dep_plan,
13355 ingested.dep_plan.volatile,
13356 ingested.dep_plan.dynamic,
13357 )?;
13358 self.clear_cell_format_state(sheet, placement);
13359 self.record_changed_cell(sheet, row, col);
13360
13361 self.clear_delta_overlay_cell(sheet, row, col);
13366
13367 self.mark_topology_edited();
13369 Ok(())
13370 }
13371
13372 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
13374 where
13375 I: IntoIterator<Item = (u32, u32, ASTNode)>,
13376 {
13377 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
13378 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
13379 let sheet_id = self.graph.sheet_id_mut(sheet);
13380 let ingested = {
13381 let mut pipeline = self.ingest_pipeline();
13382 let inputs = collected.into_iter().map(|(row, col, ast)| {
13383 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13384 (FormulaAstInput::Tree(ast), placement, None)
13385 });
13386 pipeline.ingest_batch(inputs)?
13387 };
13388 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
13389 .into_iter()
13390 .map(|formula| {
13391 (
13392 formula.placement.coord.row() + 1,
13393 formula.placement.coord.col() + 1,
13394 formula.ast_id,
13395 formula.dep_plan,
13396 )
13397 })
13398 .collect();
13399 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
13400 for (row, col) in edited_cells {
13401 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13402 self.clear_cell_format_state(sheet, cell);
13403 self.record_changed_cell(sheet, row, col);
13404 }
13405 if n > 0 {
13407 self.mark_topology_edited();
13408 }
13409 Ok(n)
13410 }
13411
13412 pub fn declare_dynamic_array_anchor(
13441 &mut self,
13442 sheet: &str,
13443 row: u32,
13444 col: u32,
13445 ) -> Result<(), ExcelError> {
13446 let not_a_formula = || {
13447 ExcelError::new(ExcelErrorKind::Ref)
13448 .with_message("Declared dynamic array anchor must be a formula cell")
13449 };
13450 if row == 0 || col == 0 {
13451 return Err(not_a_formula());
13452 }
13453 if self.get_staged_formula_text(sheet, row, col).is_some() {
13454 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
13455 "Declared dynamic array anchor is still staged; build the graph first",
13456 ));
13457 }
13458 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(not_a_formula)?;
13459 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13460 let vertex = self
13461 .graph
13462 .cell_vertex_mut(&cell)
13463 .filter(|v| self.graph.vertex_has_formula(*v))
13464 .ok_or_else(not_a_formula)?;
13465 if self.graph.declare_dynamic_anchor(vertex, cell) {
13466 self.graph.mark_dirty_many(&[vertex]);
13467 self.mark_topology_edited();
13468 }
13469 Ok(())
13470 }
13471
13472 pub fn declare_fixed_array_formula(
13478 &mut self,
13479 sheet: &str,
13480 row: u32,
13481 col: u32,
13482 rows: u32,
13483 cols: u32,
13484 ) -> Result<(), ExcelError> {
13485 if self.config.family_execution {
13486 return Err(ExcelError::new(ExcelErrorKind::Ref)
13487 .with_message("Fixed arrays require family_execution = false"));
13488 }
13489 if row == 0
13490 || col == 0
13491 || rows == 0
13492 || cols == 0
13493 || row.checked_add(rows - 1).is_none_or(|r| r > 1_048_576)
13494 || col.checked_add(cols - 1).is_none_or(|c| c > 16_384)
13495 || u64::from(rows) * u64::from(cols) > u64::from(self.config.spill.max_spill_cells)
13496 {
13497 return Err(ExcelError::new(ExcelErrorKind::Ref)
13498 .with_message("Invalid or over-cap fixed array extent"));
13499 }
13500 if self.get_staged_formula_text(sheet, row, col).is_some() {
13501 return Err(ExcelError::new(ExcelErrorKind::Ref)
13502 .with_message("Fixed array formula is still staged; build the graph first"));
13503 }
13504 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
13505 ExcelError::new(ExcelErrorKind::Ref).with_message("Unknown fixed array sheet")
13506 })?;
13507 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13508 let vertex = self
13509 .graph
13510 .cell_vertex_mut(&cell)
13511 .filter(|v| self.graph.vertex_has_formula(*v))
13512 .ok_or_else(|| {
13513 ExcelError::new(ExcelErrorKind::Ref)
13514 .with_message("Fixed array anchor must be a formula cell")
13515 })?;
13516 if (rows == 1 && cols == 1 && self.graph.fixed_single_arrays.contains(&vertex))
13517 || self.graph.fixed_array_shapes.get(&vertex) == Some(&(rows, cols))
13518 {
13519 return Ok(());
13520 }
13521 if self.graph.fixed_single_arrays.contains(&vertex)
13522 || self.graph.spill_registry_has_anchor(vertex)
13523 || self.graph.is_current_declared_dynamic_anchor(vertex)
13524 {
13525 return Err(ExcelError::new(ExcelErrorKind::Ref)
13526 .with_message("Array declaration already exists; replace the formula first"));
13527 }
13528 if rows == 1 && cols == 1 {
13529 self.graph.fixed_single_arrays.insert(vertex);
13530 } else {
13531 let mut targets = Vec::with_capacity((rows * cols) as usize);
13532 for r in 0..rows {
13533 for c in 0..cols {
13534 targets.push(self.graph.make_cell_ref(sheet, row + r, col + c));
13535 }
13536 }
13537 self.graph.plan_spill_region(vertex, &targets)?;
13538 self.graph
13540 .commit_spill_region_atomic_with_fault(vertex, targets, vec![], None)?;
13541 self.graph.fixed_array_shapes.insert(vertex, (rows, cols));
13542 }
13543 self.graph.mark_dirty_many(&[vertex]);
13544 self.mark_topology_edited();
13545 Ok(())
13546 }
13547
13548 #[inline]
13549 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
13550 match v {
13551 LiteralValue::Empty => None,
13552 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
13553 other => Some(other),
13554 }
13555 }
13556
13557 fn materialize_temporal_egress(
13558 value: LiteralValue,
13559 class: Option<&formualizer_common::numfmt::FormatClass>,
13560 policy: crate::engine::TemporalEgress,
13561 date_system: crate::engine::DateSystem,
13562 ) -> LiteralValue {
13563 use formualizer_common::numfmt::FormatClass;
13564 if policy == crate::engine::TemporalEgress::Serial {
13565 return value;
13566 }
13567 let LiteralValue::Number(serial) = value else {
13568 return value;
13569 };
13570 match class {
13571 Some(FormatClass::Date) => {
13572 formualizer_common::try_serial_to_date_for(date_system, serial)
13573 .map(LiteralValue::Date)
13574 .unwrap_or(LiteralValue::Number(serial))
13575 }
13576 Some(FormatClass::DateTime) => {
13577 formualizer_common::try_serial_to_datetime_for(date_system, serial)
13578 .map(LiteralValue::DateTime)
13579 .unwrap_or(LiteralValue::Number(serial))
13580 }
13581 Some(FormatClass::Time) => {
13582 let seconds = (serial.rem_euclid(1.0) * 86_400.0).round() as u32 % 86_400;
13583 chrono::NaiveTime::from_num_seconds_from_midnight_opt(seconds, 0)
13584 .map(LiteralValue::Time)
13585 .unwrap_or(LiteralValue::Number(serial))
13586 }
13587 Some(FormatClass::Duration) => {
13588 let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
13589 if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
13590 LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
13591 } else {
13592 LiteralValue::Number(serial)
13593 }
13594 }
13595 _ => LiteralValue::Number(serial),
13596 }
13597 }
13598
13599 pub(crate) fn effective_format_id(
13600 &self,
13601 sheet: &str,
13602 row: u32,
13603 col: u32,
13604 ) -> Option<crate::format::FormatId> {
13605 let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
13606 arrow.format_id(
13607 row.saturating_sub(1) as usize,
13608 col.saturating_sub(1) as usize,
13609 )
13610 });
13611 arrow.or_else(|| {
13612 let sheet_id = self.graph.sheet_id(sheet)?;
13613 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
13614 self.derived_formats.get(&cell)
13615 })
13616 }
13617
13618 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13620 let raw = self.read_cell_value(sheet, row, col)?;
13621 let format = self.effective_format_id(sheet, row, col);
13622 let class = format.and_then(|id| self.format_registry.class(id));
13623 Self::normalize_public_cell_read(Self::materialize_temporal_egress(
13624 raw,
13625 class,
13626 self.config.temporal_egress,
13627 self.config.date_system,
13628 ))
13629 }
13630
13631 pub fn get_range_values(
13633 &self,
13634 sheet: &str,
13635 sr: u32,
13636 sc: u32,
13637 er: u32,
13638 ec: u32,
13639 ) -> Vec<Vec<LiteralValue>> {
13640 let height = er.saturating_sub(sr).saturating_add(1) as usize;
13641 let width = ec.saturating_sub(sc).saturating_add(1) as usize;
13642 let Some(asheet) = self.sheet_store().sheet(sheet) else {
13643 return vec![vec![LiteralValue::Empty; width]; height];
13644 };
13645 let view = asheet.range_view(
13646 sr.saturating_sub(1) as usize,
13647 sc.saturating_sub(1) as usize,
13648 er.saturating_sub(1) as usize,
13649 ec.saturating_sub(1) as usize,
13650 );
13651 let sheet_id = self.graph.sheet_id(sheet);
13652 let derived_formats = &self.derived_formats;
13653 let has_derived_formats =
13654 sheet_id.is_some_and(|sheet_id| derived_formats.any(|cell| cell.sheet_id == sheet_id));
13655 let mut out = Vec::with_capacity(height);
13656 if !asheet.has_formats() && !has_derived_formats {
13657 for rr in 0..height {
13658 let mut row = Vec::with_capacity(width);
13659 for cc in 0..width {
13660 row.push(view.get_cell(rr, cc));
13661 }
13662 out.push(row);
13663 }
13664 return out;
13665 }
13666 let format_registry = &self.format_registry;
13667 for rr in 0..height {
13668 let mut row = Vec::with_capacity(width);
13669 for cc in 0..width {
13670 let raw = view.get_cell(rr, cc);
13671 let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
13672 let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
13673 let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
13674 let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
13675 derived_formats.get(&cell)
13676 });
13677 let class = format.and_then(|id| format_registry.class(id));
13678 row.push(Self::materialize_temporal_egress(
13679 raw,
13680 class,
13681 self.config.temporal_egress,
13682 self.config.date_system,
13683 ));
13684 }
13685 out.push(row);
13686 }
13687 out
13688 }
13689
13690 fn declared_anchor_holds_scalar(&self, anchor: CellRef) -> bool {
13696 let sheet = self.graph.sheet_name(anchor.sheet_id);
13697 if sheet.is_empty() {
13698 return false;
13699 }
13700 matches!(
13701 self.read_cell_value(sheet, anchor.coord.row() + 1, anchor.coord.col() + 1),
13702 Some(value) if !matches!(
13703 value,
13704 LiteralValue::Error(_)
13705 | LiteralValue::Array(_)
13706 | LiteralValue::Pending
13707 | LiteralValue::Empty
13708 )
13709 )
13710 }
13711
13712 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
13713 let asheet = self.sheet_store().sheet(sheet)?;
13714 let r0 = row.saturating_sub(1) as usize;
13715 let c0 = col.saturating_sub(1) as usize;
13716 let v = asheet.get_cell_value(r0, c0);
13717 if matches!(v, LiteralValue::Empty) {
13718 None
13719 } else {
13720 Some(v)
13721 }
13722 }
13723
13724 pub(crate) fn read_range_values(
13726 &self,
13727 sheet: &str,
13728 sr: u32,
13729 sc: u32,
13730 er: u32,
13731 ec: u32,
13732 ) -> RangeView<'_> {
13733 let Some(asheet) = self.sheet_store().sheet(sheet) else {
13734 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
13735 };
13736 if er < sr || ec < sc {
13737 return asheet.range_view(1, 1, 0, 0);
13738 }
13739 let sr0 = sr.saturating_sub(1) as usize;
13740 let sc0 = sc.saturating_sub(1) as usize;
13741 let er0 = er.saturating_sub(1) as usize;
13742 let ec0 = ec.saturating_sub(1) as usize;
13743 asheet.range_view(sr0, sc0, er0, ec0)
13744 }
13745
13746 pub fn get_cell(
13748 &self,
13749 sheet: &str,
13750 row: u32,
13751 col: u32,
13752 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
13753 let v = self.get_cell_value(sheet, row, col);
13754 let sheet_id = self.graph.sheet_id(sheet)?;
13755 let coord = Coord::from_excel(row, col, true, true);
13756 let cell = CellRef::new(sheet_id, coord);
13757 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
13758 let ast = self.graph.get_formula(vid);
13759 Some((ast, v))
13760 } else if v.is_some() || self.graph.had_legacy_cell_vertex(&cell) {
13761 Some((None, v))
13765 } else {
13766 None
13767 }
13768 }
13769
13770 pub fn begin_batch(&mut self) {
13772 self.graph.begin_batch();
13773 }
13774
13775 pub fn end_batch(&mut self) {
13777 self.graph.end_batch();
13778 }
13779
13780 pub fn begin_deferred_dirty(&mut self) {
13789 self.graph.begin_deferred_dirty();
13790 }
13791
13792 pub fn end_deferred_dirty(&mut self) {
13795 let _ = self.graph.end_deferred_dirty();
13796 }
13797
13798 pub fn dirty_propagation_visits(&self) -> u64 {
13802 self.graph.dirty_propagation_visits()
13803 }
13804
13805 #[inline]
13808 fn record_cell_if_changed(
13809 delta: &mut DeltaCollector,
13810 cell: &CellRef,
13811 old: &LiteralValue,
13812 new: &LiteralValue,
13813 ) {
13814 if old != new {
13815 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
13816 }
13817 }
13818
13819 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
13820 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
13821 engine.observe_function_semantic_epoch()?;
13822 engine.resource_checkpoint(1)?;
13824 if !engine.graph.vertex_exists(vertex_id) {
13825 return engine.evaluate_vertex_impl(vertex_id, None);
13826 }
13827 let is_formula = matches!(
13828 engine.graph.get_vertex_kind(vertex_id),
13829 VertexKind::FormulaScalar | VertexKind::FormulaArray
13830 );
13831 if is_formula {
13832 engine.begin_evaluation_request();
13833 #[cfg(any(test, feature = "legacy_oracle"))]
13834 engine.graph.flush_pending_edge_deltas();
13835 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
13836 vertex_id,
13837 )];
13838 engine.evaluate_legacy_target_roots(&roots, None)?;
13839 }
13840 engine.evaluate_vertex_impl(vertex_id, None)
13841 })
13842 }
13843
13844 fn publish_oversized_spill(
13846 &mut self,
13847 vertex_id: VertexId,
13848 error: ExcelError,
13849 mut delta: Option<&mut DeltaCollector>,
13850 ) -> LiteralValue {
13851 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
13852 let anchor = self
13853 .graph
13854 .get_cell_ref(vertex_id)
13855 .expect("cell ref for vertex");
13856 let spill_val = LiteralValue::Error(error);
13857 if let Some(d) = delta {
13858 let old = self
13859 .read_cell_value(
13860 self.graph.sheet_name(anchor.sheet_id),
13861 anchor.coord.row() + 1,
13862 anchor.coord.col() + 1,
13863 )
13864 .unwrap_or(LiteralValue::Empty);
13865 if old != spill_val {
13866 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
13867 }
13868 }
13869 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
13870 if self.config.arrow_storage_enabled
13871 && self.config.delta_overlay_enabled
13872 && self.config.write_formula_overlay_enabled
13873 {
13874 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
13875 self.mirror_value_to_computed_overlay(
13876 &sheet_name,
13877 anchor.coord.row() + 1,
13878 anchor.coord.col() + 1,
13879 &spill_val,
13880 );
13881 }
13882 spill_val
13883 }
13884
13885 fn evaluate_vertex_impl(
13886 &mut self,
13887 vertex_id: VertexId,
13888 delta: Option<&mut DeltaCollector>,
13889 ) -> Result<LiteralValue, ExcelError> {
13890 if !self.graph.vertex_exists(vertex_id) {
13894 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
13895 .with_message(format!("Vertex not found: {vertex_id:?}")));
13896 }
13897 if (self.active_resource_ledger.is_some()
13898 || self.graph.fixed_array_shapes.contains_key(&vertex_id))
13899 && matches!(
13900 self.graph.get_vertex_kind(vertex_id),
13901 VertexKind::FormulaScalar | VertexKind::FormulaArray
13902 )
13903 {
13904 let value = self
13905 .evaluate_vertex_immutable(vertex_id)
13906 .unwrap_or_else(LiteralValue::Error);
13907 let effects = self.plan_vertex_effects(vertex_id, value.clone())?;
13908 self.resource_checkpoint(0)?;
13910 let mut delta = delta;
13911 for effect in &effects {
13912 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
13913 }
13914 return Ok(value);
13915 }
13916
13917 let mut delta = delta;
13918
13919 let kind = self.graph.get_vertex_kind(vertex_id);
13921 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
13922
13923 let view = match kind {
13924 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
13925 if let Some(view) = self.graph.formula_view(vertex_id) {
13926 view
13927 } else {
13928 return Ok(LiteralValue::Number(0.0));
13929 }
13930 }
13931 VertexKind::Empty | VertexKind::Cell => {
13932 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
13933 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
13934 let row = cell_ref.coord.row() + 1;
13935 let col = cell_ref.coord.col() + 1;
13936 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
13937 return Ok(v);
13938 }
13939 }
13940 return Ok(LiteralValue::Number(0.0));
13941 }
13942 VertexKind::NamedScalar => {
13943 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
13944 return Ok(value);
13945 }
13946 VertexKind::NamedArray => {
13947 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
13948 return Ok(value);
13949 }
13950 VertexKind::InfiniteRange
13951 | VertexKind::Range
13952 | VertexKind::External
13953 | VertexKind::Table => {
13954 return Ok(LiteralValue::Number(0.0));
13956 }
13957 };
13958
13959 let sheet_name = self.graph.sheet_name(sheet_id);
13961 let cell_ref = self
13962 .graph
13963 .get_cell_ref(vertex_id)
13964 .expect("cell ref for vertex");
13965 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
13966
13967 let result = interpreter.evaluate_formula_view(
13968 view,
13969 self.graph.data_store(),
13970 self.graph.sheet_reg(),
13971 );
13972
13973 match result {
13975 Ok(cv) => {
13976 let derived_format = cv.format_id();
13977 self.record_derived_format(vertex_id, derived_format);
13978 let oversized_range = if matches!(cv, crate::traits::CalcValue::Range(_))
13979 && self.graph.fixed_single_arrays.contains(&vertex_id)
13980 {
13981 None
13982 } else {
13983 crate::engine::result_finalization::range_spill_error(
13984 &cv,
13985 self.config.spill.max_spill_cells,
13986 )
13987 };
13988 let is_oversized_range = oversized_range.is_some();
13989 let result_literal = if let Some(error) = oversized_range {
13990 drop(cv);
13991 LiteralValue::Error(error)
13992 } else {
13993 self.materialize_formula_result(vertex_id, cv)
13994 };
13995 let output_sheet_name = sheet_name.to_string();
13996 self.write_computed_overlay_format_0based(
13997 &output_sheet_name,
13998 cell_ref.coord.row(),
13999 cell_ref.coord.col(),
14000 derived_format,
14001 );
14002 if is_oversized_range {
14003 let LiteralValue::Error(error) = result_literal else {
14004 unreachable!()
14005 };
14006 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
14007 return Ok(self.publish_oversized_spill(
14008 vertex_id,
14009 error,
14010 delta.as_deref_mut(),
14011 ));
14012 }
14013 match result_literal {
14014 LiteralValue::Array(rows) => {
14015 self.graph
14017 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
14018 let anchor = self
14020 .graph
14021 .get_cell_ref(vertex_id)
14022 .expect("cell ref for vertex");
14023 let sheet_id = anchor.sheet_id;
14024 let h = rows.len() as u32;
14025 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
14026
14027 let spill_cells = (h as u64).saturating_mul(w as u64);
14029 if spill_cells > self.config.spill.max_spill_cells as u64 {
14030 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14031 .with_message("SpillTooLarge")
14032 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14033 expected_rows: h,
14034 expected_cols: w,
14035 });
14036 return Ok(self.publish_oversized_spill(
14037 vertex_id,
14038 spill_err,
14039 delta.as_deref_mut(),
14040 ));
14041 }
14042 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
14046 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
14047 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
14048 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
14049 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14050 .with_message("Spill exceeds sheet bounds")
14051 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14052 expected_rows: h,
14053 expected_cols: w,
14054 });
14055 let spill_val = LiteralValue::Error(spill_err.clone());
14056 if let Some(d) = delta.as_deref_mut() {
14057 let old = self
14058 .read_cell_value(
14059 self.graph.sheet_name(anchor.sheet_id),
14060 anchor.coord.row() + 1,
14061 anchor.coord.col() + 1,
14062 )
14063 .unwrap_or(LiteralValue::Empty);
14064 if old != spill_val {
14065 d.record_cell(
14066 anchor.sheet_id,
14067 anchor.coord.row(),
14068 anchor.coord.col(),
14069 );
14070 }
14071 }
14072 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14073 if self.config.arrow_storage_enabled
14074 && self.config.delta_overlay_enabled
14075 && self.config.write_formula_overlay_enabled
14076 {
14077 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14078 self.mirror_value_to_computed_overlay(
14079 &sheet_name,
14080 anchor.coord.row() + 1,
14081 anchor.coord.col() + 1,
14082 &spill_val,
14083 );
14084 }
14085 return Ok(spill_val);
14086 }
14087 let mut targets = Vec::new();
14088 for r in 0..h {
14089 for c in 0..w {
14090 targets.push(self.graph.make_cell_ref_internal(
14091 sheet_id,
14092 anchor.coord.row() + r,
14093 anchor.coord.col() + c,
14094 ));
14095 }
14096 }
14097
14098 match self.spill_mgr.reserve(
14100 vertex_id,
14101 anchor,
14102 SpillShape { rows: h, cols: w },
14103 SpillMeta {
14104 epoch: self.recalc_epoch,
14105 config: self.config.spill,
14106 },
14107 ) {
14108 Ok(()) => {
14109 if let Err(e) = self.commit_spill_and_mirror(
14113 vertex_id,
14114 &targets,
14115 rows.clone(),
14116 delta.as_deref_mut(),
14117 ) {
14118 if e.kind != ExcelErrorKind::Spill {
14119 return Err(e);
14120 }
14121 self.clear_spill_projection_and_mirror(
14123 vertex_id,
14124 delta.as_deref_mut(),
14125 );
14126 if let Some(d) = delta.as_deref_mut() {
14127 let old = self
14128 .read_cell_value(
14129 self.graph.sheet_name(anchor.sheet_id),
14130 anchor.coord.row() + 1,
14131 anchor.coord.col() + 1,
14132 )
14133 .unwrap_or(LiteralValue::Empty);
14134 let new = LiteralValue::Error(e.clone());
14135 if old != new {
14136 d.record_cell(
14137 anchor.sheet_id,
14138 anchor.coord.row(),
14139 anchor.coord.col(),
14140 );
14141 }
14142 }
14143 let err_val = LiteralValue::Error(e.clone());
14144 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14145 if self.config.arrow_storage_enabled
14146 && self.config.delta_overlay_enabled
14147 && self.config.write_formula_overlay_enabled
14148 {
14149 let sheet_name =
14150 self.graph.sheet_name(anchor.sheet_id).to_string();
14151 self.mirror_value_to_computed_overlay(
14152 &sheet_name,
14153 anchor.coord.row() + 1,
14154 anchor.coord.col() + 1,
14155 &err_val,
14156 );
14157 }
14158 return Ok(err_val);
14159 }
14160 let top_left = rows
14162 .first()
14163 .and_then(|r| r.first())
14164 .cloned()
14165 .unwrap_or(LiteralValue::Empty);
14166 self.graph.update_vertex_value_ref(vertex_id, &top_left);
14167 Ok(top_left)
14168 }
14169 Err(e) => {
14170 self.clear_spill_projection_and_mirror(
14171 vertex_id,
14172 delta.as_deref_mut(),
14173 );
14174 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
14175 .with_message(
14176 e.message.unwrap_or_else(|| "Spill blocked".to_string()),
14177 )
14178 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
14179 expected_rows: h,
14180 expected_cols: w,
14181 });
14182 let spill_val = LiteralValue::Error(spill_err.clone());
14183 if let Some(d) = delta.as_deref_mut() {
14184 let old = self
14185 .read_cell_value(
14186 self.graph.sheet_name(anchor.sheet_id),
14187 anchor.coord.row() + 1,
14188 anchor.coord.col() + 1,
14189 )
14190 .unwrap_or(LiteralValue::Empty);
14191 if old != spill_val {
14192 d.record_cell(
14193 anchor.sheet_id,
14194 anchor.coord.row(),
14195 anchor.coord.col(),
14196 );
14197 }
14198 }
14199 self.graph.update_vertex_value_ref(vertex_id, &spill_val);
14200 if self.config.arrow_storage_enabled
14201 && self.config.delta_overlay_enabled
14202 && self.config.write_formula_overlay_enabled
14203 {
14204 let sheet_name =
14205 self.graph.sheet_name(anchor.sheet_id).to_string();
14206 self.mirror_value_to_computed_overlay(
14207 &sheet_name,
14208 anchor.coord.row() + 1,
14209 anchor.coord.col() + 1,
14210 &spill_val,
14211 );
14212 }
14213 Ok(spill_val)
14214 }
14215 }
14216 }
14217 other => {
14218 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14220 if let Some(d) = delta.as_deref_mut()
14221 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14222 {
14223 if spill_cells.is_empty() {
14224 let old = self
14225 .read_cell_value(
14226 self.graph.sheet_name(anchor.sheet_id),
14227 anchor.coord.row() + 1,
14228 anchor.coord.col() + 1,
14229 )
14230 .unwrap_or(LiteralValue::Empty);
14231 if old != other {
14232 d.record_cell(
14233 anchor.sheet_id,
14234 anchor.coord.row(),
14235 anchor.coord.col(),
14236 );
14237 }
14238 } else {
14239 for cell in spill_cells.iter() {
14240 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14241 let old = self
14242 .get_cell_value(
14243 sheet_name,
14244 cell.coord.row() + 1,
14245 cell.coord.col() + 1,
14246 )
14247 .unwrap_or(LiteralValue::Empty);
14248 let new = if cell.sheet_id == anchor.sheet_id
14249 && cell.coord.row() == anchor.coord.row()
14250 && cell.coord.col() == anchor.coord.col()
14251 {
14252 other.clone()
14253 } else {
14254 LiteralValue::Empty
14255 };
14256 Self::record_cell_if_changed(d, cell, &old, &new);
14257 }
14258 }
14259 }
14260 self.graph.clear_spill_region(vertex_id);
14261 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14262 self.record_structural_change(scope);
14263 }
14264 if self.config.arrow_storage_enabled
14265 && self.config.delta_overlay_enabled
14266 && self.config.write_formula_overlay_enabled
14267 {
14268 let empty = LiteralValue::Empty;
14269 for cell in spill_cells.iter() {
14270 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14271 self.mirror_value_to_computed_overlay(
14272 &sheet_name,
14273 cell.coord.row() + 1,
14274 cell.coord.col() + 1,
14275 &empty,
14276 );
14277 }
14278 }
14279 self.graph.update_vertex_value_ref(vertex_id, &other);
14280 if self.config.arrow_storage_enabled
14282 && self.config.delta_overlay_enabled
14283 && self.config.write_formula_overlay_enabled
14284 {
14285 let anchor = self
14286 .graph
14287 .get_cell_ref(vertex_id)
14288 .expect("cell ref for vertex");
14289 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14290 self.mirror_value_to_computed_overlay(
14291 &sheet_name,
14292 anchor.coord.row() + 1,
14293 anchor.coord.col() + 1,
14294 &other,
14295 );
14296 }
14297 Ok(other)
14298 }
14299 }
14300 }
14301 Err(e) => {
14302 let spill_cells = self.graph.spill_cells_to_clear(vertex_id);
14305 let err_val = LiteralValue::Error(e.clone());
14306 if let Some(d) = delta
14307 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
14308 {
14309 if spill_cells.is_empty() {
14310 let old = self
14311 .read_cell_value(
14312 self.graph.sheet_name(anchor.sheet_id),
14313 anchor.coord.row() + 1,
14314 anchor.coord.col() + 1,
14315 )
14316 .unwrap_or(LiteralValue::Empty);
14317 if old != err_val {
14318 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
14319 }
14320 } else {
14321 for cell in spill_cells.iter() {
14322 let sheet_name = self.graph.sheet_name(cell.sheet_id);
14323 let old = self
14324 .get_cell_value(
14325 sheet_name,
14326 cell.coord.row() + 1,
14327 cell.coord.col() + 1,
14328 )
14329 .unwrap_or(LiteralValue::Empty);
14330 let new = if cell.sheet_id == anchor.sheet_id
14331 && cell.coord.row() == anchor.coord.row()
14332 && cell.coord.col() == anchor.coord.col()
14333 {
14334 err_val.clone()
14335 } else {
14336 LiteralValue::Empty
14337 };
14338 Self::record_cell_if_changed(d, cell, &old, &new);
14339 }
14340 }
14341 }
14342 self.graph.clear_spill_region(vertex_id);
14343 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
14344 self.record_structural_change(scope);
14345 }
14346 if self.config.arrow_storage_enabled
14347 && self.config.delta_overlay_enabled
14348 && self.config.write_formula_overlay_enabled
14349 {
14350 let empty = LiteralValue::Empty;
14351 for cell in spill_cells.iter() {
14352 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
14353 self.mirror_value_to_computed_overlay(
14354 &sheet_name,
14355 cell.coord.row() + 1,
14356 cell.coord.col() + 1,
14357 &empty,
14358 );
14359 }
14360 }
14361 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14362 if self.config.arrow_storage_enabled
14363 && self.config.delta_overlay_enabled
14364 && self.config.write_formula_overlay_enabled
14365 {
14366 let anchor = self
14367 .graph
14368 .get_cell_ref(vertex_id)
14369 .expect("cell ref for vertex");
14370 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
14371 self.mirror_value_to_computed_overlay(
14372 &sheet_name,
14373 anchor.coord.row() + 1,
14374 anchor.coord.col() + 1,
14375 &err_val,
14376 );
14377 }
14378 Ok(err_val)
14379 }
14380 }
14381 }
14382
14383 fn evaluate_named_scalar(
14384 &mut self,
14385 vertex_id: VertexId,
14386 sheet_id: SheetId,
14387 ) -> Result<LiteralValue, ExcelError> {
14388 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14389 ExcelError::new(ExcelErrorKind::Name)
14390 .with_message("Named range metadata missing".to_string())
14391 })?;
14392
14393 match &named_range.definition {
14394 NamedDefinition::Cell(cell_ref) => {
14395 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14396 let row = cell_ref.coord.row() + 1;
14397 let col = cell_ref.coord.col() + 1;
14398
14399 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
14400 && matches!(
14401 self.graph.get_vertex_kind(dep_vertex),
14402 VertexKind::FormulaScalar | VertexKind::FormulaArray
14403 )
14404 {
14405 let value = self.evaluate_vertex(dep_vertex)?;
14407 self.graph.update_vertex_value_ref(vertex_id, &value);
14408 Ok(value)
14409 } else {
14410 let value = self
14411 .get_cell_value(sheet_name, row, col)
14412 .unwrap_or(LiteralValue::Empty);
14413 self.graph.update_vertex_value_ref(vertex_id, &value);
14414 Ok(value)
14415 }
14416 }
14417 NamedDefinition::Literal(v) => {
14418 let out = v.clone();
14419 self.graph.update_vertex_value_ref(vertex_id, &out);
14420 Ok(out)
14421 }
14422 NamedDefinition::Formula { ast, .. } => {
14423 let context_sheet = match named_range.scope {
14424 NameScope::Sheet(id) => id,
14425 NameScope::Workbook => sheet_id,
14426 };
14427 let sheet_name = self.graph.sheet_name(context_sheet);
14428 let cell_ref = self
14429 .graph
14430 .get_cell_ref(vertex_id)
14431 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14432 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
14433 match interpreter.evaluate_ast(ast) {
14434 Ok(cv) => {
14435 let value = cv.into_literal();
14436 match value {
14437 LiteralValue::Array(_) => {
14438 let err = ExcelError::new(ExcelErrorKind::NImpl)
14439 .with_message("Array result in scalar named range".to_string());
14440 let err_val = LiteralValue::Error(err.clone());
14441 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14442 Ok(err_val)
14443 }
14444 other => {
14445 self.graph.update_vertex_value_ref(vertex_id, &other);
14446 Ok(other)
14447 }
14448 }
14449 }
14450 Err(err) => {
14451 let err_val = LiteralValue::Error(err.clone());
14452 self.graph.update_vertex_value_ref(vertex_id, &err_val);
14453 Ok(err_val)
14454 }
14455 }
14456 }
14457 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
14458 .with_message("Range-valued name evaluated as scalar".to_string())),
14459 }
14460 }
14461
14462 fn evaluate_named_array(
14463 &mut self,
14464 vertex_id: VertexId,
14465 sheet_id: SheetId,
14466 ) -> Result<LiteralValue, ExcelError> {
14467 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
14468 ExcelError::new(ExcelErrorKind::Name)
14469 .with_message("Named range metadata missing".to_string())
14470 })?;
14471
14472 let out = match &named_range.definition {
14473 NamedDefinition::Range(range_ref) => {
14474 if range_ref.start.sheet_id != range_ref.end.sheet_id {
14475 return Err(ExcelError::new(ExcelErrorKind::Ref)
14476 .with_message("Named range cannot span sheets".to_string()));
14477 }
14478
14479 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
14480 let sr0 = range_ref.start.coord.row();
14481 let sc0 = range_ref.start.coord.col();
14482 let er0 = range_ref.end.coord.row();
14483 let ec0 = range_ref.end.coord.col();
14484 if sr0 > er0 || sc0 > ec0 {
14485 return Err(ExcelError::new(ExcelErrorKind::Ref)
14486 .with_message("Invalid named range bounds".to_string()));
14487 }
14488
14489 let h = (er0 - sr0 + 1) as usize;
14490 let w = (ec0 - sc0 + 1) as usize;
14491 let cell_count = (h as u64).saturating_mul(w as u64);
14492 if cell_count > self.config.spill.max_spill_cells as u64 {
14493 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
14494 "Named range too large to materialize as an array".to_string(),
14495 ));
14496 }
14497
14498 let mut rows = Vec::with_capacity(h);
14499 for r0 in sr0..=er0 {
14500 let mut row = Vec::with_capacity(w);
14501 for c0 in sc0..=ec0 {
14502 let v = self
14503 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
14504 .unwrap_or(LiteralValue::Empty);
14505 row.push(v);
14506 }
14507 rows.push(row);
14508 }
14509 LiteralValue::Array(rows)
14510 }
14511 NamedDefinition::Cell(cell_ref) => {
14512 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
14513 let row = cell_ref.coord.row() + 1;
14514 let col = cell_ref.coord.col() + 1;
14515 let v = self
14516 .get_cell_value(sheet_name, row, col)
14517 .unwrap_or(LiteralValue::Empty);
14518 LiteralValue::Array(vec![vec![v]])
14519 }
14520 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
14521 NamedDefinition::Formula { ast, .. } => {
14522 let context_sheet = match named_range.scope {
14523 NameScope::Sheet(id) => id,
14524 NameScope::Workbook => sheet_id,
14525 };
14526 let sheet_name = self.graph.sheet_name(context_sheet);
14527 let cell_ref = self
14528 .graph
14529 .get_cell_ref(vertex_id)
14530 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
14531 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
14532 match interpreter.evaluate_ast(ast) {
14533 Ok(cv) => {
14534 let v = cv.into_literal();
14535 match v {
14536 LiteralValue::Array(_) => v,
14537 other => LiteralValue::Array(vec![vec![other]]),
14538 }
14539 }
14540 Err(err) => LiteralValue::Error(err),
14541 }
14542 }
14543 };
14544
14545 self.graph.update_vertex_value_ref(vertex_id, &out);
14546 Ok(out)
14547 }
14548
14549 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
14550 crate::engine::ResourceLedgerError::Exhausted(
14551 formualizer_common::ResourceExhaustionDetail {
14552 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
14553 limit: limit as u64,
14554 observed: limit.saturating_add(1) as u64,
14555 request_id: self
14556 .active_evaluation_resource_request
14557 .as_ref()
14558 .map(|stats| stats.request_id),
14559 },
14560 )
14561 .into_excel_error()
14562 .with_message(format!("{context} did not converge after {limit} replans"))
14563 }
14564
14565 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
14566 matches!(
14567 &error.extra,
14568 formualizer_common::ExcelErrorExtra::PreparationStale {
14569 reason: formualizer_common::PreparationStaleReason::Semantic
14570 | formualizer_common::PreparationStaleReason::Provider
14571 }
14572 )
14573 }
14574
14575 fn prepare_graph_for_routed_evaluation(
14576 &mut self,
14577 targets: &[crate::engine::EvaluationTarget],
14578 options: &crate::engine::TargetEvalOptions<'_>,
14579 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
14580 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
14581 let mut retries = 0usize;
14582 loop {
14583 match self.prepare_graph_for_targets_unobserved(targets, options) {
14584 Err(error)
14585 if Self::transient_target_preparation_stale(&error)
14586 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
14587 {
14588 retries = retries.saturating_add(1);
14589 }
14590 result => return result,
14591 }
14592 }
14593 }
14594
14595 fn evaluate_mixed_targets(
14596 &mut self,
14597 targets: &[crate::engine::EvaluationTarget],
14598 delta: Option<&mut DeltaCollector>,
14599 ) -> Result<EvalResult, ExcelError> {
14600 let _source_cache = self.source_cache_session();
14601 let cancel = self.active_cancel_flag.clone();
14602 let options = crate::engine::TargetEvalOptions {
14603 request_id: self
14604 .active_evaluation_resource_request
14605 .as_ref()
14606 .map(|stats| stats.request_id),
14607 cancel,
14608 deadline: None,
14609 budgets: None,
14610 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
14611 };
14612 self.prepare_and_execute_target_recipe(targets, &options, delta)
14613 }
14614
14615 fn prepare_and_execute_target_recipe(
14616 &mut self,
14617 targets: &[crate::engine::EvaluationTarget],
14618 options: &crate::engine::TargetEvalOptions<'_>,
14619 delta: Option<&mut DeltaCollector>,
14620 ) -> Result<EvalResult, ExcelError> {
14621 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
14622 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
14623 }
14624
14625 fn execute_prepared_target_recipe(
14626 &mut self,
14627 targets: &[crate::engine::EvaluationTarget],
14628 scope: &crate::engine::PrepareScope,
14629 delta: Option<&mut DeltaCollector>,
14630 ) -> Result<EvalResult, ExcelError> {
14631 self.require_unified_authority()?;
14632 if matches!(scope, crate::engine::PrepareScope::Workbook)
14633 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
14634 {
14635 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
14636 }
14637 let mut roots = self.resolve_target_producers(targets)?;
14638 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
14639 let request_id = self
14640 .active_evaluation_resource_request
14641 .as_ref()
14642 .map(|request| request.request_id);
14643 let root_count = roots.len();
14644 let mut widened_roots =
14645 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
14646 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
14647 let sheet_ids = sheets
14648 .iter()
14649 .filter_map(|sheet| self.graph.sheet_id(sheet))
14650 .collect::<FxHashSet<_>>();
14651 for vertex in self.graph.formula_vertices() {
14652 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
14653 widened_roots
14654 .push(crate::engine::target_preparation::TargetProducer::Legacy(
14655 vertex,
14656 ))
14657 .map_err(|_| {
14658 target_root_allocation_error(widened_roots.len() + 1, request_id)
14659 })?;
14660 }
14661 }
14662 roots = widened_roots.into_vec();
14663 }
14664 self.begin_evaluation_request();
14665 #[cfg(any(test, feature = "legacy_oracle"))]
14666 self.graph.flush_pending_edge_deltas();
14667 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
14668 if workbook_scope {
14669 if let Some(delta) = delta {
14670 self.evaluate_all_with_delta_collector(delta)
14671 } else {
14672 self.evaluate_all_legacy_impl()
14673 }
14674 } else {
14675 self.evaluate_legacy_target_roots(&roots, delta)
14676 }
14677 }
14678
14679 fn legacy_coordinate_targets(
14680 &mut self,
14681 targets: &[(&str, u32, u32)],
14682 ) -> Vec<crate::engine::EvaluationTarget> {
14683 targets
14684 .iter()
14685 .map(|(sheet, row, col)| {
14686 self.graph.sheet_id_mut(sheet);
14689 crate::engine::EvaluationTarget::Cell {
14690 sheet: (*sheet).to_string(),
14691 row: *row,
14692 col: *col,
14693 }
14694 })
14695 .collect()
14696 }
14697
14698 pub fn evaluate_targets(
14700 &mut self,
14701 targets: &[crate::engine::EvaluationTarget],
14702 ) -> Result<EvalResult, ExcelError> {
14703 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14704 engine.observe_function_semantic_epoch()?;
14705 engine.validate_deterministic_mode()?;
14706 engine.evaluate_mixed_targets(targets, None)
14707 })
14708 }
14709
14710 pub fn evaluate_targets_with_options(
14712 &mut self,
14713 targets: &[crate::engine::EvaluationTarget],
14714 options: crate::engine::TargetEvalOptions<'_>,
14715 ) -> Result<EvalResult, ExcelError> {
14716 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14717 engine.active_cancel_flag = options.cancel.clone();
14718 engine.active_evaluation_deadline = options.deadline;
14719 let result = (|| {
14720 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
14721 engine.observe_function_semantic_epoch()?;
14722 engine.validate_deterministic_mode()?;
14723 let _source_cache = engine.source_cache_session();
14724 engine.prepare_and_execute_target_recipe(targets, &options, None)
14725 })();
14726 engine.active_cancel_flag = None;
14727 engine.active_evaluation_deadline = None;
14728 result
14729 })
14730 }
14731
14732 pub fn evaluate_targets_with_delta(
14734 &mut self,
14735 targets: &[crate::engine::EvaluationTarget],
14736 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
14737 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
14738 engine.observe_function_semantic_epoch()?;
14739 engine.validate_deterministic_mode()?;
14740 let mut collector = DeltaCollector::new(DeltaMode::Cells);
14741 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
14742 Ok((result, collector.finish_target()))
14743 })
14744 }
14745
14746 pub fn evaluate_until(
14748 &mut self,
14749 targets: &[(&str, u32, u32)],
14750 ) -> Result<EvalResult, ExcelError> {
14751 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
14752 engine.evaluate_until_unobserved(targets)
14753 })
14754 }
14755
14756 fn evaluate_until_unobserved(
14757 &mut self,
14758 targets: &[(&str, u32, u32)],
14759 ) -> Result<EvalResult, ExcelError> {
14760 self.observe_function_semantic_epoch()?;
14761 let targets = self.legacy_coordinate_targets(targets);
14762 self.evaluate_mixed_targets(&targets, None)
14763 }
14764
14765 fn evaluate_until_with_delta_collector(
14766 &mut self,
14767 targets: &[(&str, u32, u32)],
14768 delta: &mut DeltaCollector,
14769 ) -> Result<EvalResult, ExcelError> {
14770 let targets = self.legacy_coordinate_targets(targets);
14771 self.evaluate_mixed_targets(&targets, Some(delta))
14772 }
14773
14774 fn evaluate_legacy_target_roots(
14775 &mut self,
14776 roots: &[crate::engine::target_preparation::TargetProducer],
14777 mut delta: Option<&mut DeltaCollector>,
14778 ) -> Result<EvalResult, ExcelError> {
14779 use crate::engine::target_preparation::TargetProducer;
14780 #[cfg(any(test, feature = "benchmark_internal"))]
14781 {
14782 self.recalc_reuse_probe
14783 .get_mut()
14784 .unwrap()
14785 .legacy_target_requests += 1;
14786 }
14787 let start = crate::instant::FzInstant::now();
14788 let root_vertices = roots
14789 .iter()
14790 .filter_map(|root| match root {
14791 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
14792 TargetProducer::ValueOnly(_) => None,
14793 })
14794 .collect::<Vec<_>>();
14795 let mut computed_vertices = 0usize;
14796 let mut cycle_errors = 0usize;
14797 let mut replans = 0usize;
14798 const MAX_REPLAN: usize = 5;
14799 self.graph.authority_sync();
14800 loop {
14801 let (precedents_to_eval, old_vdeps) = self.demand_subgraph(&root_vertices)?;
14802 if precedents_to_eval.is_empty() {
14803 break;
14804 }
14805 #[cfg(any(test, feature = "benchmark_internal"))]
14806 {
14807 self.recalc_reuse_probe
14808 .get_mut()
14809 .unwrap()
14810 .target_schedule_builds += 1;
14811 }
14812 let schedule = {
14813 self.graph.authority_sync();
14814 let mut ledger = self.active_resource_ledger.take();
14815 let result = self.create_authority_schedule(
14816 &precedents_to_eval,
14817 &old_vdeps,
14818 ledger.as_mut(),
14819 );
14820 self.active_resource_ledger = ledger;
14821 result?
14822 };
14823 self.begin_pass(&schedule);
14824 for (unit_index, &unit) in schedule.units.iter().enumerate() {
14825 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
14826 match unit {
14827 ScheduleUnit::Cycle(index) => {
14828 if self.handle_cycle_unit(
14829 schedule.unit_cycle(index),
14830 delta.as_deref_mut(),
14831 None,
14832 None,
14833 )? > 0
14834 {
14835 cycle_errors = cycle_errors.saturating_add(1);
14836 }
14837 }
14838 ScheduleUnit::Layer(index) => {
14839 let layer = schedule.unit_layer(index);
14840 let evaluated = if let Some(delta) = delta.as_deref_mut() {
14841 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
14842 self.evaluate_layer_parallel_with_delta(layer, delta)?
14843 } else {
14844 self.evaluate_layer_sequential_with_delta(layer, delta)?
14845 }
14846 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
14847 self.evaluate_layer_parallel(layer)?
14848 } else {
14849 self.evaluate_layer_sequential(layer)?
14850 };
14851 computed_vertices = computed_vertices.saturating_add(evaluated);
14852 }
14853 }
14854 if self.stop_after_unit(&schedule, unit_index) {
14855 break;
14856 }
14857 }
14858 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
14859 self.resource_checkpoint(0)?;
14860 if !self.finish_target_pass_dirty(&precedents_to_eval, &changed) {
14861 break;
14862 }
14863 if replans >= MAX_REPLAN {
14864 return Err(self.replan_exhausted_error(
14865 MAX_REPLAN,
14866 "targeted legacy dynamic dependency evaluation",
14867 ));
14868 }
14869 replans = replans.saturating_add(1);
14870 }
14871 self.redirty_for_next_recalc();
14872 Ok(EvalResult {
14873 computed_vertices,
14874 cycle_errors,
14875 elapsed: start.elapsed(),
14876 })
14877 }
14878
14879 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
14881 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
14882 return Err(
14883 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
14884 "compatibility recalculation plans require all staged formulas to be prepared",
14885 ),
14886 );
14887 }
14888 let key = self.recalc_plan_key();
14889 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
14890 vertices.sort_unstable();
14891 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
14892 let schedule = if vertices.is_empty() {
14893 crate::engine::Schedule {
14894 units: Vec::new(),
14895 layers: Vec::new(),
14896 cycles: Vec::new(),
14897 }
14898 } else {
14899 self.create_evaluation_schedule_uncached(&vertices, None)?.0
14900 };
14901 self.validate_recalc_plan_key(&key)?;
14902 Ok(RecalcPlan {
14903 key,
14904 kind: RecalcPlanKind::CompatibilityFull {
14905 schedule,
14906 has_dynamic_refs,
14907 },
14908 })
14909 }
14910
14911 pub fn build_recalc_plan_for_targets(
14913 &mut self,
14914 targets: &[crate::engine::EvaluationTarget],
14915 ) -> Result<RecalcPlan, ExcelError> {
14916 self.build_recalc_plan_for_targets_with_options(
14917 targets,
14918 crate::engine::TargetEvalOptions::default(),
14919 )
14920 }
14921
14922 pub fn build_recalc_plan_for_targets_with_options(
14923 &mut self,
14924 targets: &[crate::engine::EvaluationTarget],
14925 options: crate::engine::TargetEvalOptions<'_>,
14926 ) -> Result<RecalcPlan, ExcelError> {
14927 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14928 engine.observe_function_semantic_epoch()?;
14929 engine.validate_deterministic_mode()?;
14930 let _source_cache = engine.source_cache_session();
14931 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
14932 #[cfg(any(test, feature = "legacy_oracle"))]
14933 engine.graph.flush_pending_edge_deltas();
14934 let topology = if matches!(
14935 preparation.widened_scope,
14936 crate::engine::PrepareScope::Workbook
14937 ) {
14938 RecalcTopology::Workbook
14939 } else {
14940 RecalcTopology::RunLocalRecipe
14941 };
14942 Ok(RecalcPlan {
14943 key: engine.recalc_plan_key(),
14944 kind: RecalcPlanKind::Target {
14945 targets: targets.to_vec(),
14946 scope: preparation.widened_scope,
14947 topology,
14948 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
14949 },
14950 })
14951 })
14952 }
14953
14954 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
14956 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14957 engine.evaluate_recalc_plan_unobserved(plan)
14958 })
14959 }
14960
14961 pub fn evaluate_recalc_plan_with_controls(
14962 &mut self,
14963 plan: &RecalcPlan,
14964 cancel: Option<crate::engine::CancelToken>,
14965 deadline: Option<Instant>,
14966 ) -> Result<EvalResult, ExcelError> {
14967 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
14968 engine.active_cancel_flag = cancel.clone();
14969 engine.active_evaluation_deadline = deadline;
14970 let result = engine.evaluate_recalc_plan_unobserved(plan);
14971 engine.active_cancel_flag = None;
14972 engine.active_evaluation_deadline = None;
14973 result
14974 })
14975 }
14976
14977 fn evaluate_recalc_plan_unobserved(
14978 &mut self,
14979 plan: &RecalcPlan,
14980 ) -> Result<EvalResult, ExcelError> {
14981 #[cfg(any(test, feature = "legacy_oracle"))]
14982 self.graph.flush_pending_edge_deltas();
14983 self.validate_recalc_plan_key(&plan.key)?;
14984 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
14985 self.validate_deterministic_mode()?;
14986
14987 match &plan.kind {
14988 RecalcPlanKind::Target {
14989 targets,
14990 scope,
14991 topology,
14992 dynamic_policy,
14993 } => {
14994 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
14995 debug_assert_eq!(
14996 matches!(topology, RecalcTopology::Workbook),
14997 matches!(scope, crate::engine::PrepareScope::Workbook)
14998 );
14999 let _source_cache = self.source_cache_session();
15000 self.execute_prepared_target_recipe(targets, scope, None)
15001 }
15002 RecalcPlanKind::CompatibilityFull {
15003 schedule,
15004 has_dynamic_refs,
15005 } => {
15006 let _source_cache = self.source_cache_session();
15007 self.begin_evaluation_request();
15008 if *has_dynamic_refs {
15009 self.virtual_dep_fallback_activations =
15010 self.virtual_dep_fallback_activations.saturating_add(1);
15011 return self.evaluate_all_coordinator();
15012 }
15013
15014 let start = crate::instant::FzInstant::now();
15015 let dirty_vertices = self.graph.get_evaluation_vertices();
15016 if dirty_vertices.is_empty() {
15017 return Ok(EvalResult {
15018 computed_vertices: 0,
15019 cycle_errors: 0,
15020 elapsed: start.elapsed(),
15021 });
15022 }
15023
15024 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
15025 let mut computed_vertices = 0;
15026 let mut cycle_errors = 0;
15027 for &unit in &schedule.units {
15028 self.cancellation_checkpoint(
15029 "Evaluation cancelled before recalculation plan schedule unit",
15030 )?;
15031 match unit {
15032 ScheduleUnit::Cycle(i) => {
15033 let stamped = self.handle_cycle_unit(
15034 schedule.unit_cycle(i),
15035 None,
15036 Some(&dirty_set),
15037 None,
15038 )?;
15039 if stamped > 0 {
15040 cycle_errors += 1;
15041 }
15042 }
15043 ScheduleUnit::Layer(i) => {
15044 let work: Vec<VertexId> = schedule
15045 .unit_layer(i)
15046 .vertices
15047 .iter()
15048 .copied()
15049 .filter(|v| dirty_set.contains(v))
15050 .collect();
15051 if work.is_empty() {
15052 continue;
15053 }
15054 let temp_layer = crate::engine::scheduler::Layer::new(work);
15055 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
15056 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
15057 } else {
15058 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
15059 }
15060 }
15061 }
15062 }
15063
15064 self.resource_checkpoint(0)?;
15065 self.graph.clear_dirty_flags(&dirty_vertices);
15066 self.redirty_for_next_recalc();
15067 Ok(EvalResult {
15068 computed_vertices,
15069 cycle_errors,
15070 elapsed: start.elapsed(),
15071 })
15072 }
15073 }
15074 }
15075}
15076
15077impl<R> Engine<R>
15078where
15079 R: EvaluationContext,
15080{
15081 fn require_unified_authority(&mut self) -> Result<(), ExcelError> {
15086 self.graph
15087 .authority()
15088 .map(|_| ())
15089 .map_err(Self::authority_excel_error)
15090 }
15091
15092 fn authority_excel_error(error: crate::engine::authority::store::AuthorityError) -> ExcelError {
15093 use crate::engine::authority::store::AuthorityError;
15094 #[cfg(test)]
15098 if std::env::var_os("FZ_AUTHORITY_DEFERRED_TRACE").is_some() {
15099 eprintln!("M1B_AUTHORITY_ERROR {error:?}");
15100 }
15101 let kind = match error {
15102 AuthorityError::Unsupported { .. } => ExcelErrorKind::NImpl,
15103 _ => ExcelErrorKind::Error,
15104 };
15105 ExcelError::new(kind).with_message(format!("unified_authority: {error:?}"))
15106 }
15107
15108 pub fn evaluate_all_for_snapshot(
15122 &mut self,
15123 cancel: Option<crate::engine::CancelToken>,
15124 ) -> Result<EvalResult, ExcelError> {
15125 struct SnapshotScope<'e, R: EvaluationContext>(&'e mut Engine<R>);
15129 impl<R: EvaluationContext> Drop for SnapshotScope<'_, R> {
15130 fn drop(&mut self) {
15131 self.0.snapshot_evaluation_active = false;
15132 if std::thread::panicking() {
15133 self.0.snapshot_volatile_redirty_pending = true;
15134 }
15135 }
15136 }
15137 self.snapshot_evaluation_active = true;
15138 let result = {
15139 let scope = SnapshotScope(self);
15140 match cancel {
15141 Some(cancel) => scope.0.evaluate_all_cancellable(cancel),
15142 None => scope.0.evaluate_all(),
15143 }
15144 };
15145 if result.is_err() {
15146 self.snapshot_volatile_redirty_pending = false;
15147 self.graph.redirty_volatiles();
15148 }
15149 result
15150 }
15151
15152 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
15154 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
15156 engine.evaluate_all_unobserved()
15157 })
15158 }
15159
15160 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
15161 debug_assert!(
15162 !self.graph.deferred_dirty_active(),
15163 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
15164 was not balanced by end_deferred_dirty"
15165 );
15166 self.observe_function_semantic_epoch()?;
15167 self.lookup_index_cache.reset_counters();
15168 let _source_cache = self.source_cache_session();
15169 self.validate_deterministic_mode()?;
15170 if self.config.defer_graph_building {
15171 self.build_graph_all()?;
15173 }
15174 self.evaluate_all_coordinator()
15175 }
15176
15177 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
15180 self.require_unified_authority()?;
15181 self.begin_evaluation_request();
15182 self.evaluate_all_legacy_impl()
15183 }
15184
15185 fn legacy_pass_run_units(
15191 &mut self,
15192 schedule: &crate::engine::scheduler::Schedule,
15193 ) -> Result<(usize, usize), ExcelError> {
15194 let mut computed_vertices = 0;
15195 let mut cycle_count = 0;
15196 self.begin_pass(schedule);
15197 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15198 match unit {
15199 ScheduleUnit::Cycle(i) => {
15200 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
15201 cycle_count += 1;
15202 }
15203 }
15204 ScheduleUnit::Layer(i) => {
15205 let layer = schedule.unit_layer(i);
15206 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15207 computed_vertices += self.evaluate_layer_parallel(layer)?;
15208 } else {
15209 computed_vertices += self.evaluate_layer_sequential(layer)?;
15210 }
15211 }
15212 }
15213 if self.stop_after_unit(schedule, unit_index) {
15214 break;
15215 }
15216 }
15217 Ok((computed_vertices, cycle_count))
15218 }
15219
15220 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
15227 self.reset_virtual_dep_telemetry_if_disabled();
15228 let _span_eval =
15229 crate::engine::trace::fz_span!(tracing::Level::INFO, "evaluate", "evaluate.legacy");
15230 let start = crate::instant::FzInstant::now();
15231 let mut computed_vertices = 0;
15232 let mut cycle_errors = 0;
15233 let mut replan_iterations = 0;
15234 const MAX_REPLAN: usize = 5;
15235 let mut telemetry = self
15236 .config
15237 .enable_virtual_dep_telemetry
15238 .then(|| self.start_virtual_dep_telemetry());
15239
15240 loop {
15241 let to_evaluate = self.graph.get_evaluation_vertices();
15242 if to_evaluate.is_empty() {
15243 if let Some(t) = telemetry.as_mut()
15244 && t.bailout_reason.is_none()
15245 {
15246 t.bailout_reason = Some("no_work");
15247 }
15248 break;
15249 }
15250
15251 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15252 if let Some(t) = telemetry.as_mut() {
15253 Self::accumulate_schedule_meta(t, &meta);
15254 }
15255
15256 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
15257 computed_vertices += pass_computed;
15258 cycle_errors += pass_cycles;
15259
15260 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15262 if let Some(t) = telemetry.as_mut() {
15263 t.changed_vdeps_total += changed_vertices.len();
15264 }
15265
15266 self.resource_checkpoint(0)?;
15267 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15268 if let Some(t) = telemetry.as_mut() {
15269 t.bailout_reason = Some("converged");
15270 }
15271 break;
15272 }
15273 if replan_iterations >= MAX_REPLAN {
15274 if let Some(mut t) = telemetry.take() {
15275 t.bailout_reason = Some("max_replan");
15276 t.replan_iterations = replan_iterations;
15277 self.last_virtual_dep_telemetry = t;
15278 }
15279 return Err(
15280 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15281 );
15282 }
15283
15284 replan_iterations += 1;
15285 }
15286
15287 if let Some(mut t) = telemetry {
15288 t.replan_iterations = replan_iterations;
15289 self.last_virtual_dep_telemetry = t;
15290 }
15291
15292 self.redirty_for_next_recalc();
15294
15295 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15297
15298 Ok(EvalResult {
15299 computed_vertices,
15300 cycle_errors,
15301 elapsed: start.elapsed(),
15302 })
15303 }
15304
15305 pub fn evaluate_all_with_target_delta(
15306 &mut self,
15307 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
15308 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15309 engine.observe_function_semantic_epoch()?;
15310 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15311 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15312 Ok((result, collector.finish_target()))
15313 })
15314 }
15315
15316 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
15317 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
15318 }
15319
15320 pub fn evaluate_all_with_delta_policy(
15321 &mut self,
15322 policy: EvalDeltaCompatibilityPolicy,
15323 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
15324 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
15325 engine.observe_function_semantic_epoch()?;
15326 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15327 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
15328 Ok((result, collector.finish_with_policy(policy)?))
15329 })
15330 }
15331
15332 fn evaluate_all_with_delta_collector(
15333 &mut self,
15334 delta: &mut DeltaCollector,
15335 ) -> Result<EvalResult, ExcelError> {
15336 let _source_cache = self.source_cache_session();
15337 if self.config.defer_graph_building {
15338 self.build_graph_all()?;
15339 }
15340 self.require_unified_authority()?;
15341 self.begin_evaluation_request();
15342 self.reset_virtual_dep_telemetry_if_disabled();
15343 let _span_eval = crate::engine::trace::fz_span!(
15344 tracing::Level::INFO,
15345 "evaluate",
15346 "evaluate.legacy_delta"
15347 );
15348 let start = crate::instant::FzInstant::now();
15349 let mut computed_vertices = 0;
15350 let mut cycle_errors = 0;
15351
15352 let mut replan_iterations = 0;
15353 const MAX_REPLAN: usize = 5;
15354 let mut telemetry = self
15355 .config
15356 .enable_virtual_dep_telemetry
15357 .then(|| self.start_virtual_dep_telemetry());
15358
15359 loop {
15360 let to_evaluate = self.graph.get_evaluation_vertices();
15361 if to_evaluate.is_empty() {
15362 if let Some(t) = telemetry.as_mut()
15363 && t.bailout_reason.is_none()
15364 {
15365 t.bailout_reason = Some("no_work");
15366 }
15367 break;
15368 }
15369
15370 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
15371 if let Some(t) = telemetry.as_mut() {
15372 Self::accumulate_schedule_meta(t, &meta);
15373 }
15374
15375 self.begin_pass(&schedule);
15376 for (unit_index, &unit) in schedule.units.iter().enumerate() {
15377 match unit {
15378 ScheduleUnit::Cycle(i) => {
15379 if self.handle_cycle_unit(
15380 schedule.unit_cycle(i),
15381 Some(delta),
15382 None,
15383 None,
15384 )? > 0
15385 {
15386 cycle_errors += 1;
15387 }
15388 }
15389 ScheduleUnit::Layer(i) => {
15390 let layer = schedule.unit_layer(i);
15391 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
15392 computed_vertices +=
15393 self.evaluate_layer_parallel_with_delta(layer, delta)?;
15394 } else {
15395 computed_vertices +=
15396 self.evaluate_layer_sequential_with_delta(layer, delta)?;
15397 }
15398 }
15399 }
15400 if self.stop_after_unit(&schedule, unit_index) {
15401 break;
15402 }
15403 }
15404
15405 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
15406 if let Some(t) = telemetry.as_mut() {
15407 t.changed_vdeps_total += changed_vertices.len();
15408 }
15409 self.resource_checkpoint(0)?;
15410 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
15411 if let Some(t) = telemetry.as_mut() {
15412 t.bailout_reason = Some("converged");
15413 }
15414 break;
15415 }
15416 if replan_iterations >= MAX_REPLAN {
15417 if let Some(mut t) = telemetry.take() {
15418 t.bailout_reason = Some("max_replan");
15419 t.replan_iterations = replan_iterations;
15420 self.last_virtual_dep_telemetry = t;
15421 }
15422 return Err(
15423 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
15424 );
15425 }
15426 replan_iterations += 1;
15427 }
15428
15429 if let Some(mut t) = telemetry {
15430 t.replan_iterations = replan_iterations;
15431 self.last_virtual_dep_telemetry = t;
15432 }
15433
15434 self.redirty_for_next_recalc();
15435 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
15436
15437 Ok(EvalResult {
15438 computed_vertices,
15439 cycle_errors,
15440 elapsed: start.elapsed(),
15441 })
15442 }
15443
15444 pub fn evaluate_cell(
15454 &mut self,
15455 sheet: &str,
15456 row: u32,
15457 col: u32,
15458 ) -> Result<Option<LiteralValue>, ExcelError> {
15459 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
15460 engine.evaluate_cell_unobserved(sheet, row, col)
15461 })
15462 }
15463
15464 fn evaluate_cell_unobserved(
15465 &mut self,
15466 sheet: &str,
15467 row: u32,
15468 col: u32,
15469 ) -> Result<Option<LiteralValue>, ExcelError> {
15470 if row == 0 || col == 0 {
15471 return Err(ExcelError::new(ExcelErrorKind::Ref)
15472 .with_message("Row and column must be >= 1".to_string()));
15473 }
15474
15475 let result = self.evaluate_cells(&[(sheet, row, col)])?;
15476
15477 match result.len() {
15478 0 => Ok(None),
15479 1 => {
15480 let v = result.into_iter().next().unwrap();
15481 Ok(v)
15482 }
15483 _ => unreachable!("evaluate_cells returned unexpected length"),
15484 }
15485 }
15486
15487 pub fn evaluate_cells(
15494 &mut self,
15495 targets: &[(&str, u32, u32)],
15496 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15497 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
15498 engine.evaluate_cells_unobserved(targets)
15499 })
15500 }
15501
15502 fn evaluate_cells_unobserved(
15503 &mut self,
15504 targets: &[(&str, u32, u32)],
15505 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15506 self.observe_function_semantic_epoch()?;
15507 debug_assert!(
15508 !self.graph.deferred_dirty_active(),
15509 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
15510 was not balanced by end_deferred_dirty"
15511 );
15512 self.validate_deterministic_mode()?;
15513 if targets.is_empty() {
15514 return Ok(Vec::new());
15515 }
15516 let typed_targets = self.legacy_coordinate_targets(targets);
15517 self.evaluate_mixed_targets(&typed_targets, None)?;
15518 Ok(targets
15519 .iter()
15520 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15521 .collect())
15522 }
15523
15524 pub fn evaluate_cells_cancellable(
15525 &mut self,
15526 targets: &[(&str, u32, u32)],
15527 cancel: crate::engine::CancelToken,
15528 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15529 self.observe_evaluation_resource_request(
15530 EvaluationRequestKind::CellsCancellable,
15531 |engine| {
15532 engine.observe_function_semantic_epoch()?;
15533 engine.active_cancel_flag = Some(cancel.clone());
15534 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
15535 engine.active_cancel_flag = None;
15536 res
15537 },
15538 )
15539 }
15540
15541 fn evaluate_cells_cancellable_impl(
15542 &mut self,
15543 targets: &[(&str, u32, u32)],
15544 cancel_flag: &AtomicBool,
15545 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
15546 self.validate_deterministic_mode()?;
15547 if targets.is_empty() {
15548 return Ok(Vec::new());
15549 }
15550 if cancel_flag.load(Ordering::Relaxed) {
15551 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
15552 .with_message("Evaluation cancelled before target preparation"));
15553 }
15554 let typed_targets = self.legacy_coordinate_targets(targets);
15555 self.evaluate_mixed_targets(&typed_targets, None)?;
15556 Ok(targets
15557 .iter()
15558 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
15559 .collect())
15560 }
15561
15562 pub fn evaluate_cells_with_target_delta(
15563 &mut self,
15564 targets: &[(&str, u32, u32)],
15565 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
15566 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15567 engine.observe_function_semantic_epoch()?;
15568 engine.validate_deterministic_mode()?;
15569 if targets.is_empty() {
15570 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
15571 }
15572 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15573 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
15574 let values = targets
15575 .iter()
15576 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
15577 .collect();
15578 Ok((values, collector.finish_target()))
15579 })
15580 }
15581
15582 pub fn evaluate_cells_with_delta(
15583 &mut self,
15584 targets: &[(&str, u32, u32)],
15585 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15586 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
15587 }
15588
15589 pub fn evaluate_cells_with_delta_policy(
15590 &mut self,
15591 targets: &[(&str, u32, u32)],
15592 policy: EvalDeltaCompatibilityPolicy,
15593 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15594 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
15595 engine.evaluate_cells_with_delta_unobserved(targets, policy)
15596 })
15597 }
15598
15599 fn evaluate_cells_with_delta_unobserved(
15600 &mut self,
15601 targets: &[(&str, u32, u32)],
15602 policy: EvalDeltaCompatibilityPolicy,
15603 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
15604 self.observe_function_semantic_epoch()?;
15605 self.validate_deterministic_mode()?;
15606 if targets.is_empty() {
15607 return Ok((Vec::new(), EvalDelta::default()));
15608 }
15609 let mut collector = DeltaCollector::new(DeltaMode::Cells);
15610 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
15611 let values = targets
15612 .iter()
15613 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
15614 .collect();
15615 Ok((values, collector.finish_with_policy(policy)?))
15616 }
15617
15618 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
15620 if targets.is_empty() {
15621 return Ok(EvalPlan {
15622 total_vertices_to_evaluate: 0,
15623 layers: Vec::new(),
15624 cycles_detected: 0,
15625 dirty_count: 0,
15626 volatile_count: 0,
15627 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15628 estimated_parallel_layers: 0,
15629 target_cells: Vec::new(),
15630 });
15631 }
15632 if self.config.defer_graph_building && self.has_staged_formulas() {
15633 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
15634 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
15635 ));
15636 }
15637
15638 let addresses: Vec<String> = targets
15640 .iter()
15641 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
15642 .collect();
15643
15644 let mut target_addrs = Vec::new();
15646 for (sheet, row, col) in targets {
15647 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
15648 let coord = Coord::from_excel(*row, *col, true, true);
15649 target_addrs.push(CellRef::new(sheet_id, coord));
15650 }
15651 }
15652
15653 let mut target_vertex_ids = Vec::new();
15655 for addr in &target_addrs {
15656 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
15657 target_vertex_ids.push(vertex_id);
15658 }
15659 }
15660
15661 if target_vertex_ids.is_empty() {
15662 return Ok(EvalPlan {
15663 total_vertices_to_evaluate: 0,
15664 layers: Vec::new(),
15665 cycles_detected: 0,
15666 dirty_count: 0,
15667 volatile_count: 0,
15668 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15669 estimated_parallel_layers: 0,
15670 target_cells: addresses,
15671 });
15672 }
15673
15674 let (precedents_to_eval, vdeps) = self.demand_subgraph(&target_vertex_ids)?;
15676
15677 if precedents_to_eval.is_empty() {
15678 return Ok(EvalPlan {
15679 total_vertices_to_evaluate: 0,
15680 layers: Vec::new(),
15681 cycles_detected: 0,
15682 dirty_count: 0,
15683 volatile_count: 0,
15684 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
15685 estimated_parallel_layers: 0,
15686 target_cells: addresses,
15687 });
15688 }
15689
15690 let mut dirty_count = 0;
15692 let mut volatile_count = 0;
15693 for &vertex_id in &precedents_to_eval {
15694 if self.graph.is_dirty(vertex_id) {
15695 dirty_count += 1;
15696 }
15697 if self.graph.is_volatile(vertex_id) {
15698 volatile_count += 1;
15699 }
15700 }
15701
15702 let schedule = self.create_authority_schedule(&precedents_to_eval, &vdeps, None)?;
15704
15705 let mut layers = Vec::new();
15707 let mut estimated_parallel_layers = 0;
15708 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
15709
15710 for layer in &schedule.layers {
15711 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
15712 if parallel_eligible {
15713 estimated_parallel_layers += 1;
15714 }
15715
15716 let sample_cells: Vec<String> = layer
15718 .vertices
15719 .iter()
15720 .take(5)
15721 .filter_map(|&vertex_id| {
15722 self.graph
15723 .get_cell_ref_for_vertex(vertex_id)
15724 .map(|cell_ref| {
15725 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
15726 format!(
15727 "{}!{}{}",
15728 sheet_name,
15729 Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
15730 cell_ref.coord.row() + 1
15731 )
15732 })
15733 })
15734 .collect();
15735
15736 layers.push(LayerInfo {
15737 vertex_count: layer.vertices.len(),
15738 parallel_eligible,
15739 sample_cells,
15740 });
15741 }
15742
15743 Ok(EvalPlan {
15744 total_vertices_to_evaluate: precedents_to_eval.len(),
15745 layers,
15746 cycles_detected: schedule.cycles.len(),
15747 dirty_count,
15748 volatile_count,
15749 parallel_enabled,
15750 estimated_parallel_layers,
15751 target_cells: addresses,
15752 })
15753 }
15754 fn create_evaluation_schedule(
15756 &mut self,
15757 to_evaluate: &[VertexId],
15758 ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
15759 #[cfg(any(test, feature = "benchmark_internal"))]
15760 {
15761 self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
15762 }
15763 #[cfg(any(test, feature = "legacy_oracle"))]
15766 self.graph.flush_pending_edge_deltas();
15767 self.graph.authority_sync();
15769 if self.can_use_static_schedule_cache(to_evaluate) {
15770 let revision = self.schedule_cache_authority_revision();
15772 let current = |e: &CachedScheduleEntry| {
15773 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
15774 };
15775 if !self
15776 .cached_static_schedule
15777 .as_ref()
15778 .is_some_and(|c| current(c) && c.candidate_vertices.equals(to_evaluate))
15779 && let Some(i) = self.recent_schedules.iter().position(|e| {
15780 current(e)
15781 && e.candidate_vertices.len() == to_evaluate.len()
15782 && e.candidate_vertices.equals(to_evaluate)
15783 })
15784 {
15785 let hit = self.recent_schedules.remove(i);
15786 if let Some(previous) = self.cached_static_schedule.replace(hit) {
15787 self.retain_recent_schedule(previous);
15788 }
15789 }
15790 if let Some(cached) = self.cached_static_schedule.as_ref()
15791 && cached.topology_epoch == self.topology_epoch
15792 && cached.authority_revision == self.schedule_cache_authority_revision()
15793 && cached.candidate_vertices.equals(to_evaluate)
15794 {
15795 let meta = ScheduleBuildMeta {
15796 candidate_vertices: to_evaluate.len(),
15797 vdeps_vertices: 0,
15798 vdeps_edges: 0,
15799 builder_elapsed_ms: 0,
15800 used_virtual_schedule: false,
15801 schedule_cache_hit: true,
15802 schedule_cache_eligible: true,
15803 };
15804 #[cfg(any(test, feature = "benchmark_internal"))]
15805 {
15806 let mut probe = self.recalc_reuse_probe.lock().unwrap();
15807 probe.schedule_cache_hits += 1;
15808 probe.schedule_shared_handles += 1;
15809 }
15810 return Ok((
15811 EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
15812 FxHashMap::default(),
15813 meta,
15814 ));
15815 }
15816
15817 let (schedule, vdeps, mut meta) = match self.schedule_from_base(to_evaluate)? {
15818 Some(schedule) => (
15819 schedule,
15820 FxHashMap::default(),
15821 ScheduleBuildMeta {
15822 candidate_vertices: to_evaluate.len(),
15823 vdeps_vertices: 0,
15824 vdeps_edges: 0,
15825 builder_elapsed_ms: 0,
15826 used_virtual_schedule: false,
15827 schedule_cache_hit: false,
15828 schedule_cache_eligible: true,
15829 },
15830 ),
15831 None => self.create_evaluation_schedule_active(to_evaluate)?,
15832 };
15833 meta.schedule_cache_hit = false;
15834 meta.schedule_cache_eligible = true;
15835 #[cfg(any(test, feature = "benchmark_internal"))]
15836 {
15837 self.recalc_reuse_probe
15838 .get_mut()
15839 .unwrap()
15840 .schedule_cache_misses += 1;
15841 }
15842 let schedule = if vdeps.is_empty() {
15843 let mut schedule = schedule;
15846 schedule.units.shrink_to_fit();
15847 for layer in &mut schedule.layers {
15848 layer.vertices.shrink_to_fit();
15849 }
15850 schedule.layers.shrink_to_fit();
15851 for cycle in &mut schedule.cycles {
15852 cycle.shrink_to_fit();
15853 }
15854 schedule.cycles.shrink_to_fit();
15855 let schedule = Arc::new(schedule);
15856 #[cfg(any(test, feature = "benchmark_internal"))]
15857 {
15858 self.recalc_reuse_probe
15859 .get_mut()
15860 .unwrap()
15861 .schedule_shared_handles += 1;
15862 }
15863 let entry = CachedScheduleEntry {
15864 topology_epoch: self.topology_epoch,
15865 authority_revision: self.schedule_cache_authority_revision(),
15866 candidate_vertices: VertexIdRuns::from_slice(to_evaluate),
15867 schedule: Arc::clone(&schedule),
15868 };
15869 if let Some(previous) = self.cached_static_schedule.replace(entry) {
15870 self.retain_recent_schedule(previous);
15871 }
15872 EvaluationSchedule::Shared(schedule)
15873 } else {
15874 EvaluationSchedule::Owned(schedule)
15875 };
15876 return Ok((schedule, vdeps, meta));
15877 }
15878
15879 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_active(to_evaluate)?;
15880 meta.schedule_cache_hit = false;
15881 meta.schedule_cache_eligible = false;
15882 #[cfg(any(test, feature = "benchmark_internal"))]
15883 {
15884 self.recalc_reuse_probe
15885 .get_mut()
15886 .unwrap()
15887 .schedule_cache_ineligible += 1;
15888 }
15889 Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
15890 }
15891
15892 fn schedule_from_base(
15898 &mut self,
15899 to_evaluate: &[VertexId],
15900 ) -> Result<Option<crate::engine::scheduler::Schedule>, ExcelError> {
15901 let revision = self.schedule_cache_authority_revision();
15902 let current = |e: &&CachedScheduleEntry| {
15903 e.topology_epoch == self.topology_epoch && e.authority_revision == revision
15904 };
15905 let Some(base) = [
15908 self.base_schedule.as_ref(),
15909 self.cached_static_schedule.as_ref(),
15910 ]
15911 .into_iter()
15912 .flatten()
15913 .filter(current)
15914 .max_by_key(|e| e.candidate_vertices.len()) else {
15915 return Ok(None);
15916 };
15917 let base_len = base.candidate_vertices.len();
15918 if base_len <= BASE_SCHEDULE_MIN_VERTICES
15919 || to_evaluate.len() < BASE_SCHEDULE_MIN_REQUEST
15920 || to_evaluate.len() > base_len
15921 || base_len > to_evaluate.len().saturating_mul(BASE_SCHEDULE_RATIO)
15922 {
15923 return Ok(None);
15924 }
15925 let mut keep = crate::engine::idset::DenseIdSet::default();
15926 keep.extend(to_evaluate.iter().copied());
15927 let Some((schedule, kept)) = base.schedule.restrict(&keep) else {
15928 return Ok(None);
15929 };
15930 if kept != keep.len() {
15932 return Ok(None);
15933 }
15934 if let Some(ledger) = self.active_resource_ledger.as_mut() {
15935 let layers: usize = schedule
15936 .layers
15937 .iter()
15938 .map(|l| {
15939 l.vertices.capacity() * std::mem::size_of::<VertexId>()
15940 + l.runs.capacity()
15941 * std::mem::size_of::<crate::engine::scheduler::LayerRun>()
15942 })
15943 .sum();
15944 let bytes = (keep.heap_bytes()
15945 + layers
15946 + schedule.layers.capacity()
15947 * std::mem::size_of::<crate::engine::scheduler::Layer>()
15948 + schedule.units.capacity()
15949 * std::mem::size_of::<crate::engine::scheduler::ScheduleUnit>()
15950 + schedule
15951 .cycles
15952 .iter()
15953 .map(|c| c.capacity() * 4)
15954 .sum::<usize>()) as u64;
15955 ledger
15956 .reserve_schedule_discovery(bytes)
15957 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
15958 ledger
15959 .release_scratch(bytes)
15960 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
15961 }
15962 #[cfg(debug_assertions)]
15963 self.debug_check_restricted_schedule(to_evaluate, &schedule);
15964 #[cfg(any(test, feature = "benchmark_internal"))]
15965 {
15966 self.recalc_reuse_probe
15967 .get_mut()
15968 .unwrap()
15969 .schedule_base_restrictions += 1;
15970 }
15971 Ok(Some(schedule))
15972 }
15973
15974 #[cfg(debug_assertions)]
15978 fn debug_check_restricted_schedule(
15979 &self,
15980 to_evaluate: &[VertexId],
15981 schedule: &crate::engine::scheduler::Schedule,
15982 ) {
15983 let mut position: FxHashMap<VertexId, usize> = FxHashMap::default();
15984 for (u, unit) in schedule.units.iter().enumerate() {
15985 let vertices: &[VertexId] = match *unit {
15986 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
15987 &schedule.unit_layer(i).vertices
15988 }
15989 crate::engine::scheduler::ScheduleUnit::Cycle(i) => schedule.unit_cycle(i),
15990 };
15991 for &v in vertices {
15992 assert!(
15993 position.insert(v, u).is_none(),
15994 "vertex twice in a restricted schedule"
15995 );
15996 }
15997 }
15998 assert_eq!(position.len(), to_evaluate.len());
15999 if to_evaluate.len() > 512 {
16000 return;
16001 }
16002 let Ok(store) = self.graph.authority_plan_store() else {
16005 return;
16006 };
16007 let cells: Vec<(VertexId, (u16, u32, u32))> = to_evaluate
16008 .iter()
16009 .filter_map(|&v| self.graph.authority_cell_of_vertex(v).map(|c| (v, c)))
16010 .collect();
16011 for &(v, (sheet, row, col)) in &cells {
16012 let Some(owner) = store.owner_at((sheet, row, col)) else {
16013 continue;
16014 };
16015 let Ok(refined) = store.refine_owner_column(owner, col, row, row, None) else {
16016 continue;
16017 };
16018 for piece in &refined.pieces {
16019 for edge in &refined.edges[piece.edge_start..piece.edge_end] {
16020 let Some(image) = edge.proj.forward(&piece.domain) else {
16021 continue;
16022 };
16023 for &(d, (s2, r2, c2)) in &cells {
16024 if d == v
16025 || edge.proj.sheet != s2
16026 || !(image.r0 <= r2
16027 && r2 <= image.r1
16028 && image.c0 <= c2
16029 && c2 <= image.c1)
16030 {
16031 continue;
16032 }
16033 let (pd, pv) = (position[&d], position[&v]);
16034 let same_ok = pd == pv
16035 && match schedule.units[pv] {
16036 crate::engine::scheduler::ScheduleUnit::Layer(i) => {
16037 schedule.unit_layer(i).sequential
16038 }
16039 crate::engine::scheduler::ScheduleUnit::Cycle(_) => true,
16040 };
16041 assert!(
16042 pd < pv || same_ok,
16043 "restricted schedule orders {v:?} before its precedent {d:?}"
16044 );
16045 }
16046 }
16047 }
16048 }
16049 }
16050
16051 fn maybe_compress_formulas(&mut self) {
16055 if !self.config.formula_compression {
16056 return;
16057 }
16058 let builds = self.graph.authority_host().builds;
16059 if self.compressed_at_build == Some(builds) {
16060 return;
16061 }
16062 self.compressed_at_build = Some(builds);
16063 if self.has_staged_formulas() {
16064 self.graph.virtualize_family_members();
16067 return;
16068 }
16069 let pool = self.thread_pool.clone();
16070 let (_, garbage) = self.graph.compress_family_formulas(pool.as_deref());
16071 self.graph.virtualize_family_members();
16072 match pool {
16076 Some(pool) if garbage.len() >= 1024 => pool.spawn(move || drop(garbage)),
16077 _ => drop(garbage),
16078 }
16079 }
16080
16081 fn create_evaluation_schedule_active(
16082 &mut self,
16083 to_evaluate: &[VertexId],
16084 ) -> Result<ScheduleBuildOutput, ExcelError> {
16085 self.graph.authority_sync();
16086 self.maybe_compress_formulas();
16087 let mut ledger = self.active_resource_ledger.take();
16088 let result = self.create_evaluation_schedule_uncached(to_evaluate, ledger.as_mut());
16089 self.active_resource_ledger = ledger;
16090 result
16091 }
16092
16093 fn create_evaluation_schedule_uncached(
16094 &self,
16095 to_evaluate: &[VertexId],
16096 #[allow(unused_variables)] ledger: Option<&mut ResourceLedger>,
16097 ) -> Result<ScheduleBuildOutput, ExcelError> {
16098 #[cfg(any(test, feature = "benchmark_internal"))]
16099 {
16100 self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
16101 }
16102 let builder = VirtualDepBuilder::new(self);
16103 #[allow(unused_mut)]
16104 let (mut vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
16105 if self.config.enable_virtual_dep_telemetry {
16106 let build_started = crate::instant::FzInstant::now();
16107 let (vdeps, augmented) = builder.build(to_evaluate);
16108 let builder_elapsed_ms = build_started.elapsed().as_millis();
16109 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
16110 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
16111 } else {
16112 let (vdeps, augmented) = builder.build(to_evaluate);
16113 (vdeps, augmented, 0, 0)
16114 };
16115
16116 {
16118 self.freshness_merge_hints(to_evaluate, &mut vdeps);
16119 self.freshness_extent_hints(to_evaluate, &mut vdeps);
16120 }
16121 let mut final_evaluate = to_evaluate.to_vec();
16122 if !augmented.is_empty() {
16123 final_evaluate.extend(augmented);
16124 final_evaluate.sort_unstable();
16125 final_evaluate.dedup();
16126 }
16127
16128 let use_virtual = !vdeps.is_empty();
16129
16130 let schedule = self.create_authority_schedule(&final_evaluate, &vdeps, ledger)?;
16131
16132 let meta = ScheduleBuildMeta {
16133 candidate_vertices: to_evaluate.len(),
16134 vdeps_vertices: vdeps.len(),
16135 vdeps_edges,
16136 builder_elapsed_ms,
16137 used_virtual_schedule: use_virtual,
16138 schedule_cache_hit: false,
16139 schedule_cache_eligible: false,
16140 };
16141
16142 Ok((schedule, vdeps, meta))
16143 }
16144
16145 fn create_authority_schedule(
16146 &self,
16147 candidates: &[VertexId],
16148 vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16149 mut ledger: Option<&mut ResourceLedger>,
16150 ) -> Result<crate::engine::scheduler::Schedule, ExcelError> {
16151 use crate::engine::authority::{
16152 geom::{Cover, Rect},
16153 plan_schedule, planner,
16154 proj::{AxisMap, RefProj},
16155 store::{EdgeKey, Tag},
16156 };
16157 let failure = |message: String| {
16158 ExcelError::new(ExcelErrorKind::Error)
16159 .with_message(format!("unified_authority planner: {message}"))
16160 };
16161 self.cancellation_checkpoint("Evaluation cancelled before authority planning")?;
16162 let store = self
16163 .graph
16164 .authority_plan_store()
16165 .map_err(Self::authority_excel_error)?;
16166 if let [only] = candidates
16169 && vdeps.is_empty()
16170 && self.graph.authority_host().observed(*only).is_none()
16171 && let Some(cell) = self.graph.authority_cell_of_vertex(*only)
16172 && cell.0 != crate::engine::authority::geom::SYMBOL_SHEET
16173 && let Some(single) = planner::plan_single(store, cell)
16174 {
16175 #[cfg(debug_assertions)]
16176 {
16177 let mut cover = Cover::new();
16178 cover.insert_rect(cell.0, &Rect::new(cell.1, cell.2, cell.1, cell.2));
16179 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16180 .expect("general plan of one cell");
16181 assert_eq!(
16182 general.cells.as_slice(),
16183 &[single],
16184 "single-cell plan differs from the planner at {cell:?}"
16185 );
16186 }
16187 let adapted = plan_schedule::schedule(
16188 &[single],
16189 0,
16190 None,
16191 |cell| {
16192 self.graph
16193 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16194 .ok_or_else(|| failure("missing executor identity".to_owned()))
16195 },
16196 |_work| Ok(()),
16197 )
16198 .map_err(|error| match error {
16199 plan_schedule::ScheduleError::Runtime(error) => error,
16200 other => failure(format!("{other:?}")),
16201 })?;
16202 if let Some(ledger) = ledger {
16203 ledger
16204 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16205 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16206 ledger
16207 .release_scratch(adapted.peak_heap_bytes)
16208 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16209 }
16210 return Ok(adapted.schedule);
16211 }
16212 if (2..=planner::SMALL_PLAN_MAX).contains(&candidates.len())
16215 && vdeps.is_empty()
16216 && candidates
16217 .iter()
16218 .all(|&v| self.graph.authority_host().observed(v).is_none())
16219 {
16220 let cells: Option<Vec<(u16, u32, u32)>> = candidates
16221 .iter()
16222 .map(|&v| self.graph.authority_cell_of_vertex(v))
16223 .collect();
16224 if let Some(small) = cells.as_deref().and_then(|c| planner::plan_small(store, c)) {
16225 #[cfg(debug_assertions)]
16226 {
16227 let mut cover = Cover::new();
16228 for c in cells.as_deref().unwrap_or_default() {
16229 cover.insert_rect(c.0, &Rect::new(c.1, c.2, c.1, c.2));
16230 }
16231 let general = planner::plan_with_hints(store, &cover, &[], None, None, None)
16232 .expect("general plan of a small request");
16233 let key = |c: &planner::OrderedCell| (c.sheet, c.row, c.col, c.id, c.owner);
16234 let mut a: Vec<_> = general.cells.iter().map(key).collect();
16235 let mut b: Vec<_> = small.iter().map(key).collect();
16236 a.sort_unstable();
16237 b.sort_unstable();
16238 assert_eq!(a, b, "small plan cells differ from the planner");
16239 assert!(
16240 general.cells.iter().all(|c| c.cycle.is_none()),
16241 "small plan of a cyclic request"
16242 );
16243 }
16244 let adapted = plan_schedule::schedule(
16245 &small,
16246 0,
16247 None,
16248 |cell| {
16249 self.graph
16250 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16251 .ok_or_else(|| failure("missing executor identity".to_owned()))
16252 },
16253 |_work| Ok(()),
16254 )
16255 .map_err(|error| match error {
16256 plan_schedule::ScheduleError::Runtime(error) => error,
16257 other => failure(format!("{other:?}")),
16258 })?;
16259 if let Some(ledger) = ledger {
16260 ledger
16261 .reserve_schedule_discovery(adapted.peak_heap_bytes)
16262 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16263 ledger
16264 .release_scratch(adapted.peak_heap_bytes)
16265 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16266 }
16267 return Ok(adapted.schedule);
16268 }
16269 }
16270 let mut cover = Cover::new();
16275 {
16276 let cell_of = |&id: &VertexId| {
16277 self.graph
16278 .authority_cell_of_vertex(id)
16279 .map(|(sheet, row, col)| (sheet, col, row))
16280 };
16281 let cells: Vec<(u16, u32, u32)> = match self.thread_pool.as_deref() {
16284 Some(pool) if candidates.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16285 use rayon::prelude::*;
16286 pool.install(|| {
16287 let mut cells: Vec<_> = candidates.par_iter().filter_map(cell_of).collect();
16288 cells.par_sort_unstable();
16289 cells
16290 })
16291 }
16292 _ => {
16293 let mut cells: Vec<_> = candidates.iter().filter_map(cell_of).collect();
16294 cells.sort_unstable();
16295 cells
16296 }
16297 };
16298 let mut i = 0;
16299 while i < cells.len() {
16300 let (sheet, col, r0) = cells[i];
16301 let mut r1 = r0;
16302 let mut j = i + 1;
16303 while j < cells.len() && cells[j].0 == sheet && cells[j].1 == col {
16304 if cells[j].2 > r1 + 1 {
16305 break;
16306 }
16307 r1 = r1.max(cells[j].2);
16308 j += 1;
16309 }
16310 cover.insert_rect(sheet, &Rect::new(r0, col, r1, col));
16311 i = j;
16312 }
16313 }
16314 let mut hints = Vec::new();
16315 for (&reader, deps) in vdeps {
16316 let Some(reader) = self.graph.authority_cell_of_vertex(reader) else {
16317 continue;
16318 };
16319 for &dependency in deps {
16320 let Some(dep) = self.graph.authority_cell_of_vertex(dependency) else {
16321 continue;
16322 };
16323 hints.push(planner::PlanHint {
16324 reader: (reader.0, reader.2, reader.1),
16325 edge: EdgeKey {
16326 dep_sheet: reader.0,
16327 tag: Tag::X,
16328 lk: u32::MAX,
16329 proj: RefProj {
16330 sheet: dep.0,
16331 rows: AxisMap::fixed(dep.1, dep.1),
16332 cols: AxisMap::fixed(dep.2, dep.2),
16333 },
16334 },
16335 });
16336 }
16337 }
16338 let host = self.graph.authority_host();
16340 for &id in candidates.iter().filter(|_| host.has_observed()) {
16341 let Some(reads) = host.observed(id) else {
16342 continue;
16343 };
16344 let Some(reader) = self.graph.authority_cell_of_vertex(id) else {
16345 continue;
16346 };
16347 for &(sheet, r0, c0, r1, c1) in reads {
16348 hints.push(planner::PlanHint {
16349 reader: (reader.0, reader.2, reader.1),
16350 edge: EdgeKey {
16351 dep_sheet: reader.0,
16352 tag: Tag::X,
16353 lk: u32::MAX,
16354 proj: RefProj {
16355 sheet,
16356 rows: AxisMap::fixed(r0, r1),
16357 cols: AxisMap::fixed(c0, c1),
16358 },
16359 },
16360 });
16361 }
16362 }
16363 hints.sort_unstable_by_key(|hint| hint.reader);
16364 let checkpoint = ledger
16365 .as_ref()
16366 .map_or(0, |ledger| ledger.scratch_checkpoint());
16367 let scratch_limit = ledger
16368 .as_ref()
16369 .and_then(|ledger| ledger.schedule_discovery_limit())
16370 .map(|limit| limit.saturating_sub(checkpoint));
16371 let ordered = planner::plan_with_hints(store, &cover, &hints, scratch_limit, None, None)
16372 .map_err(|error| failure(format!("{error:?}")))?;
16373 let adapted = plan_schedule::schedule(
16377 &ordered.cells,
16378 ordered.heap_bytes(),
16379 scratch_limit,
16380 |cell| {
16381 self.graph
16382 .authority_vertex_of_formula(cell.id, (cell.sheet, cell.row, cell.col))
16383 .ok_or_else(|| failure("missing executor identity".to_owned()))
16384 },
16385 |_work| {
16386 self.cancellation_checkpoint("Evaluation cancelled during authority planning")?;
16387 if let Some(ledger) = ledger.as_deref_mut() {
16388 ledger
16389 .checkpoint_deadline()
16390 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16391 }
16392 Ok(())
16393 },
16394 )
16395 .map_err(|error| match error {
16396 plan_schedule::ScheduleError::Runtime(error) => error,
16397 other => failure(format!("{other:?}")),
16398 })?;
16399 if let Some(ledger) = ledger {
16400 let peak = ordered.peak_heap_bytes.max(adapted.peak_heap_bytes);
16401 ledger
16402 .reserve_schedule_discovery(peak)
16403 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16404 ledger
16405 .release_scratch(peak)
16406 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
16407 }
16408 match self.thread_pool.as_deref() {
16410 Some(pool) if ordered.cells.len() >= PARALLEL_SCHEDULE_MIN_CANDIDATES => {
16411 pool.spawn(move || drop(ordered))
16412 }
16413 _ => drop(ordered),
16414 }
16415 Ok(adapted.schedule)
16416 }
16417
16418 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
16424 {
16425 let host = self.graph.authority_host();
16429 !to_evaluate.is_empty()
16430 && !self.freshness_has_hints()
16431 && to_evaluate
16432 .iter()
16433 .all(|&v| !self.graph.is_dynamic(v) || host.observed(v).is_some())
16434 }
16435 }
16436
16437 fn schedule_cache_authority_revision(&self) -> (u64, u64) {
16438 {
16439 let host = self.graph.authority_host();
16442 (host.revision(), host.rev_dyn())
16443 }
16444 }
16445
16446 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
16447 VirtualDepTelemetry {
16448 fallback_mode_activations: self.virtual_dep_fallback_activations,
16449 ..VirtualDepTelemetry::default()
16450 }
16451 }
16452
16453 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
16454 telemetry.candidate_vertices_total += meta.candidate_vertices;
16455 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
16456 telemetry.vdeps_edges_total += meta.vdeps_edges;
16457 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
16458 if meta.schedule_cache_eligible {
16459 if meta.schedule_cache_hit {
16460 telemetry.schedule_cache_hits += 1;
16461 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
16462 } else {
16463 telemetry.schedule_cache_misses += 1;
16464 }
16465 }
16466 if meta.used_virtual_schedule {
16467 telemetry.schedule_virtual_passes += 1;
16468 } else {
16469 telemetry.schedule_static_passes += 1;
16470 }
16471 }
16472
16473 fn finish_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16478 self.finish_pass_dirty_scoped(to_evaluate, changed, true)
16479 }
16480
16481 fn finish_target_pass_dirty(&mut self, to_evaluate: &[VertexId], changed: &[VertexId]) -> bool {
16484 self.finish_pass_dirty_scoped(to_evaluate, changed, false)
16485 }
16486
16487 fn finish_pass_dirty_scoped(
16488 &mut self,
16489 to_evaluate: &[VertexId],
16490 changed: &[VertexId],
16491 #[allow(unused_variables)] whole_workbook: bool,
16492 ) -> bool {
16493 self.freshness_finish_pass(to_evaluate, changed, whole_workbook)
16494 }
16495
16496 fn begin_pass(
16498 &mut self,
16499 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16500 ) {
16501 self.freshness_begin_pass(schedule);
16502 }
16503
16504 fn stop_after_unit(
16506 &mut self,
16507 #[allow(unused_variables)] schedule: &crate::engine::scheduler::Schedule,
16508 #[allow(unused_variables)] index: usize,
16509 ) -> bool {
16510 self.freshness_stop_after_unit(schedule, index)
16511 }
16512
16513 fn changed_virtual_dep_vertices(
16514 &mut self,
16515 to_evaluate: &[VertexId],
16516 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
16517 ) -> Vec<VertexId> {
16518 #[cfg(test)]
16519 if self.force_virtual_dep_changes_remaining_for_test > 0
16520 && let Some(vertex) = to_evaluate.first().copied()
16521 {
16522 self.force_virtual_dep_changes_remaining_for_test -= 1;
16523 return vec![vertex];
16524 }
16525 if self.freshness_armed() {
16528 return Vec::new();
16529 }
16530 if !to_evaluate
16531 .iter()
16532 .copied()
16533 .any(|v| self.graph.is_dynamic(v))
16534 {
16535 return Vec::new();
16536 }
16537
16538 let builder = VirtualDepBuilder::new(self);
16539 let (new_vdeps, _) = builder.build(to_evaluate);
16540
16541 let mut candidates = FxHashSet::default();
16542 candidates.extend(old_vdeps.keys().copied());
16543 candidates.extend(new_vdeps.keys().copied());
16544
16545 let mut changed = Vec::new();
16546 for v in candidates {
16547 if old_vdeps.get(&v) != new_vdeps.get(&v) {
16548 changed.push(v);
16549 }
16550 }
16551 changed
16552 }
16553
16554 #[allow(clippy::type_complexity)]
16560 fn demand_subgraph(
16561 &self,
16562 targets: &[VertexId],
16563 ) -> Result<
16564 (
16565 Vec<VertexId>,
16566 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16567 ),
16568 ExcelError,
16569 > {
16570 self.authority_demand_subgraph(targets)
16571 }
16572
16573 #[allow(clippy::type_complexity)]
16579 fn authority_demand_subgraph(
16580 &self,
16581 targets: &[VertexId],
16582 ) -> Result<
16583 (
16584 Vec<VertexId>,
16585 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
16586 ),
16587 ExcelError,
16588 > {
16589 use crate::engine::authority::geom::{Cell, Rect, SYMBOL_SHEET};
16590 use crate::engine::authority::store::TagFilter;
16591 use rustc_hash::{FxHashMap, FxHashSet};
16592 let store = self
16593 .graph
16594 .authority_plan_store()
16595 .map_err(Self::authority_excel_error)?;
16596 let ids = store.ids();
16597 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
16598 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default();
16599 let mut visited: FxHashSet<Cell> = FxHashSet::default();
16600 let mut stack: Vec<(Cell, u32)> = Vec::new();
16603 let push_formulas = |stack: &mut Vec<(Cell, u32)>,
16605 visited: &FxHashSet<Cell>,
16606 sheet: u16,
16607 rect: Rect| {
16608 for col in rect.c0..=rect.c1 {
16609 ids.visit_runs_in(sheet, col, rect.r0, rect.r1, &mut |h| {
16610 let run = ids.run(h);
16611 let r0 = run.row_start.max(rect.r0);
16612 let r1 = (run.row_start + run.len - 1).min(rect.r1);
16613 for row in r0..=r1 {
16614 if !visited.contains(&(sheet, row, col)) {
16615 stack.push(((sheet, row, col), run.first_id + (row - run.row_start)));
16616 }
16617 }
16618 });
16619 }
16620 };
16621 for &v in targets {
16622 if let Some(table) = self.graph.table_by_vertex(v) {
16623 let (s, e) = (table.range.start, table.range.end);
16627 push_formulas(
16628 &mut stack,
16629 &visited,
16630 s.sheet_id,
16631 Rect::new(s.coord.row(), s.coord.col(), e.coord.row(), e.coord.col()),
16632 );
16633 } else if let Some(cell) = self.graph.authority_cell_of_vertex(v) {
16634 stack.push((cell, crate::engine::authority::identity::NO_VID));
16635 }
16636 }
16637 let mut hits = Vec::new();
16638 #[cfg(any(test, feature = "benchmark_internal"))]
16639 let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges, mut probe_dynamic) =
16640 (0, 0, 0, 0);
16641 while let Some((cell, id)) = stack.pop() {
16642 if !visited.insert(cell) {
16643 continue;
16644 }
16645 let Some(v) = self.graph.authority_vertex_of_formula(id, cell) else {
16646 continue;
16647 };
16648 if !self.graph.vertex_exists(v) {
16649 continue;
16650 }
16651 #[cfg(any(test, feature = "benchmark_internal"))]
16652 {
16653 probe_vertices += 1;
16654 }
16655 match self.graph.get_vertex_kind(v) {
16656 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
16657 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
16658 to_evaluate.insert(v);
16659 } else {
16660 #[cfg(any(test, feature = "benchmark_internal"))]
16661 {
16662 probe_clean_formulas += 1;
16663 }
16664 }
16665 }
16666 VertexKind::NamedScalar | VertexKind::NamedArray => {
16667 to_evaluate.insert(v);
16668 }
16669 _ => {}
16670 }
16671 hits.clear();
16672 store.direct_precedents(cell, TagFilter::All, &mut hits);
16673 #[cfg(any(test, feature = "benchmark_internal"))]
16674 {
16675 probe_edges += hits.len();
16676 }
16677 for &(_, sheet, rect) in &hits {
16678 if sheet != SYMBOL_SHEET {
16681 for anchor in self
16682 .graph
16683 .spill_anchors_in_region(sheet, rect.r0, rect.c0, rect.r1, rect.c1)
16684 {
16685 if anchor != v && self.graph.is_dirty(anchor) {
16686 vdeps.entry(v).or_default().push(anchor);
16687 if let Some(c) = self.graph.authority_cell_of_vertex(anchor) {
16688 stack.push((c, crate::engine::authority::identity::NO_VID));
16689 }
16690 }
16691 }
16692 }
16693 if sheet == SYMBOL_SHEET {
16694 stack.extend((rect.r0..=rect.r1).map(|slot| {
16695 (
16696 (SYMBOL_SHEET, slot, 0),
16697 crate::engine::authority::identity::NO_VID,
16698 )
16699 }));
16700 continue;
16701 }
16702 push_formulas(&mut stack, &visited, sheet, rect);
16703 }
16704 if self.graph.is_dynamic(v) {
16705 #[cfg(any(test, feature = "benchmark_internal"))]
16706 {
16707 probe_dynamic += 1;
16708 }
16709 if let Some(reads) = self.graph.authority_host().observed(v) {
16711 for &(sheet, r0, c0, r1, c1) in reads {
16712 push_formulas(&mut stack, &visited, sheet, Rect::new(r0, c0, r1, c1));
16713 }
16714 }
16715 let (vdeps_map, _) = VirtualDepBuilder::new(self).build(&[v]);
16716 let hinted = self.freshness_hints(v).unwrap_or(&[]);
16719 if let Some(deps) = vdeps_map
16720 .get(&v)
16721 .map(|deps| deps.iter().chain(hinted))
16722 .or(Some([].iter().chain(hinted)))
16723 {
16724 for &u in deps {
16725 vdeps.entry(v).or_default().push(u);
16726 if let Some(c) = self.graph.authority_cell_of_vertex(u) {
16727 stack.push((c, crate::engine::authority::identity::NO_VID));
16728 }
16729 }
16730 }
16731 }
16732 }
16733 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
16734 result.sort_unstable();
16735 for deps in vdeps.values_mut() {
16736 deps.sort_unstable();
16737 deps.dedup();
16738 }
16739 #[cfg(any(test, feature = "benchmark_internal"))]
16740 {
16741 let mut probe = self.recalc_reuse_probe.lock().unwrap();
16742 probe.demand_builds += 1;
16743 probe.demand_vertices += probe_vertices;
16744 probe.demand_clean_formulas += probe_clean_formulas;
16745 probe.demand_explicit_edges += probe_edges;
16746 probe.demand_virtual_builder_calls += probe_dynamic;
16747 }
16748 Ok((result, vdeps))
16749 }
16750
16751 fn col_to_letters(col: u32) -> String {
16753 col_letters_from_1based(col).expect("column index must be >= 1")
16754 }
16755
16756 pub fn evaluate_all_cancellable(
16758 &mut self,
16759 cancel: crate::engine::CancelToken,
16760 ) -> Result<EvalResult, ExcelError> {
16761 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
16762 engine.observe_function_semantic_epoch()?;
16763 engine.active_cancel_flag = Some(cancel.clone());
16764 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
16765 engine.active_cancel_flag = None;
16766 res
16767 })
16768 }
16769
16770 fn evaluate_all_cancellable_impl(
16771 &mut self,
16772 cancel_flag: &AtomicBool,
16773 ) -> Result<EvalResult, ExcelError> {
16774 let _source_cache = self.source_cache_session();
16775 self.validate_deterministic_mode()?;
16776 if self.config.defer_graph_building {
16777 self.build_graph_all()?;
16778 }
16779 if cancel_flag.load(Ordering::Relaxed) {
16780 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16781 .with_message("Evaluation cancelled before scheduling".to_string()));
16782 }
16783 self.require_unified_authority()?;
16784 self.begin_evaluation_request();
16785 self.reset_virtual_dep_telemetry_if_disabled();
16786 let start = crate::instant::FzInstant::now();
16787 let mut computed_vertices = 0;
16788 let mut cycle_errors = 0;
16789
16790 let mut replan_iterations = 0;
16791 const MAX_REPLAN: usize = 5;
16792 let mut telemetry = self
16793 .config
16794 .enable_virtual_dep_telemetry
16795 .then(|| self.start_virtual_dep_telemetry());
16796
16797 loop {
16798 if cancel_flag.load(Ordering::Relaxed) {
16799 if let Some(mut t) = telemetry {
16800 t.bailout_reason = Some("cancelled");
16801 t.replan_iterations = replan_iterations;
16802 self.last_virtual_dep_telemetry = t;
16803 }
16804 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16805 .with_message("Evaluation cancelled before scheduling".to_string()));
16806 }
16807
16808 let to_evaluate = self.graph.get_evaluation_vertices();
16809 if to_evaluate.is_empty() {
16810 if let Some(t) = telemetry.as_mut()
16811 && t.bailout_reason.is_none()
16812 {
16813 t.bailout_reason = Some("no_work");
16814 }
16815 break;
16816 }
16817
16818 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
16819 if let Some(t) = telemetry.as_mut() {
16820 Self::accumulate_schedule_meta(t, &meta);
16821 }
16822
16823 self.begin_pass(&schedule);
16826 for (unit_index, &unit) in schedule.units.iter().enumerate() {
16827 match unit {
16828 ScheduleUnit::Cycle(i) => {
16829 if cancel_flag.load(Ordering::Relaxed) {
16831 if let Some(mut t) = telemetry {
16832 t.bailout_reason = Some("cancelled");
16833 t.replan_iterations = replan_iterations;
16834 self.last_virtual_dep_telemetry = t;
16835 }
16836 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
16837 "Evaluation cancelled during cycle handling".to_string(),
16838 ));
16839 }
16840
16841 if self.handle_cycle_unit(
16842 schedule.unit_cycle(i),
16843 None,
16844 None,
16845 Some(cancel_flag),
16846 )? > 0
16847 {
16848 cycle_errors += 1;
16849 }
16850 }
16851 ScheduleUnit::Layer(i) => {
16852 let layer = schedule.unit_layer(i);
16853 if cancel_flag.load(Ordering::Relaxed) {
16855 if let Some(mut t) = telemetry {
16856 t.bailout_reason = Some("cancelled");
16857 t.replan_iterations = replan_iterations;
16858 self.last_virtual_dep_telemetry = t;
16859 }
16860 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16861 .with_message("Evaluation cancelled between layers".to_string()));
16862 }
16863
16864 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
16866 computed_vertices +=
16867 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
16868 } else {
16869 computed_vertices +=
16870 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
16871 }
16872 }
16873 }
16874 if self.stop_after_unit(&schedule, unit_index) {
16875 break;
16876 }
16877 }
16878
16879 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
16880 if let Some(t) = telemetry.as_mut() {
16881 t.changed_vdeps_total += changed_vertices.len();
16882 }
16883 self.resource_checkpoint(0)?;
16884 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
16885 if let Some(t) = telemetry.as_mut() {
16886 t.bailout_reason = Some("converged");
16887 }
16888 break;
16889 }
16890 if replan_iterations >= MAX_REPLAN {
16891 if let Some(mut t) = telemetry.take() {
16892 t.bailout_reason = Some("max_replan");
16893 t.replan_iterations = replan_iterations;
16894 self.last_virtual_dep_telemetry = t;
16895 }
16896 return Err(
16897 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
16898 );
16899 }
16900 replan_iterations += 1;
16901 }
16902
16903 if let Some(mut t) = telemetry {
16904 t.replan_iterations = replan_iterations;
16905 self.last_virtual_dep_telemetry = t;
16906 }
16907
16908 self.redirty_for_next_recalc();
16910 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
16911
16912 Ok(EvalResult {
16913 computed_vertices,
16914 cycle_errors,
16915 elapsed: start.elapsed(),
16916 })
16917 }
16918
16919 pub fn evaluate_until_cancellable(
16921 &mut self,
16922 targets: &[&str],
16923 cancel: crate::engine::CancelToken,
16924 ) -> Result<EvalResult, ExcelError> {
16925 self.observe_evaluation_resource_request(
16926 EvaluationRequestKind::TargetedCancellable,
16927 |engine| {
16928 engine.observe_function_semantic_epoch()?;
16929 engine.active_cancel_flag = Some(cancel.clone());
16930 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
16931 engine.active_cancel_flag = None;
16932 res
16933 },
16934 )
16935 }
16936
16937 fn evaluate_until_cancellable_impl(
16938 &mut self,
16939 targets: &[&str],
16940 cancel_flag: &AtomicBool,
16941 ) -> Result<EvalResult, ExcelError> {
16942 if cancel_flag.load(Ordering::Relaxed) {
16943 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
16944 .with_message("Evaluation cancelled before target preparation"));
16945 }
16946 let mut typed_targets = Vec::with_capacity(targets.len());
16947 for target in targets {
16948 let (sheet, row, col) = self.parse_a1_notation(target)?;
16949 self.graph.sheet_id_mut(&sheet);
16950 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
16951 }
16952 self.evaluate_mixed_targets(&typed_targets, None)
16953 }
16954
16955 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
16956 let mut quoted = false;
16957 let mut separator = None;
16958 let bytes = address.as_bytes();
16959 let mut index = 0usize;
16960 while index < bytes.len() {
16961 match bytes[index] {
16962 b'\'' => {
16963 if quoted && bytes.get(index + 1) == Some(&b'\'') {
16964 index = index.saturating_add(1);
16965 } else {
16966 quoted = !quoted;
16967 }
16968 }
16969 b'!' if !quoted => separator = Some(index),
16970 _ => {}
16971 }
16972 index = index.saturating_add(1);
16973 }
16974 if quoted {
16975 return Err(ExcelError::new(ExcelErrorKind::Ref)
16976 .with_message(format!("Invalid quoted sheet reference `{address}`")));
16977 }
16978 let (sheet, cell_part) = match separator {
16979 Some(separator) => {
16980 let raw_sheet = &address[..separator];
16981 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
16982 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
16983 } else {
16984 raw_sheet.to_string()
16985 };
16986 (sheet, &address[separator + 1..])
16987 }
16988 None => (self.default_sheet_name().to_string(), address),
16989 };
16990
16991 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
16992 ExcelError::new(ExcelErrorKind::Ref)
16993 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
16994 })?;
16995
16996 Ok((sheet, row, col))
16997 }
16998
16999 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
17001 use formualizer_parse::parser::ASTNodeType;
17002 match &ast.node_type {
17003 ASTNodeType::Function { name, args, .. } => {
17004 if let Some(func) = self
17005 .get_function("", name)
17006 .or_else(|| crate::function_registry::get("", name))
17007 && func.caps().contains(crate::function::FnCaps::VOLATILE)
17008 {
17009 return true;
17010 }
17011 args.iter()
17012 .any(|arg| self.is_ast_volatile_with_provider(arg))
17013 }
17014 ASTNodeType::BinaryOp { left, right, .. } => {
17015 self.is_ast_volatile_with_provider(left)
17016 || self.is_ast_volatile_with_provider(right)
17017 }
17018 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
17019 ASTNodeType::Array(rows) => rows.iter().any(|row| {
17020 row.iter()
17021 .any(|cell| self.is_ast_volatile_with_provider(cell))
17022 }),
17023 _ => false,
17024 }
17025 }
17026
17027 fn evaluate_layer_sequential(
17029 &mut self,
17030 layer: &super::scheduler::Layer,
17031 ) -> Result<usize, ExcelError> {
17032 self.resource_checkpoint(layer.vertices.len() as u64)?;
17033 self.evaluate_layer_sequential_effects(layer)
17034 }
17035
17036 fn update_vertex_value_with_delta(
17037 &mut self,
17038 vertex_id: VertexId,
17039 new_value: LiteralValue,
17040 delta: &mut DeltaCollector,
17041 ) {
17042 if delta.mode != DeltaMode::Off
17043 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
17044 {
17045 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17046 let old = self
17047 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
17048 .unwrap_or(LiteralValue::Empty);
17049 if old != new_value {
17050 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17051 }
17052 }
17053 self.graph.update_vertex_value_ref(vertex_id, &new_value);
17054 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
17055 }
17056
17057 fn evaluate_layer_sequential_with_delta(
17058 &mut self,
17059 layer: &super::scheduler::Layer,
17060 delta: &mut DeltaCollector,
17061 ) -> Result<usize, ExcelError> {
17062 self.resource_checkpoint(layer.vertices.len() as u64)?;
17063 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
17064 }
17065
17066 fn evaluate_layer_sequential_cancellable(
17068 &mut self,
17069 layer: &super::scheduler::Layer,
17070 cancel_flag: &AtomicBool,
17071 ) -> Result<usize, ExcelError> {
17072 self.resource_checkpoint(layer.vertices.len() as u64)?;
17073 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
17074 }
17075
17076 fn evaluate_layer_sequential_cancellable_demand_driven(
17078 &mut self,
17079 layer: &super::scheduler::Layer,
17080 cancel_flag: &AtomicBool,
17081 ) -> Result<usize, ExcelError> {
17082 self.resource_checkpoint(layer.vertices.len() as u64)?;
17083 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
17084 }
17085
17086 fn evaluate_layer_parallel(
17098 &mut self,
17099 layer: &super::scheduler::Layer,
17100 ) -> Result<usize, ExcelError> {
17101 if layer.sequential {
17102 return self.evaluate_layer_sequential(layer);
17103 }
17104 self.resource_checkpoint(layer.vertices.len() as u64)?;
17105 let len = layer.vertices.len();
17106 let buffered = buffer_layer_writes(layer);
17107 let (probe, worth) = (PARALLEL_LAYER_PROBE, PARALLEL_LAYER_WORTH);
17108 let start = crate::instant::FzInstant::now();
17109 let mut pos = 0usize;
17110 let mut step = 1usize;
17111 while pos < len {
17112 if pos > 0 {
17113 let elapsed = start.elapsed();
17114 let per_vertex = elapsed.as_nanos() / pos as u128 + 1;
17116 let rest_estimate = per_vertex * (len - pos) as u128;
17117 if len - pos >= 2 && (elapsed >= probe || rest_estimate >= worth.as_nanos()) {
17118 let rest = layer.sub_layer(pos, len);
17119 let min_chunk = if per_vertex >= EXPENSIVE_VERTEX_NS {
17122 1
17123 } else {
17124 8
17125 };
17126 return Ok(pos + self.evaluate_layer_parallel_effects(&rest, min_chunk)?);
17127 }
17128 let fit = (probe.saturating_sub(elapsed).as_nanos() / per_vertex) as usize + 1;
17131 step = step.saturating_mul(2).min(fit);
17132 }
17133 let end = (pos + step).min(len);
17134 let slice = layer.sub_layer(pos, end);
17135 pos += self.evaluate_layer_units_until(
17138 &slice,
17139 None,
17140 None,
17141 None,
17142 buffered,
17143 Some(start + probe),
17144 )?;
17145 }
17146 Ok(len)
17147 }
17148
17149 fn evaluate_layer_parallel_with_delta(
17150 &mut self,
17151 layer: &super::scheduler::Layer,
17152 delta: &mut DeltaCollector,
17153 ) -> Result<usize, ExcelError> {
17154 if layer.sequential {
17155 return self.evaluate_layer_sequential_with_delta(layer, delta);
17156 }
17157 self.resource_checkpoint(layer.vertices.len() as u64)?;
17158 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
17159 }
17160
17161 fn evaluate_layer_parallel_cancellable(
17163 &mut self,
17164 layer: &super::scheduler::Layer,
17165 cancel_flag: &AtomicBool,
17166 ) -> Result<usize, ExcelError> {
17167 if layer.sequential {
17168 return self.evaluate_layer_sequential_cancellable(layer, cancel_flag);
17169 }
17170 self.resource_checkpoint(layer.vertices.len() as u64)?;
17171 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
17172 }
17173
17174 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17176 if !self.graph.vertex_exists(vertex_id) {
17178 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17179 .with_message(format!("Vertex not found: {vertex_id:?}")));
17180 }
17181
17182 let kind = self.graph.get_vertex_kind(vertex_id);
17184 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17185
17186 let view = match kind {
17187 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17188 if let Some(view) = self.graph.formula_view(vertex_id) {
17189 view
17190 } else {
17191 return Ok(LiteralValue::Number(0.0));
17192 }
17193 }
17194 VertexKind::Empty | VertexKind::Cell => {
17195 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17196 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17197 let row = cell_ref.coord.row() + 1;
17198 let col = cell_ref.coord.col() + 1;
17199 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17200 return Ok(v);
17201 }
17202 }
17203 return Ok(LiteralValue::Number(0.0));
17204 }
17205 VertexKind::NamedScalar => {
17206 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17207 ExcelError::new(ExcelErrorKind::Name)
17208 .with_message("Named range metadata missing".to_string())
17209 })?;
17210
17211 return match &named_range.definition {
17212 NamedDefinition::Cell(cell_ref) => {
17213 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17214 Ok(self
17215 .get_cell_value(
17216 sheet_name,
17217 cell_ref.coord.row() + 1,
17218 cell_ref.coord.col() + 1,
17219 )
17220 .unwrap_or(LiteralValue::Empty))
17221 }
17222 NamedDefinition::Literal(v) => Ok(v.clone()),
17223 NamedDefinition::Formula { ast, .. } => {
17224 let context_sheet = match named_range.scope {
17225 NameScope::Sheet(id) => id,
17226 NameScope::Workbook => sheet_id,
17227 };
17228 let sheet_name = self.graph.sheet_name(context_sheet);
17229 let cell_ref = self
17230 .graph
17231 .get_cell_ref(vertex_id)
17232 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17233 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17234 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
17235 }
17236 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17237 .with_message("Range-valued name evaluated as scalar".to_string())),
17238 };
17239 }
17240 VertexKind::NamedArray => {
17241 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17242 ExcelError::new(ExcelErrorKind::Name)
17243 .with_message("Named range metadata missing".to_string())
17244 })?;
17245
17246 return match &named_range.definition {
17247 NamedDefinition::Range(range_ref) => {
17248 if range_ref.start.sheet_id != range_ref.end.sheet_id {
17249 return Err(ExcelError::new(ExcelErrorKind::Ref)
17250 .with_message("Named range cannot span sheets".to_string()));
17251 }
17252 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17253 let sr0 = range_ref.start.coord.row();
17254 let sc0 = range_ref.start.coord.col();
17255 let er0 = range_ref.end.coord.row();
17256 let ec0 = range_ref.end.coord.col();
17257 if sr0 > er0 || sc0 > ec0 {
17258 return Err(ExcelError::new(ExcelErrorKind::Ref)
17259 .with_message("Invalid named range bounds".to_string()));
17260 }
17261
17262 let h = (er0 - sr0 + 1) as usize;
17263 let w = (ec0 - sc0 + 1) as usize;
17264 let cell_count = (h as u64).saturating_mul(w as u64);
17265 if cell_count > self.config.spill.max_spill_cells as u64 {
17266 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17267 "Named range too large to materialize as an array".to_string(),
17268 ));
17269 }
17270
17271 let sheet_id = range_ref.start.sheet_id;
17274 let asheet = self.arrow_sheets.sheet(sheet_name);
17275 let mut rows = Vec::with_capacity(h);
17276 for r0 in sr0..=er0 {
17277 let mut row = Vec::with_capacity(w);
17278 for c0 in sc0..=ec0 {
17279 row.push(
17280 self.read_cell_formatted_in(sheet_id, asheet, r0 + 1, c0 + 1)
17281 .0,
17282 );
17283 }
17284 rows.push(row);
17285 }
17286 Ok(LiteralValue::Array(rows))
17287 }
17288 NamedDefinition::Cell(cell_ref) => {
17289 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17290 let row = cell_ref.coord.row() + 1;
17291 let col = cell_ref.coord.col() + 1;
17292 let v = self
17293 .get_cell_value(sheet_name, row, col)
17294 .unwrap_or(LiteralValue::Empty);
17295 Ok(LiteralValue::Array(vec![vec![v]]))
17296 }
17297 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
17298 NamedDefinition::Formula { ast, .. } => {
17299 let context_sheet = match named_range.scope {
17300 NameScope::Sheet(id) => id,
17301 NameScope::Workbook => sheet_id,
17302 };
17303 let sheet_name = self.graph.sheet_name(context_sheet);
17304 let cell_ref = self
17305 .graph
17306 .get_cell_ref(vertex_id)
17307 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17308 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17309 match interpreter.evaluate_ast(ast) {
17310 Ok(cv) => {
17311 let v = cv.into_literal();
17312 match v {
17313 LiteralValue::Array(_) => Ok(v),
17314 other => Ok(LiteralValue::Array(vec![vec![other]])),
17315 }
17316 }
17317 Err(err) => Ok(LiteralValue::Error(err)),
17318 }
17319 }
17320 };
17321 }
17322 VertexKind::InfiniteRange
17323 | VertexKind::Range
17324 | VertexKind::External
17325 | VertexKind::Table => {
17326 return Ok(LiteralValue::Number(0.0));
17328 }
17329 };
17330
17331 let sheet_name = self.graph.sheet_name(sheet_id);
17333 let cell_ref = self
17334 .graph
17335 .get_cell_ref(vertex_id)
17336 .expect("cell ref for vertex");
17337 if let Some(result) =
17338 self.freshness_evaluate_recorded(vertex_id, sheet_name, cell_ref, view)
17339 {
17340 return result;
17341 }
17342 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17343
17344 interpreter
17345 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
17346 .map(|cv| {
17347 let format = cv.format_id();
17348 self.record_derived_format(vertex_id, format);
17349 self.materialize_formula_result(vertex_id, cv)
17350 })
17351 .or_else(|error| self.fit_formula_error(vertex_id, error))
17352 }
17353
17354 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
17356 self.thread_pool.as_ref()
17357 }
17358}
17359
17360#[derive(Default)]
17361struct RowBoundsCache {
17362 snapshot: u64,
17363 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
17365}
17366
17367impl RowBoundsCache {
17368 fn new(snapshot: u64) -> Self {
17369 Self {
17370 snapshot,
17371 map: Default::default(),
17372 }
17373 }
17374 fn get_row_bounds(
17375 &self,
17376 sheet_id: SheetId,
17377 col_idx: usize,
17378 snapshot: u64,
17379 ) -> Option<(Option<u32>, Option<u32>)> {
17380 if self.snapshot != snapshot {
17381 return None;
17382 }
17383 self.map.get(&(sheet_id as u32, col_idx)).copied()
17384 }
17385 fn put_row_bounds(
17386 &mut self,
17387 sheet_id: SheetId,
17388 col_idx: usize,
17389 snapshot: u64,
17390 bounds: (Option<u32>, Option<u32>),
17391 ) {
17392 if self.snapshot != snapshot {
17393 self.snapshot = snapshot;
17394 self.map.clear();
17395 }
17396 self.map.insert((sheet_id as u32, col_idx), bounds);
17397 }
17398}
17399
17400struct UsedAxisBoundsCache {
17401 snapshot: u64,
17402 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17403 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
17404 #[cfg(test)]
17405 row_hits: std::sync::atomic::AtomicUsize,
17406 #[cfg(test)]
17407 row_misses: std::sync::atomic::AtomicUsize,
17408 #[cfg(test)]
17409 col_hits: std::sync::atomic::AtomicUsize,
17410 #[cfg(test)]
17411 col_misses: std::sync::atomic::AtomicUsize,
17412}
17413
17414impl UsedAxisBoundsCache {
17415 fn new(snapshot: u64) -> Self {
17416 Self {
17417 snapshot,
17418 row_bounds_by_col_span: Default::default(),
17419 col_bounds_by_row_span: Default::default(),
17420 #[cfg(test)]
17421 row_hits: std::sync::atomic::AtomicUsize::new(0),
17422 #[cfg(test)]
17423 row_misses: std::sync::atomic::AtomicUsize::new(0),
17424 #[cfg(test)]
17425 col_hits: std::sync::atomic::AtomicUsize::new(0),
17426 #[cfg(test)]
17427 col_misses: std::sync::atomic::AtomicUsize::new(0),
17428 }
17429 }
17430
17431 fn reset_for_snapshot(&mut self, snapshot: u64) {
17432 if self.snapshot != snapshot {
17433 self.snapshot = snapshot;
17434 self.row_bounds_by_col_span.clear();
17435 self.col_bounds_by_row_span.clear();
17436 }
17437 }
17438
17439 fn get_row_bounds(
17440 &self,
17441 sheet_id: SheetId,
17442 start_col: u32,
17443 end_col: u32,
17444 snapshot: u64,
17445 ) -> Option<Option<(u32, u32)>> {
17446 if self.snapshot != snapshot {
17447 return None;
17448 }
17449 let cached = self
17450 .row_bounds_by_col_span
17451 .get(&(sheet_id, start_col, end_col))
17452 .copied();
17453 #[cfg(test)]
17454 if cached.is_some() {
17455 self.row_hits.fetch_add(1, Ordering::Relaxed);
17456 }
17457 cached
17458 }
17459
17460 fn put_row_bounds(
17461 &mut self,
17462 sheet_id: SheetId,
17463 start_col: u32,
17464 end_col: u32,
17465 snapshot: u64,
17466 bounds: Option<(u32, u32)>,
17467 ) {
17468 self.reset_for_snapshot(snapshot);
17469 self.row_bounds_by_col_span
17470 .insert((sheet_id, start_col, end_col), bounds);
17471 #[cfg(test)]
17472 self.row_misses.fetch_add(1, Ordering::Relaxed);
17473 }
17474
17475 fn get_col_bounds(
17476 &self,
17477 sheet_id: SheetId,
17478 start_row: u32,
17479 end_row: u32,
17480 snapshot: u64,
17481 ) -> Option<Option<(u32, u32)>> {
17482 if self.snapshot != snapshot {
17483 return None;
17484 }
17485 let cached = self
17486 .col_bounds_by_row_span
17487 .get(&(sheet_id, start_row, end_row))
17488 .copied();
17489 #[cfg(test)]
17490 if cached.is_some() {
17491 self.col_hits.fetch_add(1, Ordering::Relaxed);
17492 }
17493 cached
17494 }
17495
17496 fn put_col_bounds(
17497 &mut self,
17498 sheet_id: SheetId,
17499 start_row: u32,
17500 end_row: u32,
17501 snapshot: u64,
17502 bounds: Option<(u32, u32)>,
17503 ) {
17504 self.reset_for_snapshot(snapshot);
17505 self.col_bounds_by_row_span
17506 .insert((sheet_id, start_row, end_row), bounds);
17507 #[cfg(test)]
17508 self.col_misses.fetch_add(1, Ordering::Relaxed);
17509 }
17510}
17511
17512#[derive(Default)]
17514pub struct ShimSpillManager {
17515 region_locks: RegionLockManager,
17516 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
17517}
17518
17519impl ShimSpillManager {
17520 pub(crate) fn reserve(
17521 &mut self,
17522 owner: VertexId,
17523 anchor_cell: CellRef,
17524 shape: SpillShape,
17525 _meta: SpillMeta,
17526 ) -> Result<(), ExcelError> {
17527 let region = crate::engine::spill::Region {
17529 sheet_id: anchor_cell.sheet_id as u32,
17530 row_start: anchor_cell.coord.row(),
17531 row_end: anchor_cell
17532 .coord
17533 .row()
17534 .saturating_add(shape.rows)
17535 .saturating_sub(1),
17536 col_start: anchor_cell.coord.col(),
17537 col_end: anchor_cell
17538 .coord
17539 .col()
17540 .saturating_add(shape.cols)
17541 .saturating_sub(1),
17542 };
17543 match self.region_locks.reserve(region, owner) {
17544 Ok(id) => {
17545 if id != 0 {
17546 self.active_locks.insert(owner, id);
17547 }
17548 Ok(())
17549 }
17550 Err(e) => Err(e),
17551 }
17552 }
17553
17554 pub(crate) fn release_owner(&mut self, owner: VertexId) {
17560 if let Some(id) = self.active_locks.remove(&owner) {
17561 self.region_locks.release(id);
17562 }
17563 }
17564
17565 pub(crate) fn commit_array_with_value_probe<F>(
17566 &mut self,
17567 graph: &mut DependencyGraph,
17568 anchor_vertex: VertexId,
17569 targets: &[CellRef],
17570 rows: Vec<Vec<LiteralValue>>,
17571 mut value_probe: F,
17572 ) -> Result<(), ExcelError>
17573 where
17574 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
17575 {
17576 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
17577
17578 let plan_res = graph.plan_spill_region(anchor_vertex, targets);
17582 if let Err(e) = plan_res {
17583 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17584 self.region_locks.release(id);
17585 }
17586 return Err(e);
17587 }
17588
17589 if !graph.value_cache_enabled() {
17590 let (expected_rows, expected_cols) = if targets.is_empty() {
17592 (0u32, 0u32)
17593 } else {
17594 let mut min_r = u32::MAX;
17595 let mut max_r = 0u32;
17596 let mut min_c = u32::MAX;
17597 let mut max_c = 0u32;
17598 for cell in targets {
17599 let r = cell.coord.row();
17600 let c = cell.coord.col();
17601 min_r = min_r.min(r);
17602 max_r = max_r.max(r);
17603 min_c = min_c.min(c);
17604 max_c = max_c.max(c);
17605 }
17606 (
17607 max_r.saturating_sub(min_r).saturating_add(1),
17608 max_c.saturating_sub(min_c).saturating_add(1),
17609 )
17610 };
17611
17612 let anchor_cell = graph
17613 .get_cell_ref(anchor_vertex)
17614 .expect("anchor cell ref for spill commit");
17615
17616 for cell in targets {
17617 if *cell == anchor_cell {
17619 continue;
17620 }
17621 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
17623 continue;
17624 }
17625 if let Some(vid) = graph.get_vertex_id_for_address(cell)
17627 && vid != anchor_vertex
17628 {
17629 match graph.get_vertex_kind(vid) {
17630 crate::engine::vertex::VertexKind::FormulaScalar
17631 | crate::engine::vertex::VertexKind::FormulaArray => {
17632 continue;
17634 }
17635 _ => {}
17636 }
17637 }
17638
17639 if let Some(v) = value_probe(graph, cell)
17640 && !matches!(v, LiteralValue::Empty)
17641 {
17642 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17643 self.region_locks.release(id);
17644 }
17645 return Err(ExcelError::new(ExcelErrorKind::Spill)
17646 .with_message("BlockedByValue")
17647 .with_extra(ExcelErrorExtra::Spill {
17648 expected_rows,
17649 expected_cols,
17650 }));
17651 }
17652 }
17653 }
17654
17655 let commit_res = graph.commit_spill_region_atomic_with_fault(
17656 anchor_vertex,
17657 targets.to_vec(),
17658 rows,
17659 None,
17660 );
17661 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17662 self.region_locks.release(id);
17663 }
17664 commit_res.map(|_| ())
17665 }
17666
17667 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
17669 &mut self,
17670 engine: &mut Engine<R>,
17671 anchor_vertex: VertexId,
17672 targets: &[CellRef],
17673 rows: Vec<Vec<LiteralValue>>,
17674 ) -> Result<(), ExcelError> {
17675 if let Err(error) = engine.guard_pending_spill_commit(anchor_vertex, targets) {
17676 self.release_owner(anchor_vertex);
17677 return Err(error);
17678 }
17679 let plan_res = engine.graph.plan_spill_region(anchor_vertex, targets);
17681 if let Err(e) = plan_res {
17682 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17683 self.region_locks.release(id);
17684 }
17685 return Err(e);
17686 }
17687
17688 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
17689 anchor_vertex,
17690 targets.to_vec(),
17691 rows.clone(),
17692 None,
17693 );
17694 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
17695 self.region_locks.release(id);
17696 }
17697 commit_res.map(|_| ())?;
17698 engine
17699 .blocked_pending_spills
17700 .retain(|entry| entry.0 != anchor_vertex);
17701 engine.spill_blocker_waiters.forget(anchor_vertex);
17702
17703 if engine.config.arrow_storage_enabled
17705 && engine.config.delta_overlay_enabled
17706 && engine.config.write_formula_overlay_enabled
17707 {
17708 for (idx, cell) in targets.iter().enumerate() {
17710 let (r_off, c_off) = {
17711 if rows.is_empty() || rows[0].is_empty() {
17712 (0usize, 0usize)
17713 } else {
17714 let width = rows[0].len();
17715 (idx / width, idx % width)
17716 }
17717 };
17718 let v = rows
17719 .get(r_off)
17720 .and_then(|r| r.get(c_off))
17721 .cloned()
17722 .unwrap_or(LiteralValue::Empty);
17723 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
17724 engine.mirror_value_to_computed_overlay(
17725 &sheet_name,
17726 cell.coord.row() + 1,
17727 cell.coord.col() + 1,
17728 &v,
17729 );
17730 }
17731 }
17732 Ok(())
17733 }
17734}
17735
17736impl<R> Engine<R>
17737where
17738 R: EvaluationContext,
17739{
17740 fn resolve_shared_ref(
17741 &self,
17742 reference: &ReferenceType,
17743 current_sheet: &str,
17744 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
17745 use formualizer_common::{
17746 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
17747 SheetRef as SharedRef,
17748 };
17749
17750 let sr = match reference {
17752 ReferenceType::Cell {
17753 sheet,
17754 row,
17755 col,
17756 row_abs,
17757 col_abs,
17758 } => {
17759 let row0 = row
17760 .checked_sub(1)
17761 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17762 let col0 = col
17763 .checked_sub(1)
17764 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17765 let sheet_loc = match sheet.as_deref() {
17766 Some(name) => SheetLocator::from_name(name),
17767 None => SheetLocator::Current,
17768 };
17769 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
17770 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
17771 }
17772 ReferenceType::Range {
17773 sheet,
17774 start_row,
17775 start_col,
17776 end_row,
17777 end_col,
17778 start_row_abs,
17779 start_col_abs,
17780 end_row_abs,
17781 end_col_abs,
17782 } => {
17783 let sheet_loc = match sheet.as_deref() {
17784 Some(name) => SheetLocator::from_name(name),
17785 None => SheetLocator::Current,
17786 };
17787 let sr = start_row
17788 .map(|r| {
17789 r.checked_sub(1)
17790 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17791 })
17792 .transpose()?;
17793 let sc = start_col
17794 .map(|c| {
17795 c.checked_sub(1)
17796 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17797 })
17798 .transpose()?;
17799 let er = end_row
17800 .map(|r| {
17801 r.checked_sub(1)
17802 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17803 })
17804 .transpose()?;
17805 let ec = end_col
17806 .map(|c| {
17807 c.checked_sub(1)
17808 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17809 })
17810 .transpose()?;
17811 let range = SharedRangeRef::from_parts(
17812 sheet_loc,
17813 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
17814 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
17815 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
17816 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
17817 )
17818 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
17819 SharedRef::Range(range)
17820 }
17821 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
17822 };
17823
17824 let current_id = self
17825 .graph
17826 .sheet_id(current_sheet)
17827 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
17828
17829 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
17830 match loc {
17831 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
17832 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
17833 SheetLocator::Name(name) => {
17834 let n = name.as_ref();
17835 self.graph
17836 .sheet_id(n)
17837 .map(SheetLocator::Id)
17838 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
17839 }
17840 }
17841 };
17842
17843 match sr {
17844 SharedRef::Cell(cell) => {
17845 let owned = cell.into_owned();
17846 let sheet = resolve_loc(owned.sheet)?;
17847 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
17848 }
17849 SharedRef::Range(range) => {
17850 let owned = range.into_owned();
17851 let sheet = resolve_loc(owned.sheet)?;
17852 Ok(SharedRef::Range(SharedRangeRef {
17853 sheet,
17854 start_row: owned.start_row,
17855 start_col: owned.start_col,
17856 end_row: owned.end_row,
17857 end_col: owned.end_col,
17858 }))
17859 }
17860 }
17861 }
17862}
17863
17864impl<R> crate::traits::ReferenceResolver for Engine<R>
17867where
17868 R: EvaluationContext,
17869{
17870 fn resolve_cell_reference(
17871 &self,
17872 sheet: Option<&str>,
17873 row: u32,
17874 col: u32,
17875 ) -> Result<LiteralValue, ExcelError> {
17876 let Some(sheet_name) = sheet else {
17887 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
17888 "Unqualified cell reference resolved without sheet context".to_string(),
17889 ));
17890 };
17891 if let Some(v) = self.get_cell_value(sheet_name, row, col) {
17894 Ok(v)
17895 } else {
17896 Ok(LiteralValue::Number(0.0))
17898 }
17899 }
17900}
17901
17902impl<R> crate::traits::RangeResolver for Engine<R>
17903where
17904 R: EvaluationContext,
17905{
17906 fn resolve_range_reference(
17907 &self,
17908 sheet: Option<&str>,
17909 sr: Option<u32>,
17910 sc: Option<u32>,
17911 er: Option<u32>,
17912 ec: Option<u32>,
17913 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
17914 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
17917 }
17918}
17919
17920impl<R> crate::traits::NamedRangeResolver for Engine<R>
17921where
17922 R: EvaluationContext,
17923{
17924 fn resolve_named_range_reference(
17925 &self,
17926 name: &str,
17927 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
17928 self.resolver.resolve_named_range_reference(name)
17929 }
17930}
17931
17932impl<R> crate::traits::TableResolver for Engine<R>
17933where
17934 R: EvaluationContext,
17935{
17936 fn resolve_table_reference(
17937 &self,
17938 tref: &formualizer_parse::parser::TableReference,
17939 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
17940 self.resolver.resolve_table_reference(tref)
17941 }
17942}
17943
17944impl<R> crate::traits::SourceResolver for Engine<R>
17945where
17946 R: EvaluationContext,
17947{
17948 fn source_scalar_version(&self, name: &str) -> Option<u64> {
17949 self.resolver.source_scalar_version(name)
17950 }
17951
17952 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
17953 self.resolver.resolve_source_scalar(name)
17954 }
17955
17956 fn source_table_version(&self, name: &str) -> Option<u64> {
17957 self.resolver.source_table_version(name)
17958 }
17959
17960 fn resolve_source_table(
17961 &self,
17962 name: &str,
17963 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
17964 self.resolver.resolve_source_table(name)
17965 }
17966}
17967
17968impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
17970
17971impl<R> crate::traits::FunctionProvider for Engine<R>
17973where
17974 R: EvaluationContext,
17975{
17976 fn planning_semantic_revision(&self) -> Option<u64> {
17977 self.resolver.planning_semantic_revision()
17978 }
17979
17980 fn get_function(
17981 &self,
17982 prefix: &str,
17983 name: &str,
17984 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
17985 self.resolver.get_function(prefix, name)
17986 }
17987
17988 fn get_function_for_planning(
17989 &self,
17990 prefix: &str,
17991 name: &str,
17992 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
17993 self.resolver.get_function_for_planning(prefix, name)
17994 }
17995}
17996
17997impl<R> Engine<R>
17998where
17999 R: EvaluationContext,
18000{
18001 pub(crate) fn semantic_used_rows_for_columns(
18009 &self,
18010 sheet: &str,
18011 start_col: u32,
18012 end_col: u32,
18013 ) -> Option<(u32, u32)> {
18014 let arrow_bounds = self
18015 .sheet_store()
18016 .sheet(sheet)
18017 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18018 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18019 Self::union_used_bounds(arrow_bounds, formula_bounds)
18020 }
18021
18022 pub(crate) fn semantic_used_cols_for_rows(
18023 &self,
18024 sheet: &str,
18025 start_row: u32,
18026 end_row: u32,
18027 ) -> Option<(u32, u32)> {
18028 let arrow_bounds = self
18029 .sheet_store()
18030 .sheet(sheet)
18031 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18032 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18033 Self::union_used_bounds(arrow_bounds, formula_bounds)
18034 }
18035}
18036
18037impl<R> crate::traits::EvaluationContext for Engine<R>
18039where
18040 R: EvaluationContext,
18041{
18042 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
18043 &self.clock
18044 }
18045
18046 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
18047 self.thread_pool.as_ref()
18048 }
18049
18050 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
18051 self.active_cancel_flag.clone()
18052 }
18053
18054 fn chunk_hint(&self) -> Option<usize> {
18055 let hint =
18057 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
18058 Some(hint.clamp(1024, 1 << 20)) }
18060
18061 fn volatile_level(&self) -> crate::traits::VolatileLevel {
18062 self.config.volatile_level
18063 }
18064
18065 fn workbook_seed(&self) -> u64 {
18066 self.config.workbook_seed
18067 }
18068
18069 fn recalc_epoch(&self) -> u64 {
18070 self.recalc_epoch
18071 }
18072
18073 fn workbook_sheet_count(&self) -> Option<usize> {
18074 Some(self.graph.sheet_reg().active_len())
18075 }
18076
18077 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
18078 self.graph.sheet_reg().active_position(sheet)
18079 }
18080
18081 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
18082 self.sheet_index_by_name(current_sheet)
18083 }
18084
18085 fn inspect_reference(
18086 &self,
18087 reference: &ReferenceType,
18088 current_sheet: &str,
18089 ) -> Result<Option<ReferenceInfo>, ExcelError> {
18090 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
18091 let sheet_id = self
18092 .graph
18093 .sheet_id(sheet_name)
18094 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18095 let sheet_index = self
18096 .graph
18097 .sheet_reg()
18098 .active_position_by_id(sheet_id)
18099 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18100 Ok((sheet_id, sheet_index))
18101 };
18102
18103 let cell_info =
18104 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
18105 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18106 let row0 = row
18107 .checked_sub(1)
18108 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18109 let col0 = col
18110 .checked_sub(1)
18111 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18112 Ok(ReferenceInfo {
18113 first_sheet_index: Some(sheet_index),
18114 sheet_count: Some(1),
18115 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18116 })
18117 };
18118
18119 let range_info = |sheet_name: &str,
18120 start_row: Option<u32>,
18121 start_col: Option<u32>|
18122 -> Result<ReferenceInfo, ExcelError> {
18123 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
18124 let row = start_row.unwrap_or(1);
18125 let col = start_col.unwrap_or(1);
18126 let row0 = row
18127 .checked_sub(1)
18128 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18129 let col0 = col
18130 .checked_sub(1)
18131 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18132 Ok(ReferenceInfo {
18133 first_sheet_index: Some(sheet_index),
18134 sheet_count: Some(1),
18135 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
18136 })
18137 };
18138
18139 let info = match reference {
18140 ReferenceType::Cell {
18141 sheet, row, col, ..
18142 } => {
18143 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18144 cell_info(sheet_name, *row, *col)?
18145 }
18146 ReferenceType::Range {
18147 sheet,
18148 start_row,
18149 start_col,
18150 ..
18151 } => {
18152 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18153 range_info(sheet_name, *start_row, *start_col)?
18154 }
18155 ReferenceType::Cell3D {
18156 sheet_first,
18157 sheet_last,
18158 row,
18159 col,
18160 ..
18161 } => {
18162 let first = cell_info(sheet_first, *row, *col)?;
18163 ReferenceInfo {
18164 first_sheet_index: first.first_sheet_index,
18165 sheet_count: self
18166 .graph
18167 .sheet_reg()
18168 .active_span_len(sheet_first, sheet_last),
18169 first_cell: first.first_cell,
18170 }
18171 }
18172 ReferenceType::Range3D {
18173 sheet_first,
18174 sheet_last,
18175 start_row,
18176 start_col,
18177 ..
18178 } => {
18179 let first = range_info(sheet_first, *start_row, *start_col)?;
18180 ReferenceInfo {
18181 first_sheet_index: first.first_sheet_index,
18182 sheet_count: self
18183 .graph
18184 .sheet_reg()
18185 .active_span_len(sheet_first, sheet_last),
18186 first_cell: first.first_cell,
18187 }
18188 }
18189 ReferenceType::NamedRange(name) => {
18190 let current_id = self
18191 .graph
18192 .sheet_id(current_sheet)
18193 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18194 let named = self
18195 .graph
18196 .resolve_name_entry(name, current_id)
18197 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18198 match &named.definition {
18199 NamedDefinition::Cell(cell) => ReferenceInfo {
18200 first_sheet_index: self
18201 .graph
18202 .sheet_reg()
18203 .active_position_by_id(cell.sheet_id),
18204 sheet_count: Some(1),
18205 first_cell: Some(*cell),
18206 },
18207 NamedDefinition::Range(range) => ReferenceInfo {
18208 first_sheet_index: self
18209 .graph
18210 .sheet_reg()
18211 .active_position_by_id(range.start.sheet_id),
18212 sheet_count: Some(1),
18213 first_cell: Some(range.start),
18214 },
18215 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
18216 ReferenceInfo {
18217 first_sheet_index: None,
18218 sheet_count: None,
18219 first_cell: None,
18220 }
18221 }
18222 }
18223 }
18224 ReferenceType::Table(tref) => {
18225 let table = self
18226 .graph
18227 .resolve_table_entry(&tref.name)
18228 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
18229 ReferenceInfo {
18230 first_sheet_index: self
18231 .graph
18232 .sheet_reg()
18233 .active_position_by_id(table.range.start.sheet_id),
18234 sheet_count: Some(1),
18235 first_cell: Some(table.range.start),
18236 }
18237 }
18238 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
18239 };
18240
18241 Ok(Some(info))
18242 }
18243
18244 fn resolve_spill_reference(
18245 &self,
18246 anchor: &ReferenceType,
18247 current_sheet: &str,
18248 ) -> Result<ReferenceType, ExcelError> {
18249 let no_spill = || {
18250 ExcelError::new(ExcelErrorKind::Ref)
18251 .with_message("Spill reference anchor has no current spill")
18252 };
18253 let mut qualify = true;
18256 let anchor_cell = match anchor {
18257 ReferenceType::Cell {
18258 sheet, row, col, ..
18259 } => {
18260 qualify = sheet.is_some();
18261 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
18262 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(no_spill)?;
18263 if *row == 0 || *col == 0 {
18264 return Err(no_spill());
18265 }
18266 CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true))
18267 }
18268 ReferenceType::NamedRange(name) => {
18269 let current_id = self.graph.sheet_id(current_sheet).ok_or_else(no_spill)?;
18270 let named = self
18271 .graph
18272 .resolve_name_entry(name, current_id)
18273 .ok_or_else(no_spill)?;
18274 match &named.definition {
18275 NamedDefinition::Cell(cell) => *cell,
18276 NamedDefinition::Range(range)
18277 if range.start.sheet_id == range.end.sheet_id
18278 && range.start.coord.row() == range.end.coord.row()
18279 && range.start.coord.col() == range.end.coord.col() =>
18280 {
18281 range.start
18282 }
18283 _ => {
18284 return Err(ExcelError::new(ExcelErrorKind::Ref)
18285 .with_message("Spill reference name must refer to a single cell"));
18286 }
18287 }
18288 }
18289 _ => {
18290 return Err(ExcelError::new(ExcelErrorKind::Ref)
18291 .with_message("Spill reference operand must be a single cell"));
18292 }
18293 };
18294 let vertex = self
18295 .graph
18296 .get_vertex_id_for_address(&anchor_cell)
18297 .ok_or_else(no_spill)?;
18298 if self.graph.fixed_single_arrays.contains(&vertex)
18301 || self.graph.fixed_array_shapes.contains_key(&vertex)
18302 {
18303 return Err(ExcelError::new(ExcelErrorKind::Ref)
18304 .with_message("Spill reference to a fixed array formula"));
18305 }
18306 let (first, last) = match self.graph.spill_extent_for_anchor(vertex) {
18307 Some(extent) => extent,
18308 None if !self.graph.spill_registry_has_anchor(vertex)
18312 && self.graph.is_current_declared_dynamic_anchor(vertex)
18313 && self.declared_anchor_holds_scalar(anchor_cell) =>
18314 {
18315 (anchor_cell, anchor_cell)
18316 }
18317 None => return Err(no_spill()),
18318 };
18319 let sheet_name = qualify.then(|| self.graph.sheet_name(first.sheet_id).to_string());
18320 let (sr, sc) = (first.coord.row() + 1, first.coord.col() + 1);
18321 let (er, ec) = (last.coord.row() + 1, last.coord.col() + 1);
18322 Ok(if sr == er && sc == ec {
18324 ReferenceType::cell_with_abs(sheet_name, sr, sc, true, true)
18325 } else {
18326 ReferenceType::Range {
18327 sheet: sheet_name,
18328 start_row: Some(sr),
18329 start_col: Some(sc),
18330 end_row: Some(er),
18331 end_col: Some(ec),
18332 start_row_abs: true,
18333 start_col_abs: true,
18334 end_row_abs: true,
18335 end_col_abs: true,
18336 }
18337 })
18338 }
18339
18340 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
18341 let sheet_name = self.graph.sheet_name(cell.sheet_id);
18342 if sheet_name.is_empty() {
18343 return Err(ExcelError::new(ExcelErrorKind::Ref));
18344 }
18345 let row = cell.coord.row() + 1;
18346 let col = cell.coord.col() + 1;
18347
18348 if let Some(entries) = self.staged_formulas.get(sheet_name)
18349 && let Some(text) = entries.get(row, col)
18350 {
18351 return Ok(Some(if text.starts_with('=') {
18352 text.to_owned()
18353 } else {
18354 format!("={text}")
18355 }));
18356 }
18357
18358 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
18359 return Ok(None);
18360 };
18361 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
18362 }
18363
18364 fn used_rows_for_columns(
18365 &self,
18366 sheet: &str,
18367 start_col: u32,
18368 end_col: u32,
18369 ) -> Option<(u32, u32)> {
18370 let sheet_id = self.graph.sheet_id(sheet)?;
18372 let snap = self.data_snapshot_id();
18373 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18374 guard
18375 .as_ref()
18376 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
18377 }) {
18378 return cached;
18379 }
18380
18381 let arrow_bounds = self
18382 .sheet_store()
18383 .sheet(sheet)
18384 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
18385 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
18386 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18387 Some(bounds)
18388 } else {
18389 let sc0 = start_col.saturating_sub(1);
18390 let ec0 = end_col.saturating_sub(1);
18391 self.graph
18392 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
18393 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18394 };
18395
18396 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18397 guard
18398 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18399 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
18400 }
18401
18402 computed
18403 }
18404
18405 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
18406 let sheet_id = self.graph.sheet_id(sheet)?;
18408 let snap = self.data_snapshot_id();
18409 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
18410 guard
18411 .as_ref()
18412 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
18413 }) {
18414 return cached;
18415 }
18416
18417 let arrow_bounds = self
18418 .sheet_store()
18419 .sheet(sheet)
18420 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
18421 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
18422 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
18423 Some(bounds)
18424 } else {
18425 let sr0 = start_row.saturating_sub(1);
18426 let er0 = end_row.saturating_sub(1);
18427 self.graph
18428 .used_col_bounds_for_rows(sheet_id, sr0, er0)
18429 .map(|(a0, b0)| (a0 + 1, b0 + 1))
18430 };
18431
18432 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
18433 guard
18434 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
18435 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
18436 }
18437
18438 computed
18439 }
18440
18441 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
18442 let _ = self.graph.sheet_id(sheet)?;
18443 Some((1_048_576, 16_384)) }
18447
18448 fn data_snapshot_id(&self) -> u64 {
18449 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
18450 }
18451
18452 fn backend_caps(&self) -> crate::traits::BackendCaps {
18453 crate::traits::BackendCaps {
18454 streaming: true,
18455 used_region: true,
18456 write: false,
18457 tables: false,
18458 async_stream: false,
18459 }
18460 }
18461
18462 fn build_lookup_index(
18463 &self,
18464 view: &RangeView<'_>,
18465 axis: LookupAxis,
18466 ) -> Option<Arc<LookupIndex>> {
18467 self.build_lookup_index_impl(view, axis)
18468 }
18469
18470 fn date_system(&self) -> crate::engine::DateSystem {
18473 self.config.date_system
18474 }
18475 fn resolve_range_view<'c>(
18477 &'c self,
18478 reference: &ReferenceType,
18479 current_sheet: &str,
18480 ) -> Result<RangeView<'c>, ExcelError> {
18481 match reference {
18482 ReferenceType::External(ext) => {
18483 let name = ext.raw.as_str();
18484 match ext.kind {
18485 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
18486 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
18487 return Err(ExcelError::new(ExcelErrorKind::Name)
18488 .with_message(format!("Undefined name: {name}")));
18489 };
18490 let version = source
18491 .version
18492 .or_else(|| self.resolver.source_scalar_version(name));
18493 let v = self.resolve_source_scalar_cached(name, version)?;
18494 Ok(RangeView::from_owned_rows(
18495 vec![vec![v]],
18496 self.config.date_system,
18497 ))
18498 }
18499 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
18500 let Some(source) = self.graph.resolve_source_table_entry(name) else {
18501 let kind =
18505 if crate::engine::refs::unbound_external_range_defers(&ext.kind) {
18506 ExcelErrorKind::Ref
18507 } else {
18508 ExcelErrorKind::Name
18509 };
18510 return Err(ExcelError::new(kind)
18511 .with_message(format!("Undefined table: {name}")));
18512 };
18513 let version = source
18514 .version
18515 .or_else(|| self.resolver.source_table_version(name));
18516 let table = self.resolve_source_table_cached(name, version)?;
18517 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
18518 self.source_table_to_range_view(table.as_ref(), &spec)
18519 }
18520 }
18521 }
18522 ReferenceType::Range { .. } => {
18523 let shared = self.resolve_shared_ref(reference, current_sheet)?;
18524 let formualizer_common::SheetRef::Range(range) = shared else {
18525 return Err(ExcelError::new(ExcelErrorKind::Ref));
18526 };
18527 let sheet_id = match range.sheet {
18530 formualizer_common::SheetLocator::Id(id) => id,
18531 formualizer_common::SheetLocator::Current
18532 | formualizer_common::SheetLocator::Name(_) => {
18533 return Err(ExcelError::new(ExcelErrorKind::Ref));
18534 }
18535 };
18536 let sheet_name = self.graph.sheet_name(sheet_id);
18537
18538 let bounded_range = if range.start_row.is_some()
18539 && range.start_col.is_some()
18540 && range.end_row.is_some()
18541 && range.end_col.is_some()
18542 {
18543 Some(RangeRef::try_from_shared(range.as_ref())?)
18544 } else {
18545 None
18546 };
18547
18548 let sr = bounded_range
18549 .as_ref()
18550 .map(|r| r.start.coord.row() + 1)
18551 .or_else(|| range.start_row.map(|b| b.index + 1));
18552 let sc = bounded_range
18553 .as_ref()
18554 .map(|r| r.start.coord.col() + 1)
18555 .or_else(|| range.start_col.map(|b| b.index + 1));
18556 let er = bounded_range
18557 .as_ref()
18558 .map(|r| r.end.coord.row() + 1)
18559 .or_else(|| range.end_row.map(|b| b.index + 1));
18560 let ec = bounded_range
18561 .as_ref()
18562 .map(|r| r.end.coord.col() + 1)
18563 .or_else(|| range.end_col.map(|b| b.index + 1));
18564
18565 let extent = resolve_used_extent_with_fallback(
18566 OpenRangeBounds {
18567 start_row: sr,
18568 start_column: sc,
18569 end_row: er,
18570 end_column: ec,
18571 },
18572 ExtentPolicy::EvaluationCompat {
18573 fallback_row: None,
18574 fallback_column: None,
18575 },
18576 || {
18577 self.sheet_bounds(sheet_name)
18578 .map(|_| self.config.max_open_ended_rows)
18579 },
18580 || {
18581 self.sheet_bounds(sheet_name)
18582 .map(|_| self.config.max_open_ended_cols)
18583 },
18584 |first, last| self.used_rows_for_columns(sheet_name, first, last),
18585 |first, last| self.used_cols_for_rows(sheet_name, first, last),
18586 );
18587 let (sr, sc, er, ec) = extent
18588 .map(|extent| {
18589 (
18590 extent.start_row,
18591 extent.start_column,
18592 extent.end_row,
18593 extent.end_column,
18594 )
18595 })
18596 .unwrap_or((1, 1, 0, 0));
18597
18598 if self.force_materialize_range_views {
18599 if er < sr || ec < sc {
18600 return Ok(RangeView::from_owned_rows(
18601 Vec::new(),
18602 self.config.date_system,
18603 ));
18604 }
18605 let h = (er - sr + 1) as u64;
18606 let w = (ec - sc + 1) as u64;
18607 let cell_count = h.saturating_mul(w);
18608 if cell_count <= self.config.spill.max_spill_cells as u64 {
18609 let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
18610 for r in sr..=er {
18611 let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
18612 for c in sc..=ec {
18613 rowv.push(
18614 self.get_cell_value(sheet_name, r, c)
18615 .unwrap_or(LiteralValue::Empty),
18616 );
18617 }
18618 rows.push(rowv);
18619 }
18620 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
18621 }
18622 }
18623
18624 let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
18625 return Ok(RangeView::from_owned_rows(
18626 Vec::new(),
18627 self.config.date_system,
18628 ));
18629 };
18630
18631 let rv = if er < sr || ec < sc {
18632 asheet.range_view(1, 1, 0, 0)
18633 } else {
18634 let sr0 = sr.saturating_sub(1) as usize;
18635 let sc0 = sc.saturating_sub(1) as usize;
18636 let er0 = er.saturating_sub(1) as usize;
18637 let ec0 = ec.saturating_sub(1) as usize;
18638 asheet.range_view(sr0, sc0, er0, ec0)
18639 };
18640
18641 Ok(rv)
18642 }
18643 ReferenceType::Cell { .. } => {
18644 let shared = self.resolve_shared_ref(reference, current_sheet)?;
18645 let formualizer_common::SheetRef::Cell(cell) = shared else {
18646 return Err(ExcelError::new(ExcelErrorKind::Ref));
18647 };
18648 let addr = CellRef::try_from_shared(cell)?;
18649 let sheet_id = addr.sheet_id;
18650 let sheet_name = self.graph.sheet_name(sheet_id);
18651 let row = addr.coord.row() + 1;
18652 let col = addr.coord.col() + 1;
18653
18654 if self.force_materialize_range_views {
18655 let v = self
18656 .get_cell_value(sheet_name, row, col)
18657 .unwrap_or(LiteralValue::Empty);
18658 return Ok(RangeView::from_owned_rows(
18659 vec![vec![v]],
18660 self.config.date_system,
18661 ));
18662 }
18663
18664 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
18665 let r0 = row.saturating_sub(1) as usize;
18666 let c0 = col.saturating_sub(1) as usize;
18667 let rv = asheet.range_view(r0, c0, r0, c0);
18668 Ok(rv)
18669 } else {
18670 let v = self
18671 .get_cell_value(sheet_name, row, col)
18672 .unwrap_or(LiteralValue::Empty);
18673 Ok(RangeView::from_owned_rows(
18674 vec![vec![v]],
18675 self.config.date_system,
18676 ))
18677 }
18678 }
18679 ReferenceType::NamedRange(name) => {
18680 if let Some(current_id) = self.graph.sheet_id(current_sheet)
18681 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
18682 {
18683 match &named.definition {
18684 NamedDefinition::Cell(cell_ref) => {
18685 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
18686 if self.force_materialize_range_views {
18687 let v = self
18688 .get_cell_value(
18689 sheet_name,
18690 cell_ref.coord.row() + 1,
18691 cell_ref.coord.col() + 1,
18692 )
18693 .unwrap_or(LiteralValue::Empty);
18694 return Ok(RangeView::from_owned_rows(
18695 vec![vec![v]],
18696 self.config.date_system,
18697 ));
18698 } else {
18699 let asheet = self
18700 .sheet_store()
18701 .sheet(sheet_name)
18702 .expect("Arrow sheet missing for named cell");
18703 let r0 = cell_ref.coord.row() as usize;
18704 let c0 = cell_ref.coord.col() as usize;
18705 let rv = asheet.range_view(r0, c0, r0, c0);
18706 return Ok(rv);
18707 }
18708 }
18709 NamedDefinition::Range(range_ref) => {
18710 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
18711 let sr = range_ref.start.coord.row() + 1;
18712 let sc = range_ref.start.coord.col() + 1;
18713 let er = range_ref.end.coord.row() + 1;
18714 let ec = range_ref.end.coord.col() + 1;
18715 if self.force_materialize_range_views {
18716 let h = (er.saturating_sub(sr) + 1) as u64;
18717 let w = (ec.saturating_sub(sc) + 1) as u64;
18718 let cell_count = h.saturating_mul(w);
18719 if cell_count <= self.config.spill.max_spill_cells as u64 {
18720 let mut rows: Vec<Vec<LiteralValue>> =
18721 Vec::with_capacity(h as usize);
18722 for r in sr..=er {
18723 let mut rowv: Vec<LiteralValue> =
18724 Vec::with_capacity(w as usize);
18725 for c in sc..=ec {
18726 rowv.push(
18727 self.get_cell_value(sheet_name, r, c)
18728 .unwrap_or(LiteralValue::Empty),
18729 );
18730 }
18731 rows.push(rowv);
18732 }
18733 return Ok(RangeView::from_owned_rows(
18734 rows,
18735 self.config.date_system,
18736 ));
18737 }
18738 }
18739 let asheet = self
18740 .sheet_store()
18741 .sheet(sheet_name)
18742 .expect("Arrow sheet missing for named range");
18743 let sr0 = range_ref.start.coord.row() as usize;
18744 let sc0 = range_ref.start.coord.col() as usize;
18745 let er0 = range_ref.end.coord.row() as usize;
18746 let ec0 = range_ref.end.coord.col() as usize;
18747 let rv = asheet.range_view(sr0, sc0, er0, ec0);
18748 return Ok(rv);
18749 }
18750 NamedDefinition::Literal(v) => {
18751 return Ok(RangeView::from_owned_rows(
18752 vec![vec![v.clone()]],
18753 self.config.date_system,
18754 ));
18755 }
18756 NamedDefinition::Formula { .. } => {
18757 if let Some(value) = self.graph.get_value(named.vertex) {
18758 return Ok(RangeView::from_owned_rows(
18759 vec![vec![value]],
18760 self.config.date_system,
18761 ));
18762 }
18763 }
18764 }
18765 }
18766
18767 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
18768 let version = source
18769 .version
18770 .or_else(|| self.resolver.source_scalar_version(name));
18771 let v = self.resolve_source_scalar_cached(name, version)?;
18772 return Ok(RangeView::from_owned_rows(
18773 vec![vec![v]],
18774 self.config.date_system,
18775 ));
18776 }
18777
18778 let data = self.resolver.resolve_named_range_reference(name)?;
18779 Ok(RangeView::from_owned_rows(data, self.config.date_system))
18780 }
18781 ReferenceType::Table(tref) => {
18782 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
18783 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
18784 let asheet = self
18785 .sheet_store()
18786 .sheet(sheet_name)
18787 .expect("Arrow sheet missing for table reference");
18788
18789 let sr0 = table.range.start.coord.row() as usize;
18790 let sc0 = table.range.start.coord.col() as usize;
18791 let er0 = table.range.end.coord.row() as usize;
18792 let ec0 = table.range.end.coord.col() as usize;
18793
18794 let has_totals = table.totals_row;
18795 let has_headers = table.header_row;
18796 let data_sr = if has_headers {
18797 sr0.saturating_add(1)
18798 } else {
18799 sr0
18800 };
18801 let data_er = if has_totals {
18802 er0.saturating_sub(1)
18803 } else {
18804 er0
18805 };
18806
18807 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
18808 if sr > er || sc > ec {
18809 asheet.range_view(1, 1, 0, 0)
18810 } else {
18811 asheet.range_view(sr, sc, er, ec)
18812 }
18813 };
18814
18815 let av = match &tref.specifier {
18816 None => {
18817 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18818 "Table reference without specifier is unsupported".to_string(),
18819 ));
18820 }
18821 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
18822 let Some(idx) = table.col_index(col) else {
18823 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18824 "Column refers to unknown table column".to_string(),
18825 ));
18826 };
18827 let c0 = sc0 + idx;
18828 select(data_sr, c0, data_er, c0)
18829 }
18830 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
18831 start,
18832 end,
18833 )) => {
18834 let Some(si) = table.col_index(start) else {
18835 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18836 "Column range refers to unknown column(s)".to_string(),
18837 ));
18838 };
18839 let Some(ei) = table.col_index(end) else {
18840 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
18841 "Column range refers to unknown column(s)".to_string(),
18842 ));
18843 };
18844 let (mut a, mut b) = (si, ei);
18845 if a > b {
18846 std::mem::swap(&mut a, &mut b);
18847 }
18848 let c_start = sc0 + a;
18849 let c_end = sc0 + b;
18850 select(data_sr, c_start, data_er, c_end)
18851 }
18852 Some(formualizer_parse::parser::TableSpecifier::All)
18853 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18854 formualizer_parse::parser::SpecialItem::All,
18855 )) => select(sr0, sc0, er0, ec0),
18856 Some(formualizer_parse::parser::TableSpecifier::Data)
18857 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18858 formualizer_parse::parser::SpecialItem::Data,
18859 )) => select(data_sr, sc0, data_er, ec0),
18860 Some(formualizer_parse::parser::TableSpecifier::Headers)
18861 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18862 formualizer_parse::parser::SpecialItem::Headers,
18863 )) => {
18864 if !has_headers {
18865 asheet.range_view(1, 1, 0, 0)
18866 } else {
18867 select(sr0, sc0, sr0, ec0)
18868 }
18869 }
18870 Some(formualizer_parse::parser::TableSpecifier::Totals)
18871 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18872 formualizer_parse::parser::SpecialItem::Totals,
18873 )) => {
18874 if !has_totals {
18875 asheet.range_view(1, 1, 0, 0)
18876 } else {
18877 select(er0, sc0, er0, ec0)
18878 }
18879 }
18880 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
18881 formualizer_parse::parser::SpecialItem::ThisRow,
18882 )) => {
18883 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18884 "@ (This Row) requires table-aware context; not yet supported"
18885 .to_string(),
18886 ));
18887 }
18888 Some(formualizer_parse::parser::TableSpecifier::Row(_))
18889 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
18890 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18891 "Complex structured references not yet supported".to_string(),
18892 ));
18893 }
18894 };
18895
18896 return Ok(av);
18897 }
18898
18899 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
18900 let version = source
18901 .version
18902 .or_else(|| self.resolver.source_table_version(&tref.name));
18903 let table = self.resolve_source_table_cached(&tref.name, version)?;
18904 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
18905 }
18906
18907 let boxed = self
18912 .resolve_range_like(&ReferenceType::Table(tref.clone()))
18913 .map_err(|e| {
18914 if e.kind == ExcelErrorKind::NImpl {
18915 ExcelError::new(ExcelErrorKind::Name)
18916 .with_message(format!("Unknown table: {}", tref.name))
18917 } else {
18918 e
18919 }
18920 })?;
18921 let owned = boxed.materialise().into_owned();
18922 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
18923 }
18924 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
18925 Err(ExcelError::new(ExcelErrorKind::NImpl)
18926 .with_message("3D references are not yet supported".to_string()))
18927 }
18928 }
18929 }
18930
18931 fn resolve_cell_format(
18932 &self,
18933 sheet: Option<&str>,
18934 row: u32,
18935 col: u32,
18936 current_sheet: &str,
18937 ) -> Option<crate::format::FormatId> {
18938 self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
18939 }
18940
18941 fn format_class(
18942 &self,
18943 format: crate::format::FormatId,
18944 ) -> Option<formualizer_common::numfmt::FormatClass> {
18945 self.format_registry.class(format).cloned()
18946 }
18947
18948 fn record_cell_derived_format(
18949 &self,
18950 sheet: &str,
18951 row: u32,
18952 col: u32,
18953 format: Option<crate::format::FormatId>,
18954 ) {
18955 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
18956 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18957 self.record_derived_format_at(cell, format);
18958 }
18959 }
18960
18961 fn resolve_cell_reference_value(
18962 &self,
18963 sheet: Option<&str>,
18964 row: u32,
18965 col: u32,
18966 current_sheet: &str,
18967 ) -> Result<LiteralValue, ExcelError> {
18968 let sheet_name = sheet.unwrap_or(current_sheet);
18969 if self.graph.sheet_id(sheet_name).is_none() {
18970 return Err(ExcelError::new(ExcelErrorKind::Ref));
18971 }
18972 Ok(self
18973 .get_cell_value(sheet_name, row, col)
18974 .unwrap_or(LiteralValue::Empty))
18975 }
18976
18977 fn resolve_cell_reference_value_formatted(
18978 &self,
18979 sheet: Option<&str>,
18980 row: u32,
18981 col: u32,
18982 current_sheet: &str,
18983 ) -> Result<(LiteralValue, Option<crate::format::FormatId>), ExcelError> {
18984 let sheet_name = sheet.unwrap_or(current_sheet);
18987 let Some(sheet_id) = self.graph.sheet_id(sheet_name) else {
18988 return Err(ExcelError::new(ExcelErrorKind::Ref));
18989 };
18990 let asheet = self.arrow_sheets.sheet(sheet_name);
18991 Ok(self.read_cell_formatted_in(sheet_id, asheet, row, col))
18992 }
18993
18994 fn build_criteria_mask(
18995 &self,
18996 view: &RangeView<'_>,
18997 col_in_view: usize,
18998 pred: &crate::args::CriteriaPredicate,
18999 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19000 #[cfg(any(test, feature = "test-support"))]
19001 criteria_mask_test_hooks::note_mask(view.dims().0);
19002 if view.dims().1 == 0 {
19003 return None;
19004 }
19005 let sheet_rows = view.sheet().nrows as usize;
19008 if sheet_rows == 0 || view.start_row() >= sheet_rows {
19009 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
19010 }
19011 compute_criteria_mask(view, col_in_view, pred)
19012 }
19013
19014 fn build_row_visibility_mask(
19015 &self,
19016 view: &RangeView<'_>,
19017 mode: VisibilityMaskMode,
19018 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
19019 self.build_row_visibility_mask_for_view(view, mode)
19020 }
19021
19022 fn nested_subtotal_cells(
19023 &self,
19024 view: &RangeView<'_>,
19025 include_aggregate: bool,
19026 ) -> Option<Vec<(usize, usize, usize)>> {
19027 self.nested_subtotal_cells_for_view(view, include_aggregate)
19028 }
19029}
19030
19031impl<R> Engine<R>
19032where
19033 R: EvaluationContext,
19034{
19035 fn clear_spill_projection_and_mirror(
19036 &mut self,
19037 anchor_vertex: VertexId,
19038 delta: Option<&mut DeltaCollector>,
19039 ) {
19040 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
19041 if spill_cells.is_empty() {
19042 return;
19043 }
19044
19045 if let Some(delta) = delta
19046 && delta.mode != DeltaMode::Off
19047 {
19048 let empty = LiteralValue::Empty;
19049 for cell in spill_cells.iter() {
19050 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19051 let old = self
19052 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19053 .unwrap_or(LiteralValue::Empty);
19054 if old != empty {
19055 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19056 }
19057 }
19058 }
19059
19060 self.graph.clear_spill_region(anchor_vertex);
19061 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
19062 self.record_structural_change(scope);
19063 }
19064
19065 if self.config.arrow_storage_enabled
19066 && self.config.delta_overlay_enabled
19067 && self.config.write_formula_overlay_enabled
19068 {
19069 let empty = LiteralValue::Empty;
19070 for cell in spill_cells.iter() {
19071 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19072 self.mirror_value_to_computed_overlay(
19073 &sheet_name,
19074 cell.coord.row() + 1,
19075 cell.coord.col() + 1,
19076 &empty,
19077 );
19078 }
19079 }
19080 }
19081
19082 fn apply_cycle_outcome(
19096 &mut self,
19097 cycle: &[VertexId],
19098 mut delta: Option<&mut DeltaCollector>,
19099 dirty_filter: Option<&FxHashSet<VertexId>>,
19100 ) -> usize {
19101 let circ_error = LiteralValue::Error(
19102 ExcelError::new(ExcelErrorKind::Circ)
19103 .with_message("Circular dependency detected".to_string()),
19104 );
19105 let mut stamped = 0usize;
19106 for &vertex_id in cycle {
19107 if let Some(filter) = dirty_filter
19108 && !filter.contains(&vertex_id)
19109 {
19110 continue;
19111 }
19112 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
19113 stamped += 1;
19114 }
19115 stamped
19116 }
19117
19118 fn stamp_cycle_error(
19133 &mut self,
19134 vertex_id: VertexId,
19135 circ_error: &LiteralValue,
19136 mut delta: Option<&mut DeltaCollector>,
19137 ) {
19138 if self.graph.spill_registry_has_anchor(vertex_id) {
19140 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
19141 }
19142 self.spill_mgr.release_owner(vertex_id);
19145
19146 if let Some(d) = delta
19148 && d.mode != DeltaMode::Off
19149 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
19150 {
19151 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19152 let old = self
19153 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19154 .unwrap_or(LiteralValue::Empty);
19155 if old != *circ_error {
19156 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19157 }
19158 }
19159
19160 self.graph.update_vertex_value_ref(vertex_id, circ_error);
19161 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
19162 }
19163
19164 fn handle_cycle_unit(
19177 &mut self,
19178 cycle: &[VertexId],
19179 mut delta: Option<&mut DeltaCollector>,
19180 dirty_filter: Option<&FxHashSet<VertexId>>,
19181 cancel_flag: Option<&AtomicBool>,
19182 ) -> Result<usize, ExcelError> {
19183 self.resource_checkpoint(cycle.len() as u64)?;
19184 match self.config.cycle.detection {
19185 CycleDetection::Static => {
19186 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
19187 }
19188 CycleDetection::Runtime => {
19189 if let Some(filter) = dirty_filter
19190 && !cycle.iter().any(|v| filter.contains(v))
19191 {
19192 return Ok(0);
19193 }
19194 self.evaluate_scc_unit(cycle, delta, cancel_flag)
19198 }
19199 }
19200 }
19201
19202 pub(crate) fn evaluate_scc_unit(
19222 &mut self,
19223 cycle: &[VertexId],
19224 mut delta: Option<&mut DeltaCollector>,
19225 cancel_flag: Option<&AtomicBool>,
19226 ) -> Result<usize, ExcelError> {
19227 struct SccMember {
19228 vertex: VertexId,
19229 cell: Option<CellRef>,
19230 }
19231
19232 let task_start = crate::instant::FzInstant::now();
19233
19234 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
19239 let mut name_members: Vec<(VertexId, String)> = Vec::new();
19240 let mut other_members: Vec<VertexId> = Vec::new();
19241 for &v in cycle {
19242 match self.graph.get_vertex_kind(v) {
19243 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19244 match self.graph.get_cell_ref(v) {
19245 Some(cell) => cell_members.push((v, cell)),
19246 None => other_members.push(v),
19247 }
19248 }
19249 VertexKind::NamedScalar | VertexKind::NamedArray => {
19250 match self.graph.name_key_for_vertex(v) {
19251 Some(key) => name_members.push((v, key)),
19252 None => other_members.push(v),
19253 }
19254 }
19255 _ => other_members.push(v),
19256 }
19257 }
19258 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
19259 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
19260 other_members.sort_unstable();
19261
19262 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
19263 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
19264 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
19265 for (v, c) in &cell_members {
19266 members.push(SccMember {
19267 vertex: *v,
19268 cell: Some(*c),
19269 });
19270 }
19271 for (v, _) in &name_members {
19272 members.push(SccMember {
19273 vertex: *v,
19274 cell: None,
19275 });
19276 }
19277 for v in &other_members {
19278 members.push(SccMember {
19279 vertex: *v,
19280 cell: None,
19281 });
19282 }
19283 let n = members.len();
19284 let recordable = cell_refs.len() + name_keys.len();
19287
19288 let circ_error = LiteralValue::Error(
19289 ExcelError::new(ExcelErrorKind::Circ)
19290 .with_message("Circular dependency detected".to_string()),
19291 );
19292
19293 if !self.iterative_state_values.is_empty() {
19304 let restore: Vec<(VertexId, LiteralValue)> = members
19305 .iter()
19306 .filter_map(|m| {
19307 let cell = m.cell?;
19308 let persisted = self.iterative_state_values.get(&m.vertex)?;
19309 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19310 let overlay = self
19311 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19312 .unwrap_or(LiteralValue::Empty);
19313 if matches!(overlay, LiteralValue::Empty) {
19314 Some((m.vertex, persisted.clone()))
19315 } else {
19316 None
19317 }
19318 })
19319 .collect();
19320 for (vertex, value) in restore {
19321 self.mirror_vertex_value_to_overlay(vertex, &value);
19322 }
19323 }
19324
19325 let snapshot: Vec<LiteralValue> = members
19328 .iter()
19329 .map(|m| match m.cell {
19330 Some(cell) => {
19331 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19332 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
19333 .unwrap_or(LiteralValue::Empty)
19334 }
19335 None => self
19336 .graph
19337 .get_value(m.vertex)
19338 .unwrap_or(LiteralValue::Empty),
19339 })
19340 .collect();
19341
19342 let mut excluded = vec![false; n];
19347 let mut last_value = snapshot.clone();
19348 let mut stamped = 0usize;
19349 for (i, m) in members.iter().enumerate() {
19350 match self.graph.get_vertex_kind(m.vertex) {
19351 VertexKind::FormulaArray => {
19352 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
19356 excluded[i] = true;
19357 last_value[i] = circ_error.clone();
19358 stamped += 1;
19359 }
19360 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
19361 _ => excluded[i] = true,
19362 }
19363 }
19364
19365 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
19366
19367 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
19369 let mut pos: Vec<i64> = vec![-1; n];
19371 let mut changed = vec![false; n];
19373
19374 macro_rules! run_member {
19377 ($i:expr) => {{
19378 let i: usize = $i;
19379 let m = &members[i];
19380 if i < recordable {
19381 collector.set_current(i as u32);
19382 }
19383 let value = {
19384 let ctx = RecordingContext::new(&*self, &collector);
19385 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
19386 Ok(v) => v,
19387 Err(e) => LiteralValue::Error(e),
19388 }
19389 };
19390 let is_cell_formula = m.cell.is_some();
19391 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
19392 self.stamp_cycle_error(m.vertex, &circ_error, None);
19397 excluded[i] = true;
19398 stamped += 1;
19399 changed[i] = last_value[i] != circ_error;
19400 last_value[i] = circ_error.clone();
19401 } else {
19402 self.graph.update_vertex_value_ref(m.vertex, &value);
19403 self.mirror_vertex_value_to_overlay(m.vertex, &value);
19404 #[cfg(debug_assertions)]
19408 if let Some(cell) = m.cell {
19409 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19410 debug_assert!(
19411 self.read_delta_overlay_cell(
19412 &sheet_name,
19413 cell.coord.row() + 1,
19414 cell.coord.col() + 1
19415 )
19416 .is_none(),
19417 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
19418 cell.coord.row() + 1,
19419 cell.coord.col() + 1
19420 );
19421 }
19422 changed[i] = last_value[i] != value;
19423 last_value[i] = value;
19424 }
19425 }};
19426 }
19427
19428 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
19429 if let Some(flag) = flag
19430 && flag.load(Ordering::Relaxed)
19431 {
19432 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
19433 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
19434 }
19435 Ok(())
19436 };
19437
19438 check_cancel(cancel_flag)?;
19440 let mut passes = 1usize;
19441 {
19442 let mut p = 0i64;
19443 for i in 0..n {
19444 if excluded[i] {
19445 continue;
19446 }
19447 run_member!(i);
19448 pos[i] = p;
19449 p += 1;
19450 }
19451 }
19452
19453 let policy = self.config.cycle.policy;
19469 let cap = n + 2;
19470 let mut witnessed_cycles = 0usize;
19471 let mut capped = false;
19472 let mut iterating = false;
19474 let mut converged = false;
19475 let mut exact_fixed_point = false;
19476 let mut prev_pass: Option<Vec<LiteralValue>> = None;
19480 let mut iter_max_delta = 0f64;
19483 let mut iter_nan_converged = 0usize;
19484 let mut settle_passes = 0usize;
19488 loop {
19489 let drained = collector.take_edges();
19492 for i in 0..n {
19493 if pos[i] >= 0 {
19494 out_edges[i].clear();
19495 }
19496 }
19497 for (from, to) in drained {
19498 debug_assert!(
19499 pos[from as usize] >= 0,
19500 "edge from a member that did not run"
19501 );
19502 out_edges[from as usize].push(to);
19503 }
19504 let mut edges: Vec<(u32, u32)> = Vec::new();
19505 for (i, outs) in out_edges.iter().enumerate() {
19506 if excluded[i] {
19507 continue;
19508 }
19509 for &t in outs {
19510 edges.push((i as u32, t));
19511 }
19512 }
19513 edges.sort_unstable();
19514 edges.dedup();
19515
19516 let analysis = analyze_live_graph(n, &edges);
19517
19518 if analysis.cycle_count > 0 {
19519 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
19523 match policy {
19524 CyclePolicy::Error => {
19525 for i in 0..n {
19531 if analysis.in_cycle[i] && !excluded[i] {
19532 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
19533 excluded[i] = true;
19534 last_value[i] = circ_error.clone();
19535 stamped += 1;
19536 }
19537 }
19538 check_cancel(cancel_flag)?;
19539 let order: Vec<usize> = analysis
19540 .topo
19541 .iter()
19542 .map(|&i| i as usize)
19543 .filter(|&i| !excluded[i])
19544 .collect();
19545 if !order.is_empty() {
19546 passes += 1;
19547 for i in order {
19548 run_member!(i);
19549 }
19550 }
19551 break;
19552 }
19553 CyclePolicy::Iterate {
19554 max_iterations,
19555 max_change,
19556 } => {
19557 iterating = true;
19559
19560 if let Some(prev) = &prev_pass {
19566 let mut round_max_delta = 0f64;
19567 let mut round_nan = 0usize;
19568 let mut all_converged = true;
19569 let mut round_exact = true;
19570 for i in 0..n {
19571 if excluded[i] {
19572 continue;
19576 }
19577 let out = crate::engine::convergence::values_converged(
19578 &prev[i],
19579 &last_value[i],
19580 max_change,
19581 self.config.date_system,
19582 );
19583 if out.nan_converged {
19584 round_nan += 1;
19585 }
19586 if let Some(d) = out.abs_delta {
19587 round_max_delta = round_max_delta.max(d);
19588 if d != 0.0 {
19589 round_exact = false;
19590 }
19591 }
19592 if !out.converged {
19593 all_converged = false;
19594 }
19595 }
19596 iter_max_delta = round_max_delta;
19599 iter_nan_converged = round_nan;
19600 if all_converged {
19601 exact_fixed_point = round_exact;
19602 converged = true;
19603 break;
19604 }
19605 }
19606
19607 if passes >= max_iterations as usize {
19619 capped = true;
19620 break;
19621 }
19622
19623 check_cancel(cancel_flag)?;
19624 prev_pass = Some(last_value.clone());
19632 for x in pos.iter_mut() {
19633 *x = -1;
19634 }
19635 changed.fill(false);
19636 passes += 1;
19637 let mut p = 0i64;
19638 for i in 0..n {
19639 if excluded[i] {
19640 continue;
19641 }
19642 run_member!(i);
19643 pos[i] = p;
19644 p += 1;
19645 }
19646 continue;
19647 }
19648 }
19649 }
19650
19651 let mut stale: Vec<usize> = Vec::new();
19654 for i in 0..n {
19655 if excluded[i] {
19656 continue;
19657 }
19658 let is_stale = out_edges[i].iter().any(|&t| {
19659 let t = t as usize;
19660 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
19661 });
19662 if is_stale {
19663 stale.push(i);
19664 }
19665 }
19666 if stale.is_empty() {
19667 break; }
19669 if 1 + settle_passes >= cap {
19670 capped = true;
19672 for (i, m) in members.iter().enumerate() {
19673 if !excluded[i] {
19674 self.stamp_cycle_error(m.vertex, &circ_error, None);
19675 excluded[i] = true;
19676 last_value[i] = circ_error.clone();
19677 stamped += 1;
19678 }
19679 }
19680 break;
19681 }
19682
19683 check_cancel(cancel_flag)?;
19684 prev_pass = None;
19691 let topo_pos = analysis.topo_positions();
19692 stale.sort_unstable_by_key(|&i| topo_pos[i]);
19693 for x in pos.iter_mut() {
19694 *x = -1;
19695 }
19696 changed.fill(false);
19697 passes += 1;
19698 settle_passes += 1;
19699 for (p, i) in stale.into_iter().enumerate() {
19700 run_member!(i);
19701 pos[i] = p as i64;
19702 }
19703 }
19704
19705 self.live_cancellation_after_work(SCC_CANCELLED)?;
19711
19712 if iterating && !converged && !capped {
19717 converged = true;
19718 }
19719
19720 collector.clear_current();
19723 if let Some(d) = delta
19724 && d.mode != DeltaMode::Off
19725 {
19726 for (i, m) in members.iter().enumerate() {
19727 if let Some(cell) = m.cell
19728 && last_value[i] != snapshot[i]
19729 {
19730 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
19731 }
19732 }
19733 }
19734
19735 if !self.retained_scc_members.is_empty() {
19753 for m in members.iter() {
19754 self.retained_scc_members.remove(&m.vertex);
19755 }
19756 }
19757 if iterating {
19758 let retain = converged
19759 && !capped
19760 && exact_fixed_point
19761 && iter_nan_converged == 0
19762 && members
19763 .iter()
19764 .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
19765 if retain {
19766 if self.retained_scc_members.is_empty() {
19767 self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
19768 self.retained_scc_function_epoch_seen =
19769 crate::function_registry::semantic_epoch();
19770 self.retained_scc_provider_revision_seen =
19771 self.resolver.planning_semantic_revision();
19772 }
19773 let scc_id = self.next_retained_scc_id;
19774 self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
19775 for (i, m) in members.iter().enumerate() {
19776 self.retained_scc_members.insert(m.vertex, scc_id);
19777 if matches!(last_value[i], LiteralValue::Empty) {
19780 self.iterative_state_values.remove(&m.vertex);
19781 } else {
19782 self.iterative_state_values
19783 .insert(m.vertex, last_value[i].clone());
19784 }
19785 }
19786 } else {
19787 self.pending_iterative_redirty
19788 .extend(members.iter().map(|m| m.vertex));
19789 }
19790 } else if !self.iterative_state_values.is_empty() {
19791 for m in members.iter() {
19795 self.iterative_state_values.remove(&m.vertex);
19796 }
19797 }
19798
19799 {
19800 let t = &mut self.last_cycle_telemetry;
19801 t.static_sccs += 1;
19802 if witnessed_cycles == 0 && stamped == 0 && !capped {
19803 t.phantom_sccs += 1;
19804 }
19805 t.live_cycles_witnessed += witnessed_cycles;
19806 t.circ_cells_stamped += stamped;
19807 t.settle_passes_total += passes;
19808 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
19809 if iterating {
19810 t.iterated_sccs += 1;
19811 if converged {
19812 t.converged_sccs += 1;
19813 }
19814 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
19815 t.nan_converged += iter_nan_converged;
19816 }
19817 if capped {
19818 t.capped_sccs += 1;
19819 }
19820 t.elapsed_ms += task_start.elapsed().as_millis();
19821 }
19822
19823 Ok(stamped)
19824 }
19825
19826 fn evaluate_vertex_recorded(
19834 &self,
19835 vertex_id: VertexId,
19836 ctx: &RecordingContext<'_, R>,
19837 collector: &LiveEdgeCollector,
19838 ) -> Result<LiteralValue, ExcelError> {
19839 if !self.graph.vertex_exists(vertex_id) {
19840 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
19841 .with_message(format!("Vertex not found: {vertex_id:?}")));
19842 }
19843
19844 let kind = self.graph.get_vertex_kind(vertex_id);
19845 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
19846
19847 match kind {
19848 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
19849 let Some(view) = self.graph.formula_view(vertex_id) else {
19850 return Ok(LiteralValue::Number(0.0)); };
19852 let sheet_name = self.graph.sheet_name(sheet_id);
19853 let cell_ref = self
19854 .graph
19855 .get_cell_ref(vertex_id)
19856 .expect("cell ref for vertex");
19857 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
19858 interpreter
19859 .evaluate_formula_view(view, self.graph.data_store(), self.graph.sheet_reg())
19860 .map(|cv| {
19861 let format = cv.format_id();
19862 self.record_derived_format(vertex_id, format);
19863 self.materialize_formula_result(vertex_id, cv)
19864 })
19865 .or_else(|error| self.fit_formula_error(vertex_id, error))
19866 }
19867 VertexKind::NamedScalar | VertexKind::NamedArray => {
19868 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19869 ExcelError::new(ExcelErrorKind::Name)
19870 .with_message("Named range metadata missing".to_string())
19871 })?;
19872
19873 match &named_range.definition {
19874 NamedDefinition::Formula { ast, .. } => {
19875 let context_sheet = match named_range.scope {
19876 NameScope::Sheet(id) => id,
19877 NameScope::Workbook => sheet_id,
19878 };
19879 let sheet_name = self.graph.sheet_name(context_sheet);
19880 let cell_ref = self
19881 .graph
19882 .get_cell_ref(vertex_id)
19883 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19884 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
19885 if kind == VertexKind::NamedScalar {
19886 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
19887 } else {
19888 match interpreter.evaluate_ast(ast) {
19889 Ok(cv) => match cv.into_literal() {
19890 v @ LiteralValue::Array(_) => Ok(v),
19891 other => Ok(LiteralValue::Array(vec![vec![other]])),
19892 },
19893 Err(err) => Ok(LiteralValue::Error(err)),
19894 }
19895 }
19896 }
19897 NamedDefinition::Cell(cell_ref) => {
19898 collector.record_scalar(
19902 cell_ref.sheet_id,
19903 cell_ref.coord.row(),
19904 cell_ref.coord.col(),
19905 );
19906 self.evaluate_vertex_immutable(vertex_id)
19907 }
19908 NamedDefinition::Range(range_ref) => {
19909 if range_ref.start.sheet_id == range_ref.end.sheet_id {
19910 collector.record_rect(
19911 range_ref.start.sheet_id,
19912 range_ref.start.coord.row(),
19913 range_ref.start.coord.col(),
19914 range_ref.end.coord.row(),
19915 range_ref.end.coord.col(),
19916 );
19917 }
19918 self.evaluate_vertex_immutable(vertex_id)
19919 }
19920 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
19921 }
19922 }
19923 _ => self.evaluate_vertex_immutable(vertex_id),
19924 }
19925 }
19926
19927 fn pending_spill_occupied(&self, anchor: CellRef, end_row: u32, end_col: u32) -> bool {
19929 let sheet = self.graph.sheet_name(anchor.sheet_id);
19930 let package = self
19931 .staged_formulas
19932 .get(sheet)
19933 .and_then(|staged| staged.deferred_package.as_ref());
19934 self.staged_formula_index.occupies_spill(
19935 sheet,
19936 (anchor.coord.row() + 1, anchor.coord.col() + 1),
19937 (end_row + 1, end_col + 1),
19938 |point| package.is_some_and(|package| package.suppressed.contains(&point)),
19939 )
19940 }
19941
19942 fn remember_pending_spill(
19943 &mut self,
19944 vertex: VertexId,
19945 anchor: CellRef,
19946 region: Region,
19947 ) -> Result<(), ExcelError> {
19948 self.cancellation_checkpoint("pending spill occupancy")?;
19949 self.resource_checkpoint(1)?;
19950 if let Some(entry) = self
19951 .blocked_pending_spills
19952 .iter_mut()
19953 .find(|entry| entry.0 == vertex)
19954 {
19955 *entry = (vertex, anchor, region);
19956 return Ok(());
19957 }
19958 if self.blocked_pending_spills.len() == self.blocked_pending_spills.capacity() {
19959 let additional = self.blocked_pending_spills.capacity().max(1);
19962 let bytes =
19963 (additional as u64)
19964 .saturating_mul(std::mem::size_of::<(VertexId, CellRef, Region)>() as u64);
19965 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19966 ledger
19967 .reserve_retained(bytes)
19968 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19969 self.source_cache_accounted = self.source_cache_accounted.saturating_add(bytes);
19970 }
19971 if self
19972 .blocked_pending_spills
19973 .try_reserve_exact(additional)
19974 .is_err()
19975 {
19976 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19977 ledger
19978 .release_retained(bytes)
19979 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19980 self.source_cache_accounted -= bytes;
19981 }
19982 return Err(crate::engine::ResourceLedgerError::Exhausted(
19983 formualizer_common::ResourceExhaustionDetail {
19984 reason: formualizer_common::ResourceExhaustionReason::RetainedMemory,
19985 limit: u64::MAX,
19986 observed: bytes,
19987 request_id: None,
19988 },
19989 )
19990 .into_excel_error());
19991 }
19992 }
19993 self.blocked_pending_spills.push((vertex, anchor, region));
19994 Ok(())
19995 }
19996
19997 fn invalidate_pending_spills(&mut self, scope: StructuralScope) {
20001 if let StructuralScope::RemovedSheet(sheet) = scope {
20002 self.blocked_pending_spills
20003 .retain(|entry| entry.1.sheet_id != sheet);
20004 return;
20005 }
20006 for &(vertex, anchor, region) in &self.blocked_pending_spills {
20007 let affected = match scope {
20008 StructuralScope::Cell { sheet, row, col } => {
20009 region.intersects(&Region::point(sheet, row, col))
20010 }
20011 StructuralScope::Region(changed) => region.intersects(&changed),
20012 StructuralScope::Sheet(sheet) | StructuralScope::RemovedSheet(sheet) => {
20013 region.sheet_id() == sheet
20014 }
20015 StructuralScope::OpaqueGlobal | StructuralScope::AllSheets => true,
20016 };
20017 if affected
20018 && self.graph.vertex_exists(vertex)
20019 && self.graph.get_cell_ref(vertex) == Some(anchor)
20020 && matches!(
20021 self.graph.get_vertex_kind(vertex),
20022 VertexKind::FormulaScalar | VertexKind::FormulaArray
20023 )
20024 {
20025 self.graph.mark_vertex_dirty(vertex);
20026 }
20027 }
20028 }
20029
20030 fn guard_pending_spill_commit(
20031 &mut self,
20032 anchor_vertex: VertexId,
20033 targets: &[CellRef],
20034 ) -> Result<(), ExcelError> {
20035 let Some(anchor) = self.graph.get_cell_ref(anchor_vertex) else {
20036 return Ok(());
20037 };
20038 let Some(last) = targets.last() else {
20039 return Ok(());
20040 };
20041 let occupied = self.pending_spill_occupied(anchor, last.coord.row(), last.coord.col())
20042 || self
20043 .graph
20044 .table_intersects_spill(anchor, last.coord.row(), last.coord.col());
20045 if (occupied
20046 || self
20047 .blocked_pending_spills
20048 .iter()
20049 .any(|entry| entry.0 == anchor_vertex))
20050 && let Err(error) = self.remember_pending_spill(
20051 anchor_vertex,
20052 anchor,
20053 Region::rect(
20054 anchor.sheet_id,
20055 anchor.coord.row(),
20056 last.coord.row(),
20057 anchor.coord.col(),
20058 last.coord.col(),
20059 ),
20060 )
20061 {
20062 self.spill_mgr.release_owner(anchor_vertex);
20063 return Err(error);
20064 }
20065 if occupied {
20066 self.spill_mgr.release_owner(anchor_vertex);
20067 return Err(ExcelError::new(ExcelErrorKind::Spill)
20068 .with_message("Spill blocked")
20069 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20070 expected_rows: last.coord.row() - anchor.coord.row() + 1,
20071 expected_cols: last.coord.col() - anchor.coord.col() + 1,
20072 }));
20073 }
20074 Ok(())
20075 }
20076
20077 fn commit_spill_and_mirror(
20079 &mut self,
20080 anchor_vertex: VertexId,
20081 targets: &[CellRef],
20082 rows: Vec<Vec<LiteralValue>>,
20083 delta: Option<&mut DeltaCollector>,
20084 ) -> Result<(), ExcelError> {
20085 self.guard_pending_spill_commit(anchor_vertex, targets)?;
20086 let prev_spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20087
20088 if let Some(delta) = delta
20089 && delta.mode != DeltaMode::Off
20090 {
20091 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20092 targets.iter().copied().collect();
20093 let empty = LiteralValue::Empty;
20094
20095 for cell in prev_spill_cells.iter() {
20097 if target_set.contains(cell) {
20098 continue;
20099 }
20100 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20101 let old = self
20102 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20103 .unwrap_or(LiteralValue::Empty);
20104 if old != empty {
20105 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20106 }
20107 }
20108
20109 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
20111 let width = rows[0].len();
20112 for (idx, cell) in targets.iter().enumerate() {
20113 let r_off = idx / width;
20114 let c_off = idx % width;
20115 let new = rows
20116 .get(r_off)
20117 .and_then(|r| r.get(c_off))
20118 .cloned()
20119 .unwrap_or(LiteralValue::Empty);
20120 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20121 let old = self
20122 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20123 .unwrap_or(LiteralValue::Empty);
20124 if old != new {
20125 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20126 }
20127 }
20128 } else {
20129 for cell in targets.iter() {
20131 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20132 let old = self
20133 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20134 .unwrap_or(LiteralValue::Empty);
20135 if !matches!(old, LiteralValue::Empty) {
20136 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20137 }
20138 }
20139 }
20140 }
20141
20142 let arrow_sheets = &self.arrow_sheets;
20145 self.spill_mgr.commit_array_with_value_probe(
20146 &mut self.graph,
20147 anchor_vertex,
20148 targets,
20149 rows.clone(),
20150 |g, cell| {
20151 let sheet_name = g.sheet_name(cell.sheet_id);
20152 let asheet = arrow_sheets.sheet(sheet_name)?;
20153 let r0 = cell.coord.row() as usize;
20154 let c0 = cell.coord.col() as usize;
20155 let v = asheet.get_cell_value(r0, c0);
20156 if matches!(v, LiteralValue::Empty) {
20157 None
20158 } else {
20159 Some(v)
20160 }
20161 },
20162 )?;
20163
20164 self.blocked_pending_spills
20165 .retain(|entry| entry.0 != anchor_vertex);
20166 self.spill_blocker_waiters.forget(anchor_vertex);
20167 if let Some(scope) = Self::structural_scope_from_cells(&prev_spill_cells) {
20168 self.record_structural_change(scope);
20169 }
20170 if let Some(scope) = Self::structural_scope_from_cells(targets) {
20171 self.record_structural_change(scope);
20172 }
20173
20174 if self.config.arrow_storage_enabled
20175 && self.config.delta_overlay_enabled
20176 && self.config.write_formula_overlay_enabled
20177 {
20178 if !prev_spill_cells.is_empty() {
20179 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
20180 targets.iter().copied().collect();
20181 let empty = LiteralValue::Empty;
20182 for cell in prev_spill_cells.iter() {
20183 if !target_set.contains(cell) {
20184 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20185 self.mirror_value_to_computed_overlay(
20186 &sheet_name,
20187 cell.coord.row() + 1,
20188 cell.coord.col() + 1,
20189 &empty,
20190 );
20191 }
20192 }
20193 }
20194
20195 for (idx, cell) in targets.iter().enumerate() {
20196 if rows.is_empty() || rows[0].is_empty() {
20197 break;
20198 }
20199 let width = rows[0].len();
20200 let r_off = idx / width;
20201 let c_off = idx % width;
20202 let v = rows[r_off][c_off].clone();
20203 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20204 self.mirror_value_to_computed_overlay(
20205 &sheet_name,
20206 cell.coord.row() + 1,
20207 cell.coord.col() + 1,
20208 &v,
20209 );
20210 }
20211 }
20212 Ok(())
20213 }
20214}
20215
20216#[cfg(test)]
20217#[path = "tests/authority_schedule_execution.rs"]
20218mod authority_schedule_execution;
20219
20220#[cfg(test)]
20221#[path = "tests/pending_spill.rs"]
20222mod pending_spill_tests;
20223
20224#[cfg(test)]
20225#[path = "tests/spill_batch_abort_192.rs"]
20226mod spill_batch_abort_192;
20227
20228use crate::engine::effects::Effect;
20233use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
20234
20235impl<R> Engine<R>
20236where
20237 R: EvaluationContext,
20238{
20239 pub(crate) fn plan_vertex_effects(
20246 &mut self,
20247 vertex_id: VertexId,
20248 computed_value: LiteralValue,
20249 ) -> Result<Vec<Effect>, ExcelError> {
20250 if self.freshness_armed() {
20253 if self.freshness_drop_stale(vertex_id) {
20254 return Ok(Vec::new());
20255 }
20256 let effects = self.plan_vertex_effects_unrecorded(vertex_id, computed_value)?;
20257 self.freshness_mark_committed(vertex_id);
20258 return Ok(effects);
20259 }
20260 self.plan_vertex_effects_unrecorded(vertex_id, computed_value)
20261 }
20262
20263 fn plan_vertex_effects_unrecorded(
20264 &mut self,
20265 vertex_id: VertexId,
20266 computed_value: LiteralValue,
20267 ) -> Result<Vec<Effect>, ExcelError> {
20268 let kind = self.graph.get_vertex_kind(vertex_id);
20269 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
20270
20271 if !is_formula {
20275 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
20276 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
20277 && owner != vertex_id
20278 {
20279 return Ok(Vec::new());
20280 }
20281 return Ok(vec![Effect::WriteCell {
20283 vertex_id,
20284 value: computed_value,
20285 }]);
20286 }
20287
20288 match computed_value {
20289 LiteralValue::Array(rows) => self.plan_array_effects(vertex_id, rows),
20290 other => self.plan_scalar_effects(vertex_id, other),
20291 }
20292 }
20293
20294 fn plan_scalar_effects(
20296 &mut self,
20297 vertex_id: VertexId,
20298 value: LiteralValue,
20299 ) -> Result<Vec<Effect>, ExcelError> {
20300 if let LiteralValue::Error(error) = &value
20303 && error.kind == ExcelErrorKind::Spill
20304 && error.message.as_deref() == Some("SpillTooLarge")
20305 && let formualizer_common::ExcelErrorExtra::Spill {
20306 expected_rows,
20307 expected_cols,
20308 } = error.extra
20309 && u64::from(expected_rows).saturating_mul(u64::from(expected_cols))
20310 > u64::from(self.config.spill.max_spill_cells)
20311 {
20312 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20313 return self.plan_spill_error_effects(
20314 vertex_id,
20315 "SpillTooLarge",
20316 expected_rows,
20317 expected_cols,
20318 );
20319 }
20320 if !matches!(&value, LiteralValue::Error(e) if e.kind == ExcelErrorKind::Spill) {
20321 self.blocked_pending_spills
20322 .retain(|entry| entry.0 != vertex_id);
20323 self.spill_blocker_waiters.forget(vertex_id);
20324 }
20325 let has_spill = self
20326 .graph
20327 .spill_cells_for_anchor(vertex_id)
20328 .is_some_and(|c| !c.is_empty());
20329
20330 let mut effects = Vec::new();
20331 if has_spill {
20332 effects.push(Effect::SpillClear {
20333 anchor_vertex: vertex_id,
20334 });
20335 }
20336 effects.push(Effect::WriteCell { vertex_id, value });
20337 Ok(effects)
20338 }
20339
20340 fn plan_array_effects(
20342 &mut self,
20343 vertex_id: VertexId,
20344 rows: Vec<Vec<LiteralValue>>,
20345 ) -> Result<Vec<Effect>, ExcelError> {
20346 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
20348
20349 let anchor = self
20350 .graph
20351 .get_cell_ref(vertex_id)
20352 .expect("cell ref for vertex");
20353 let sheet_id = anchor.sheet_id;
20354 let h = rows.len() as u32;
20355 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
20356
20357 let spill_cells = (h as u64).saturating_mul(w as u64);
20359 if spill_cells > self.config.spill.max_spill_cells as u64 {
20360 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
20361 }
20362
20363 const PACKED_MAX_ROW: u32 = 1_048_575;
20365 const PACKED_MAX_COL: u32 = 16_383;
20366 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
20367 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
20368 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
20369 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
20370 }
20371
20372 if h != 0 && w != 0 && self.graph.table_intersects_spill(anchor, end_row, end_col) {
20373 return self.plan_spill_error_effects(vertex_id, "Spill blocked by table", h, w);
20374 }
20375 let mut targets = Vec::new();
20376 for r in 0..h {
20377 for c in 0..w {
20378 targets.push(self.graph.make_cell_ref_internal(
20379 sheet_id,
20380 anchor.coord.row() + r,
20381 anchor.coord.col() + c,
20382 ));
20383 }
20384 }
20385
20386 if h != 0 && w != 0 {
20387 let occupied = self.pending_spill_occupied(anchor, end_row, end_col);
20388 if occupied
20389 || self
20390 .blocked_pending_spills
20391 .iter()
20392 .any(|entry| entry.0 == vertex_id)
20393 {
20394 self.spill_mgr.release_owner(vertex_id);
20395 self.remember_pending_spill(
20396 vertex_id,
20397 anchor,
20398 Region::rect(
20399 sheet_id,
20400 anchor.coord.row(),
20401 end_row,
20402 anchor.coord.col(),
20403 end_col,
20404 ),
20405 )?;
20406 }
20407 if occupied {
20408 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20409 }
20410 }
20411
20412 match self.spill_mgr.reserve(
20414 vertex_id,
20415 anchor,
20416 SpillShape { rows: h, cols: w },
20417 SpillMeta {
20418 epoch: self.recalc_epoch,
20419 config: self.config.spill,
20420 },
20421 ) {
20422 Ok(()) => {
20423 let mut yield_effects = Vec::new();
20425 if let Err((e, blocker)) = self
20426 .graph
20427 .plan_spill_region_with_blocker(vertex_id, &targets)
20428 {
20429 let yielding = if e.message.as_deref() == Some("BlockedBySpill") {
20430 self.graph.spills_yielding_to(vertex_id, &targets)
20431 } else {
20432 None
20433 };
20434 if let Some(yielding) = yielding {
20435 for &other in &yielding {
20439 let (rows, cols) = self
20440 .graph
20441 .spill_extent_for_anchor(other)
20442 .map(|(first, last)| {
20443 (
20444 last.coord.row() - first.coord.row() + 1,
20445 last.coord.col() - first.coord.col() + 1,
20446 )
20447 })
20448 .unwrap_or((0, 0));
20449 yield_effects.push(Effect::SpillClear {
20450 anchor_vertex: other,
20451 });
20452 yield_effects.push(Effect::WriteCell {
20453 vertex_id: other,
20454 value: Self::spill_error_value("Spill blocked", rows, cols),
20455 });
20456 }
20457 self.graph.mark_dirty_many(&yielding);
20458 } else {
20459 if matches!(
20463 e.message.as_deref(),
20464 Some("BlockedByFormula" | "BlockedBySpill")
20465 ) {
20466 self.spill_blocker_waiters.wait(vertex_id, blocker);
20467 }
20468 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
20469 }
20470 }
20471
20472 if !self.graph.value_cache_enabled() {
20476 let sheet_name = self.graph.sheet_name(sheet_id);
20477 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
20478 for cell in targets.iter() {
20479 if *cell == anchor {
20481 continue;
20482 }
20483 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
20485 continue;
20486 }
20487 if let Some(vid) = self.graph.get_vertex_id_for_address(cell)
20489 && vid != vertex_id
20490 {
20491 match self.graph.get_vertex_kind(vid) {
20492 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
20493 continue;
20494 }
20495 _ => {}
20496 }
20497 }
20498
20499 let v = asheet.get_cell_value(
20500 cell.coord.row() as usize,
20501 cell.coord.col() as usize,
20502 );
20503 if !matches!(v, LiteralValue::Empty) {
20504 return self.plan_spill_error_effects(
20505 vertex_id,
20506 "BlockedByValue",
20507 h,
20508 w,
20509 );
20510 }
20511 }
20512 }
20513 }
20514
20515 let top_left = rows
20516 .first()
20517 .and_then(|r| r.first())
20518 .cloned()
20519 .unwrap_or(LiteralValue::Empty);
20520
20521 let mut effects = yield_effects;
20522 let has_prev = self
20524 .graph
20525 .spill_cells_for_anchor(vertex_id)
20526 .is_some_and(|c| !c.is_empty());
20527 if has_prev {
20528 effects.push(Effect::SpillClear {
20529 anchor_vertex: vertex_id,
20530 });
20531 }
20532 effects.push(Effect::SpillCommit {
20533 anchor_vertex: vertex_id,
20534 anchor_cell: anchor,
20535 target_cells: targets,
20536 values: rows,
20537 });
20538 effects.push(Effect::WriteCell {
20539 vertex_id,
20540 value: top_left,
20541 });
20542 Ok(effects)
20543 }
20544 Err(e) => {
20545 let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
20546 self.plan_spill_error_effects(vertex_id, &msg, h, w)
20547 }
20548 }
20549 }
20550
20551 fn spill_error_value(message: &str, expected_rows: u32, expected_cols: u32) -> LiteralValue {
20553 LiteralValue::Error(
20554 ExcelError::new(ExcelErrorKind::Spill)
20555 .with_message(message)
20556 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
20557 expected_rows,
20558 expected_cols,
20559 }),
20560 )
20561 }
20562
20563 fn plan_spill_error_effects(
20565 &mut self,
20566 vertex_id: VertexId,
20567 message: &str,
20568 expected_rows: u32,
20569 expected_cols: u32,
20570 ) -> Result<Vec<Effect>, ExcelError> {
20571 self.spill_mgr.release_owner(vertex_id);
20572 let spill_val = Self::spill_error_value(message, expected_rows, expected_cols);
20573
20574 let effects = vec![
20575 Effect::SpillClear {
20576 anchor_vertex: vertex_id,
20577 },
20578 Effect::WriteCell {
20579 vertex_id,
20580 value: spill_val,
20581 },
20582 ];
20583 Ok(effects)
20584 }
20585
20586 pub(crate) fn apply_effect(
20588 &mut self,
20589 effect: &Effect,
20590 delta: Option<&mut DeltaCollector>,
20591 log: Option<&mut ChangeLog>,
20592 ) -> Result<(), ExcelError> {
20593 self.apply_effect_with_computed_writes(effect, delta, log, None)
20594 }
20595
20596 fn apply_effect_with_computed_writes(
20597 &mut self,
20598 effect: &Effect,
20599 delta: Option<&mut DeltaCollector>,
20600 log: Option<&mut ChangeLog>,
20601 computed_writes: Option<&mut ComputedWriteBuffer>,
20602 ) -> Result<(), ExcelError> {
20603 match effect {
20604 Effect::WriteCell { vertex_id, value } => {
20605 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
20606 }
20607 Effect::SpillClear { anchor_vertex } => {
20608 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
20609 }
20610 Effect::SpillCommit {
20611 anchor_vertex,
20612 anchor_cell: _,
20613 target_cells,
20614 values,
20615 } => {
20616 self.apply_spill_commit(
20617 *anchor_vertex,
20618 target_cells,
20619 values.clone(),
20620 delta,
20621 log,
20622 computed_writes,
20623 )?;
20624 }
20625 }
20626 Ok(())
20627 }
20628
20629 fn apply_write_cell(
20631 &mut self,
20632 vertex_id: VertexId,
20633 value: &LiteralValue,
20634 delta: Option<&mut DeltaCollector>,
20635 mut computed_writes: Option<&mut ComputedWriteBuffer>,
20636 ) -> Result<(), ExcelError> {
20637 if let Some(d) = delta
20638 && d.mode != DeltaMode::Off
20639 {
20640 if let Some(buffer) = computed_writes.as_deref_mut() {
20641 self.flush_computed_write_buffer(buffer)?;
20642 }
20643 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
20644 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20645 let old = self
20646 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20647 .unwrap_or(LiteralValue::Empty);
20648 if old != *value {
20649 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20650 }
20651 }
20652 }
20653 self.graph.update_vertex_value_ref(vertex_id, value);
20654 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
20655 Ok(())
20656 }
20657
20658 fn apply_spill_clear(
20660 &mut self,
20661 anchor_vertex: VertexId,
20662 delta: Option<&mut DeltaCollector>,
20663 log: Option<&mut ChangeLog>,
20664 computed_writes: Option<&mut ComputedWriteBuffer>,
20665 ) -> Result<(), ExcelError> {
20666 if let Some(buffer) = computed_writes {
20667 self.flush_computed_write_buffer(buffer)?;
20668 }
20669
20670 let spill_cells = self.graph.spill_cells_to_clear(anchor_vertex);
20671 if spill_cells.is_empty() {
20672 return Ok(());
20673 }
20674
20675 let snapshot = if log.is_some() {
20677 self.snapshot_spill_for_anchor(anchor_vertex)
20678 } else {
20679 None
20680 };
20681
20682 if let Some(d) = delta
20684 && d.mode != DeltaMode::Off
20685 {
20686 let empty = LiteralValue::Empty;
20687 for cell in spill_cells.iter() {
20688 let sheet_name = self.graph.sheet_name(cell.sheet_id);
20689 let old = self
20690 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20691 .unwrap_or(LiteralValue::Empty);
20692 if old != empty {
20693 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
20694 }
20695 }
20696 }
20697
20698 self.graph.clear_spill_region(anchor_vertex);
20699 if let Some(scope) = Self::structural_scope_from_cells(&spill_cells) {
20700 self.record_structural_change(scope);
20701 }
20702
20703 if self.config.arrow_storage_enabled
20705 && self.config.delta_overlay_enabled
20706 && self.config.write_formula_overlay_enabled
20707 {
20708 let empty = LiteralValue::Empty;
20709 for cell in spill_cells.iter() {
20710 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
20711 self.mirror_value_to_computed_overlay(
20712 &sheet_name,
20713 cell.coord.row() + 1,
20714 cell.coord.col() + 1,
20715 &empty,
20716 );
20717 }
20718 }
20719
20720 if let Some(log) = log
20722 && let Some(old) = snapshot
20723 {
20724 log.record(ChangeEvent::SpillCleared {
20725 anchor: anchor_vertex,
20726 old,
20727 });
20728 }
20729 Ok(())
20730 }
20731
20732 fn apply_spill_commit(
20734 &mut self,
20735 anchor_vertex: VertexId,
20736 target_cells: &[CellRef],
20737 values: Vec<Vec<LiteralValue>>,
20738 delta: Option<&mut DeltaCollector>,
20739 log: Option<&mut ChangeLog>,
20740 computed_writes: Option<&mut ComputedWriteBuffer>,
20741 ) -> Result<(), ExcelError> {
20742 self.guard_pending_spill_commit(anchor_vertex, target_cells)?;
20743 if let Some(buffer) = computed_writes {
20744 self.flush_computed_write_buffer(buffer)?;
20745 }
20746
20747 let old_snapshot = if log.is_some() {
20749 self.snapshot_spill_for_anchor(anchor_vertex)
20750 } else {
20751 None
20752 };
20753
20754 self.commit_spill_and_mirror(anchor_vertex, target_cells, values.clone(), delta)?;
20756
20757 if let Some(log) = log {
20759 log.record(ChangeEvent::SpillCommitted {
20760 anchor: anchor_vertex,
20761 old: old_snapshot,
20762 new: SpillSnapshot {
20763 target_cells: target_cells.to_vec(),
20764 values,
20765 },
20766 });
20767 }
20768 Ok(())
20769 }
20770
20771 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
20776 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
20777 if cells.is_empty() {
20778 return None;
20779 }
20780
20781 let max = self.config.spill.max_spill_cells as usize;
20782 let mut cells = cells;
20783 if cells.len() > max {
20784 cells.truncate(max);
20785 }
20786
20787 let first = *cells.first().expect("non-empty spill cells");
20788 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
20789 let row0 = first.coord.row();
20790 let col0 = first.coord.col();
20791
20792 let mut max_row = row0;
20793 let mut max_col = col0;
20794 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
20795 for cell in &cells {
20796 max_row = max_row.max(cell.coord.row());
20797 max_col = max_col.max(cell.coord.col());
20798 let v = self
20799 .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
20800 .unwrap_or(LiteralValue::Empty);
20801 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
20802 }
20803
20804 let rows = (max_row - row0 + 1) as usize;
20805 let cols = (max_col - col0 + 1) as usize;
20806 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
20807 for r in 0..rows {
20808 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
20809 for c in 0..cols {
20810 row.push(
20811 by_coord
20812 .get(&(row0 + r as u32, col0 + c as u32))
20813 .cloned()
20814 .unwrap_or(LiteralValue::Empty),
20815 );
20816 }
20817 values.push(row);
20818 }
20819
20820 Some(SpillSnapshot {
20821 target_cells: cells,
20822 values,
20823 })
20824 }
20825
20826 fn flush_before_range_dependent_vertex(
20827 &mut self,
20828 vertex_id: VertexId,
20829 computed_writes: &mut ComputedWriteBuffer,
20830 ) -> Result<(), ExcelError> {
20831 if self.graph.reads_compressed_range(vertex_id) {
20832 self.flush_computed_write_buffer(computed_writes)?;
20833 }
20834 Ok(())
20835 }
20836
20837 fn plan_vertex_effects_with_computed_flush(
20838 &mut self,
20839 vertex_id: VertexId,
20840 computed_value: LiteralValue,
20841 computed_writes: &mut ComputedWriteBuffer,
20842 ) -> Result<Vec<Effect>, ExcelError> {
20843 if matches!(&computed_value, LiteralValue::Array(_)) {
20844 self.flush_computed_write_buffer(computed_writes)?;
20845 }
20846 self.plan_vertex_effects(vertex_id, computed_value)
20847 }
20848
20849 fn evaluate_small_layer_direct_effects(
20852 &mut self,
20853 layer: &super::scheduler::Layer,
20854 delta: Option<&mut DeltaCollector>,
20855 log: Option<&mut ChangeLog>,
20856 cancel_flag: Option<&AtomicBool>,
20857 cancel_check_every: usize,
20858 cancel_message: &'static str,
20859 ) -> Result<usize, ExcelError> {
20860 let cancel = cancel_flag.map(|flag| (flag, cancel_check_every, cancel_message));
20861 self.evaluate_layer_units(layer, delta, log, cancel, false)
20862 }
20863
20864 fn evaluate_layer_units(
20871 &mut self,
20872 layer: &super::scheduler::Layer,
20873 delta: Option<&mut DeltaCollector>,
20874 log: Option<&mut ChangeLog>,
20875 cancel: Option<(&AtomicBool, usize, &'static str)>,
20876 buffered: bool,
20877 ) -> Result<usize, ExcelError> {
20878 self.evaluate_layer_units_until(layer, delta, log, cancel, buffered, None)
20879 }
20880
20881 fn evaluate_layer_units_until(
20885 &mut self,
20886 layer: &super::scheduler::Layer,
20887 mut delta: Option<&mut DeltaCollector>,
20888 mut log: Option<&mut ChangeLog>,
20889 cancel: Option<(&AtomicBool, usize, &'static str)>,
20890 buffered: bool,
20891 stop_at: Option<crate::instant::FzInstant>,
20892 ) -> Result<usize, ExcelError> {
20893 let mut chain_values = None;
20896 if layer.sequential && !layer.runs.is_empty() {
20897 chain_values = match layer.runs.as_slice() {
20898 [run] if run.start == 0 && run.len as usize == layer.vertices.len() => {
20899 self.try_chain_lift(*run, &layer.vertices)
20900 }
20901 _ => None,
20902 };
20903 if chain_values.is_none() {
20904 let cells = super::scheduler::Layer {
20905 vertices: layer.vertices.clone(),
20906 runs: Vec::new(),
20907 sequential: true,
20908 };
20909 return self.evaluate_layer_units_until(&cells, delta, log, cancel, false, stop_at);
20910 }
20911 }
20912 let chained = chain_values.is_some();
20913 let buffered = buffered || chained;
20915 let track_unflushed = buffered
20922 && self.freshness_armed()
20923 && layer.vertices.iter().any(|&v| self.graph.is_dynamic(v));
20924 let mut committed_unit: &[VertexId] = &[];
20925 let mut computed_writes = ComputedWriteBuffer::default();
20926 let mut next_check = 0usize;
20927 let mut done = 0usize;
20928 for unit in layer_units(layer) {
20929 if done > 0
20930 && let Some(stop_at) = stop_at
20931 && crate::instant::FzInstant::now() >= stop_at
20932 {
20933 self.flush_computed_write_buffer(&mut computed_writes)?;
20934 if track_unflushed {
20935 self.freshness_flushed();
20936 }
20937 return Ok(done);
20938 }
20939 if let Some((flag, every, message)) = cancel
20940 && every > 0
20941 && done >= next_check
20942 {
20943 next_check = (done / every + 1) * every;
20944 if flag.load(Ordering::Relaxed) {
20945 if buffered {
20946 self.flush_computed_write_buffer(&mut computed_writes)?;
20947 }
20948 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
20949 .with_message(message.to_string()));
20950 }
20951 }
20952 if buffered && self.unit_reads_compressed_range(layer, unit) {
20953 self.flush_computed_write_buffer(&mut computed_writes)?;
20954 }
20955 if track_unflushed {
20959 if computed_writes.is_empty() {
20960 self.freshness_flushed();
20961 } else {
20962 self.freshness_note_unflushed(committed_unit);
20963 }
20964 committed_unit = unit_members(layer, unit);
20965 }
20966 let values = match (chain_values.take(), unit) {
20967 (Some(chain), LayerUnit::Run(run)) => {
20968 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
20969 self.flush_computed_write_buffer(&mut computed_writes)?;
20970 return Err(e);
20971 }
20972 let members =
20973 &layer.vertices[run.start as usize..(run.start + run.len) as usize];
20974 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
20975 match self.commit_run_numbers(
20976 run,
20977 members,
20978 &chain,
20979 delta_active,
20980 Some(&mut computed_writes),
20981 ) {
20982 Ok(true) => {
20983 done += chain.len();
20984 continue;
20985 }
20986 Ok(false) => {}
20987 Err(e) => {
20988 self.flush_computed_write_buffer(&mut computed_writes)?;
20989 return Err(e);
20990 }
20991 }
20992 members
20993 .iter()
20994 .copied()
20995 .zip(chain.into_iter().map(LiteralValue::Number))
20996 .collect()
20997 }
20998 (_, unit) => self.evaluate_unit_immutable(layer, unit),
20999 };
21000 if let Err(e) = self.live_cancellation_after_work(UNIT_CANCELLED) {
21003 self.flush_computed_write_buffer(&mut computed_writes)?;
21004 return Err(e);
21005 }
21006 done += values.len();
21007 if let LayerUnit::Run(run) = unit {
21008 let members = &layer.vertices[run.start as usize..(run.start + run.len) as usize];
21009 let delta_active = delta.as_deref().is_some_and(|d| d.mode != DeltaMode::Off);
21010 let committed = self.commit_run_scalars(
21011 run,
21012 members,
21013 &values,
21014 delta_active,
21015 buffered.then_some(&mut computed_writes),
21016 );
21017 match committed {
21018 Ok(true) => continue,
21019 Ok(false) => {}
21020 Err(e) => {
21021 self.flush_computed_write_buffer(&mut computed_writes)?;
21022 return Err(e);
21023 }
21024 }
21025 }
21026 let guarded = self.freshness_begin_batch_commit(&values);
21029 for (vertex_id, value) in values {
21030 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21031 let effects = if buffered {
21032 self.plan_vertex_effects_with_computed_flush(
21033 vertex_id,
21034 value,
21035 &mut computed_writes,
21036 )
21037 } else {
21038 self.plan_vertex_effects(vertex_id, value)
21039 };
21040 let effects = match effects {
21041 Ok(effects) => effects,
21042 Err(e) => {
21043 self.flush_computed_write_buffer(&mut computed_writes)?;
21044 return Err(e);
21045 }
21046 };
21047 for effect in &effects {
21048 if let Err(e) = self.apply_effect_with_computed_writes(
21049 effect,
21050 delta.as_deref_mut(),
21051 log.as_deref_mut(),
21052 buffered.then_some(&mut computed_writes),
21053 ) {
21054 self.flush_computed_write_buffer(&mut computed_writes)?;
21055 return Err(e);
21056 }
21057 }
21058 self.freshness_keep_redirtied(redirtied, vertex_id);
21059 }
21060 }
21061 self.flush_computed_write_buffer(&mut computed_writes)?;
21062 if track_unflushed {
21063 self.freshness_flushed();
21064 }
21065 #[cfg(debug_assertions)]
21068 if chained {
21069 for &v in &layer.vertices {
21070 let cell = self.graph.get_cell_ref(v);
21071 let (sheet, row, col) = cell
21072 .map(|c| {
21073 (
21074 self.graph.sheet_name(c.sheet_id).to_string(),
21075 c.coord.row() + 1,
21076 c.coord.col() + 1,
21077 )
21078 })
21079 .expect("chain member cell");
21080 let written = self.get_cell_value(&sheet, row, col);
21081 let oracle = self
21082 .evaluate_vertex_immutable(v)
21083 .unwrap_or_else(LiteralValue::Error);
21084 assert!(
21085 written
21086 .as_ref()
21087 .is_some_and(|w| same_value_bits(w, &oracle)),
21088 "chain member {sheet}!R{row}C{col}: {written:?} vs per-cell {oracle:?}"
21089 );
21090 }
21091 }
21092 #[cfg(not(debug_assertions))]
21093 let _ = chained;
21094 Ok(layer.vertices.len())
21095 }
21096
21097 fn evaluate_layer_sequential_effects(
21099 &mut self,
21100 layer: &super::scheduler::Layer,
21101 ) -> Result<usize, ExcelError> {
21102 let buffered = buffer_layer_writes(layer);
21103 self.evaluate_layer_units(layer, None, None, None, buffered)
21104 }
21105
21106 fn evaluate_layer_sequential_with_delta_effects(
21108 &mut self,
21109 layer: &super::scheduler::Layer,
21110 delta: &mut DeltaCollector,
21111 ) -> Result<usize, ExcelError> {
21112 let buffered = buffer_layer_writes(layer);
21113 self.evaluate_layer_units(layer, Some(delta), None, None, buffered)
21114 }
21115
21116 fn evaluate_layer_sequential_cancellable_effects(
21118 &mut self,
21119 layer: &super::scheduler::Layer,
21120 cancel_flag: &AtomicBool,
21121 ) -> Result<usize, ExcelError> {
21122 let buffered = buffer_layer_writes(layer);
21123 let cancel = (cancel_flag, 256, "Evaluation cancelled within layer");
21124 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21125 }
21126
21127 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
21129 &mut self,
21130 layer: &super::scheduler::Layer,
21131 cancel_flag: &AtomicBool,
21132 ) -> Result<usize, ExcelError> {
21133 let buffered = buffer_layer_writes(layer);
21134 let cancel = (
21135 cancel_flag,
21136 128,
21137 "Demand-driven evaluation cancelled within layer",
21138 );
21139 self.evaluate_layer_units(layer, None, None, Some(cancel), buffered)
21140 }
21141
21142 fn evaluate_layer_parallel_effects(
21144 &mut self,
21145 layer: &super::scheduler::Layer,
21146 min_chunk: u32,
21147 ) -> Result<usize, ExcelError> {
21148 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21149
21150 let phases = self.parallel_phases(layer);
21151 let mut applied = 0usize;
21152
21153 for (units, group) in &phases {
21154 let group = &group[..];
21155 if group.is_empty() {
21156 continue;
21157 }
21158 let mut computed_writes = ComputedWriteBuffer::default();
21159
21160 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21161 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, min_chunk));
21162
21163 let results = results.and_then(|results| {
21166 self.live_cancellation_after_work(GROUP_CANCELLED)
21167 .map(|()| results)
21168 });
21169 self.freshness_gate_group(group);
21172 match results {
21173 Ok(vertex_results) => {
21174 let (vertex_results, committed) = self.commit_parallel_runs(
21175 layer,
21176 units,
21177 vertex_results,
21178 false,
21179 &mut computed_writes,
21180 )?;
21181 applied = applied.saturating_add(committed);
21182 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21187 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21188 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21189 for (vertex_id, result) in vertex_results {
21190 if matches!(result, LiteralValue::Array(_)) {
21191 arrays.push((vertex_id, result));
21192 } else {
21193 others.push((vertex_id, result));
21194 }
21195 }
21196 for (vertex_id, result) in arrays {
21197 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21198 let effects = match self.plan_vertex_effects_with_computed_flush(
21199 vertex_id,
21200 result,
21201 &mut computed_writes,
21202 ) {
21203 Ok(effects) => effects,
21204 Err(e) => {
21205 self.flush_computed_write_buffer(&mut computed_writes)?;
21206 return Err(e);
21207 }
21208 };
21209 for effect in &effects {
21210 if let Err(e) = self.apply_effect_with_computed_writes(
21211 effect,
21212 None,
21213 None,
21214 Some(&mut computed_writes),
21215 ) {
21216 self.flush_computed_write_buffer(&mut computed_writes)?;
21217 return Err(e);
21218 }
21219 }
21220 self.freshness_keep_redirtied(redirtied, vertex_id);
21221 applied = applied.saturating_add(1);
21222 }
21223 self.flush_computed_write_buffer(&mut computed_writes)?;
21225 for (vertex_id, result) in others {
21226 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21227 let effects = match self.plan_vertex_effects_with_computed_flush(
21228 vertex_id,
21229 result,
21230 &mut computed_writes,
21231 ) {
21232 Ok(effects) => effects,
21233 Err(e) => {
21234 self.flush_computed_write_buffer(&mut computed_writes)?;
21235 return Err(e);
21236 }
21237 };
21238 for effect in &effects {
21239 if let Err(e) = self.apply_effect_with_computed_writes(
21240 effect,
21241 None,
21242 None,
21243 Some(&mut computed_writes),
21244 ) {
21245 self.flush_computed_write_buffer(&mut computed_writes)?;
21246 return Err(e);
21247 }
21248 }
21249 self.freshness_keep_redirtied(redirtied, vertex_id);
21250 applied = applied.saturating_add(1);
21251 }
21252 self.flush_computed_write_buffer(&mut computed_writes)?;
21254 }
21255 Err(e) => {
21256 self.flush_computed_write_buffer(&mut computed_writes)?;
21257 return Err(e);
21258 }
21259 }
21260 }
21261
21262 Ok(applied)
21263 }
21264
21265 fn evaluate_layer_parallel_with_delta_effects(
21267 &mut self,
21268 layer: &super::scheduler::Layer,
21269 delta: &mut DeltaCollector,
21270 ) -> Result<usize, ExcelError> {
21271 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21272
21273 let phases = self.parallel_phases(layer);
21274 let mut applied = 0usize;
21275
21276 for (units, group) in &phases {
21277 let group = &group[..];
21278 if group.is_empty() {
21279 continue;
21280 }
21281 let mut computed_writes = ComputedWriteBuffer::default();
21282 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
21283 thread_pool.install(|| self.evaluate_units_parallel(layer, units, None, 8));
21284
21285 let results = results.and_then(|results| {
21288 self.live_cancellation_after_work(GROUP_CANCELLED)
21289 .map(|()| results)
21290 });
21291 self.freshness_gate_group(group);
21294 match results {
21295 Ok(vertex_results) => {
21296 let (vertex_results, committed) = self.commit_parallel_runs(
21297 layer,
21298 units,
21299 vertex_results,
21300 delta.mode != DeltaMode::Off,
21301 &mut computed_writes,
21302 )?;
21303 applied = applied.saturating_add(committed);
21304 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21307 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21308 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21309 for (vertex_id, result) in vertex_results {
21310 if matches!(result, LiteralValue::Array(_)) {
21311 arrays.push((vertex_id, result));
21312 } else {
21313 others.push((vertex_id, result));
21314 }
21315 }
21316 for (vertex_id, result) in arrays {
21317 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21318 let effects = match self.plan_vertex_effects_with_computed_flush(
21319 vertex_id,
21320 result,
21321 &mut computed_writes,
21322 ) {
21323 Ok(effects) => effects,
21324 Err(e) => {
21325 self.flush_computed_write_buffer(&mut computed_writes)?;
21326 return Err(e);
21327 }
21328 };
21329 for effect in &effects {
21330 if let Err(e) = self.apply_effect_with_computed_writes(
21331 effect,
21332 Some(delta),
21333 None,
21334 Some(&mut computed_writes),
21335 ) {
21336 self.flush_computed_write_buffer(&mut computed_writes)?;
21337 return Err(e);
21338 }
21339 }
21340 self.freshness_keep_redirtied(redirtied, vertex_id);
21341 applied = applied.saturating_add(1);
21342 }
21343 self.flush_computed_write_buffer(&mut computed_writes)?;
21344 for (vertex_id, result) in others {
21345 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21346 let effects = match self.plan_vertex_effects_with_computed_flush(
21347 vertex_id,
21348 result,
21349 &mut computed_writes,
21350 ) {
21351 Ok(effects) => effects,
21352 Err(e) => {
21353 self.flush_computed_write_buffer(&mut computed_writes)?;
21354 return Err(e);
21355 }
21356 };
21357 for effect in &effects {
21358 if let Err(e) = self.apply_effect_with_computed_writes(
21359 effect,
21360 Some(delta),
21361 None,
21362 Some(&mut computed_writes),
21363 ) {
21364 self.flush_computed_write_buffer(&mut computed_writes)?;
21365 return Err(e);
21366 }
21367 }
21368 self.freshness_keep_redirtied(redirtied, vertex_id);
21369 applied = applied.saturating_add(1);
21370 }
21371 self.flush_computed_write_buffer(&mut computed_writes)?;
21372 }
21373 Err(e) => {
21374 self.flush_computed_write_buffer(&mut computed_writes)?;
21375 return Err(e);
21376 }
21377 }
21378 }
21379
21380 Ok(applied)
21381 }
21382
21383 fn evaluate_layer_parallel_cancellable_effects(
21385 &mut self,
21386 layer: &super::scheduler::Layer,
21387 cancel_flag: &AtomicBool,
21388 ) -> Result<usize, ExcelError> {
21389 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
21390
21391 if cancel_flag.load(Ordering::Relaxed) {
21392 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21393 .with_message("Parallel evaluation cancelled before starting".to_string()));
21394 }
21395
21396 let phases = self.parallel_phases(layer);
21397 let mut applied = 0usize;
21398
21399 for (units, group) in &phases {
21400 let group = &group[..];
21401 if group.is_empty() {
21402 continue;
21403 }
21404 let mut computed_writes = ComputedWriteBuffer::default();
21405
21406 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> = thread_pool
21407 .install(|| self.evaluate_units_parallel(layer, units, Some(cancel_flag), 8));
21408
21409 let results = results.and_then(|results| {
21412 self.live_cancellation_after_work(GROUP_CANCELLED)
21413 .map(|()| results)
21414 });
21415 self.freshness_gate_group(group);
21418 match results {
21419 Ok(vertex_results) => {
21420 let (vertex_results, committed) = self.commit_parallel_runs(
21421 layer,
21422 units,
21423 vertex_results,
21424 false,
21425 &mut computed_writes,
21426 )?;
21427 applied = applied.saturating_add(committed);
21428 let guarded = self.freshness_begin_batch_commit(&vertex_results);
21431 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
21432 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
21433 for (vertex_id, result) in vertex_results {
21434 if matches!(result, LiteralValue::Array(_)) {
21435 arrays.push((vertex_id, result));
21436 } else {
21437 others.push((vertex_id, result));
21438 }
21439 }
21440 for (vertex_id, result) in arrays {
21441 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21442 let effects = match self.plan_vertex_effects_with_computed_flush(
21443 vertex_id,
21444 result,
21445 &mut computed_writes,
21446 ) {
21447 Ok(effects) => effects,
21448 Err(e) => {
21449 self.flush_computed_write_buffer(&mut computed_writes)?;
21450 return Err(e);
21451 }
21452 };
21453 for effect in &effects {
21454 if let Err(e) = self.apply_effect_with_computed_writes(
21455 effect,
21456 None,
21457 None,
21458 Some(&mut computed_writes),
21459 ) {
21460 self.flush_computed_write_buffer(&mut computed_writes)?;
21461 return Err(e);
21462 }
21463 }
21464 self.freshness_keep_redirtied(redirtied, vertex_id);
21465 applied = applied.saturating_add(1);
21466 }
21467 self.flush_computed_write_buffer(&mut computed_writes)?;
21468 for (vertex_id, result) in others {
21469 let redirtied = self.freshness_batch_redirtied(guarded, vertex_id);
21470 let effects = match self.plan_vertex_effects_with_computed_flush(
21471 vertex_id,
21472 result,
21473 &mut computed_writes,
21474 ) {
21475 Ok(effects) => effects,
21476 Err(e) => {
21477 self.flush_computed_write_buffer(&mut computed_writes)?;
21478 return Err(e);
21479 }
21480 };
21481 for effect in &effects {
21482 if let Err(e) = self.apply_effect_with_computed_writes(
21483 effect,
21484 None,
21485 None,
21486 Some(&mut computed_writes),
21487 ) {
21488 self.flush_computed_write_buffer(&mut computed_writes)?;
21489 return Err(e);
21490 }
21491 }
21492 self.freshness_keep_redirtied(redirtied, vertex_id);
21493 applied = applied.saturating_add(1);
21494 }
21495 self.flush_computed_write_buffer(&mut computed_writes)?;
21496 }
21497 Err(e) => {
21498 self.flush_computed_write_buffer(&mut computed_writes)?;
21499 return Err(e);
21500 }
21501 }
21502 }
21503
21504 Ok(applied)
21505 }
21506
21507 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
21514 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
21515 engine.evaluate_all_logged_unobserved(log)
21516 })
21517 }
21518
21519 fn evaluate_all_logged_unobserved(
21520 &mut self,
21521 log: &mut ChangeLog,
21522 ) -> Result<EvalResult, ExcelError> {
21523 self.observe_function_semantic_epoch()?;
21524 let _source_cache = self.source_cache_session();
21525 self.validate_deterministic_mode()?;
21526 if self.config.defer_graph_building {
21527 self.build_graph_all()?;
21528 }
21529 self.require_unified_authority()?;
21530 self.begin_evaluation_request();
21531 self.reset_virtual_dep_telemetry_if_disabled();
21532 let start = crate::instant::FzInstant::now();
21533 let mut computed_vertices = 0;
21534 let mut cycle_errors = 0;
21535
21536 let mut replan_iterations = 0;
21537 const MAX_REPLAN: usize = 5;
21538 let mut telemetry = self
21539 .config
21540 .enable_virtual_dep_telemetry
21541 .then(|| self.start_virtual_dep_telemetry());
21542
21543 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
21544
21545 let result = (|| -> Result<EvalResult, ExcelError> {
21546 loop {
21547 let to_evaluate = self.graph.get_evaluation_vertices();
21548 if to_evaluate.is_empty() {
21549 if let Some(t) = telemetry.as_mut()
21550 && t.bailout_reason.is_none()
21551 {
21552 t.bailout_reason = Some("no_work");
21553 }
21554 break;
21555 }
21556
21557 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21558 if let Some(t) = telemetry.as_mut() {
21559 Self::accumulate_schedule_meta(t, &meta);
21560 }
21561
21562 self.begin_pass(&schedule);
21565 for (unit_index, &unit) in schedule.units.iter().enumerate() {
21566 match unit {
21567 ScheduleUnit::Cycle(i) => {
21568 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
21579 {
21580 cycle_errors += 1;
21581 }
21582 }
21583 ScheduleUnit::Layer(i) => {
21584 computed_vertices +=
21585 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
21586 }
21587 }
21588 if self.stop_after_unit(&schedule, unit_index) {
21589 break;
21590 }
21591 }
21592
21593 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21594 if let Some(t) = telemetry.as_mut() {
21595 t.changed_vdeps_total += changed_vertices.len();
21596 }
21597 self.resource_checkpoint(0)?;
21598 if !self.finish_pass_dirty(&to_evaluate, &changed_vertices) {
21599 if let Some(t) = telemetry.as_mut() {
21600 t.bailout_reason = Some("converged");
21601 }
21602 break;
21603 }
21604 if replan_iterations >= MAX_REPLAN {
21605 if let Some(mut t) = telemetry.take() {
21606 t.bailout_reason = Some("max_replan");
21607 t.replan_iterations = replan_iterations;
21608 self.last_virtual_dep_telemetry = t;
21609 }
21610 return Err(
21611 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21612 );
21613 }
21614 replan_iterations += 1;
21615 }
21616
21617 if let Some(mut t) = telemetry {
21618 t.replan_iterations = replan_iterations;
21619 self.last_virtual_dep_telemetry = t;
21620 }
21621
21622 self.redirty_for_next_recalc();
21623 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21624
21625 Ok(EvalResult {
21626 computed_vertices,
21627 cycle_errors,
21628 elapsed: start.elapsed(),
21629 })
21630 })();
21631 log.end_compound();
21632 result
21633 }
21634
21635 fn evaluate_layer_logged(
21637 &mut self,
21638 layer: &super::scheduler::Layer,
21639 log: &mut ChangeLog,
21640 ) -> Result<usize, ExcelError> {
21641 self.resource_checkpoint(layer.vertices.len() as u64)?;
21642 self.evaluate_layer_units(layer, None, Some(log), None, true)
21643 }
21644}