1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::prepared_legacy_graph::{
8 PreparedLegacyGraphError, PreparedLegacyGraphPlan,
9};
10use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
11use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
12use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
13use crate::engine::live_graph::analyze_live_graph;
14use crate::engine::lookup_index_cache::{
15 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
16 LookupIndexKey, estimate_bytes,
17};
18use crate::engine::named_range::{NameScope, NamedDefinition};
19use crate::engine::range_view::RangeView;
20use crate::engine::row_visibility::RowVisibilityState;
21use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
22use crate::engine::target_preparation::{
23 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
24};
25use crate::engine::used_extent::{
26 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
27};
28use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
29use crate::engine::{
30 CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
31 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
32 EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
33 FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
34 FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
35 FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
36 FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
37 VertexId, VertexKind, VisibilityMaskMode,
38};
39use crate::formula_plane::placement::prepare_anchor_once_fragment;
40use crate::formula_plane::placement::{
41 CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
42 place_candidate_family_with_analyses, prepare_anchor_once_family,
43 split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
44 validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
45};
46use crate::formula_plane::producer::{
47 DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
48 FormulaProducerWork, ProducerDirtyDomain, SpanReadSummary,
49};
50use crate::formula_plane::region_index::{DirtyDomain, Region};
51use crate::formula_plane::runtime::{
52 FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
53};
54use crate::formula_plane::scheduler::{
55 MixedSchedule, MixedScheduleFallbackReason, MixedTopology, MixedTopologyCompileResult,
56 MixedTopologyCompileStats, MixedTopologyConfig, build_demand_closure_cached,
57 build_demand_closure_in_memory_runs, build_demand_closure_paged,
58 build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
59 schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
60 schedule_dirty_work_repeated_passes,
61};
62#[cfg(not(target_arch = "wasm32"))]
63use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
64#[cfg(not(target_arch = "wasm32"))]
65use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
66#[cfg(test)]
67use crate::formula_plane::span_eval::SpanEvalReport;
68use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
69use crate::formula_plane::structural::relocate_ast_for_template_placement;
70use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
71use crate::function::FnCaps;
72use crate::interpreter::Interpreter;
73use crate::reference::{CellRef, Coord, RangeRef};
74use crate::traits::FunctionProvider;
75use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
76use formualizer_common::{
77 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
78};
79use formualizer_parse::parser::ReferenceType;
80use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
81use rayon::ThreadPoolBuilder;
82use rustc_hash::{FxHashMap, FxHashSet};
83use std::collections::{BTreeMap, BTreeSet, VecDeque};
84use std::sync::Arc;
85use std::sync::atomic::{AtomicBool, Ordering};
86use std::time::Instant;
87
88type StagedFormulaEntry = (u32, u32, String);
89type StagedSheetParts = (
90 Vec<StagedFormulaEntry>,
91 Option<crate::engine::DeferredFormulaPackage>,
92);
93
94#[derive(Default)]
104pub(crate) struct StagedSheet {
105 entries: Vec<StagedFormulaEntry>,
106 index: FxHashMap<(u32, u32), usize>,
107 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
108}
109
110impl StagedSheet {
111 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
112 package
113 .invalidated
114 .extend(package.families.iter().map(|family| family.source_id));
115 package.invalidated.extend(
116 package
117 .partitioned_families
118 .iter()
119 .map(|family| family.source_id),
120 );
121 }
122
123 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
124 let Some(package) = self.deferred_package.as_mut() else {
125 return;
126 };
127 let family = package
128 .replay
129 .lock()
130 .ok()
131 .and_then(|mut replay| replay.formula_at(row, col).ok())
132 .flatten()
133 .and_then(|record| record.partition_owner.or(record.family));
134 if let Some(family) = family {
135 package.invalidated.insert(family);
136 } else {
137 Self::invalidate_all_deferred_families(package);
141 }
142 package.suppressed.insert((row, col));
143 }
144
145 fn reconcile_attached_deferred_package(&mut self) {
146 let Some(package) = self.deferred_package.as_mut() else {
147 return;
148 };
149 if self.entries.is_empty() {
150 return;
151 }
152
153 let replayed = (|| {
154 let mut disposition = crate::engine::FormulaReplayDisposition::default();
155 for partition in &package.partitioned_families {
156 disposition.register_partition(partition, false)?;
157 }
158 package
159 .replay
160 .lock()
161 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
162 .replay_partitioned(&disposition, &package.partitioned_families)
163 })();
164
165 let Ok(records) = replayed else {
166 Self::invalidate_all_deferred_families(package);
167 package
168 .suppressed
169 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
170 return;
171 };
172 let mut owners = FxHashMap::default();
173 for record in &records {
174 owners
175 .entry((record.row, record.col))
176 .or_insert(record.partition_owner.or(record.family));
177 }
178 for (row, col, _) in &self.entries {
179 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
180 package.invalidated.insert(family);
181 } else {
182 Self::invalidate_all_deferred_families(package);
183 }
184 package.suppressed.insert((*row, *col));
185 }
186 package.reconciliation_replay = Some(records);
187 }
188
189 fn stage(&mut self, row: u32, col: u32, text: String) {
190 self.invalidate_deferred_at(row, col);
191 match self.index.entry((row, col)) {
192 std::collections::hash_map::Entry::Occupied(slot) => {
193 self.entries[*slot.get()].2 = text;
194 }
195 std::collections::hash_map::Entry::Vacant(slot) => {
196 slot.insert(self.entries.len());
197 self.entries.push((row, col, text));
198 }
199 }
200 }
201
202 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
203 let deferred_text = self.get(row, col);
204 self.invalidate_deferred_at(row, col);
205 let Some(idx) = self.index.remove(&(row, col)) else {
206 return deferred_text;
207 };
208 let (_, _, text) = self.entries.remove(idx);
209 for slot in self.index.values_mut() {
211 if *slot > idx {
212 *slot -= 1;
213 }
214 }
215 Some(text)
216 }
217
218 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
219 self.index
220 .get(&(row, col))
221 .map(|&i| self.entries[i].2.as_str())
222 }
223
224 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
225 let idx = self.index.remove(&(row, col))?;
226 let (_, _, text) = self.entries.remove(idx);
227 for slot in self.index.values_mut() {
228 if *slot > idx {
229 *slot -= 1;
230 }
231 }
232 Some(text)
233 }
234
235 fn get(&self, row: u32, col: u32) -> Option<String> {
236 if let Some(text) = self.get_ordinary(row, col) {
237 return Some(text.to_string());
238 }
239 let package = self.deferred_package.as_ref()?;
240 if package.suppressed.contains(&(row, col)) {
241 return None;
242 }
243 package
244 .replay
245 .lock()
246 .ok()?
247 .formula_at(row, col)
248 .ok()?
249 .map(|record| record.text)
250 }
251
252 fn len(&self) -> usize {
253 self.deferred_package
254 .as_ref()
255 .map_or(self.entries.len(), |package| {
256 usize::try_from(package.report.source_formula_records_spooled)
257 .unwrap_or(usize::MAX)
258 .saturating_sub(package.suppressed.len())
259 .saturating_add(self.entries.len())
260 })
261 }
262
263 fn is_empty(&self) -> bool {
264 self.entries.is_empty() && self.deferred_package.is_none()
265 }
266
267 fn into_parts(self) -> StagedSheetParts {
268 (self.entries, self.deferred_package)
269 }
270}
271
272type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
273
274#[derive(Clone, Debug, PartialEq, Eq, Hash)]
275struct PreparationRegion {
276 sheet: String,
277 sheet_id: SheetId,
278 start_row: u32,
279 start_col: u32,
280 end_row: u32,
281 end_col: u32,
282}
283
284struct OrderedTargetProducers {
287 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
288 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
289}
290
291impl OrderedTargetProducers {
292 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
293 let mut ordered = Vec::new();
294 ordered.try_reserve(capacity)?;
295 let mut seen = FxHashSet::default();
296 seen.try_reserve(capacity)?;
297 Ok(Self { ordered, seen })
298 }
299
300 fn from_ordered(
301 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
302 ) -> Result<Self, std::collections::TryReserveError> {
303 let mut seen = FxHashSet::default();
304 seen.try_reserve(ordered.len())?;
305 seen.extend(ordered.iter().copied());
306 Ok(Self { ordered, seen })
307 }
308
309 fn push(
310 &mut self,
311 producer: crate::engine::target_preparation::TargetProducer,
312 ) -> Result<bool, std::collections::TryReserveError> {
313 #[cfg(test)]
314 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
315 if self.seen.contains(&producer) {
316 return Ok(false);
317 }
318 self.seen.try_reserve(1)?;
319 self.ordered.try_reserve(1)?;
320 self.seen.insert(producer);
321 self.ordered.push(producer);
322 Ok(true)
323 }
324
325 fn len(&self) -> usize {
326 self.ordered.len()
327 }
328
329 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
330 self.ordered
331 }
332}
333
334#[cfg(test)]
335thread_local! {
336 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
337}
338
339fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
340 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
341 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
342 limit: u64::MAX,
343 observed: observed as u64,
344 request_id,
345 })
346 .into_excel_error()
347}
348
349#[derive(Debug)]
350struct PreparedOrdinaryStagedFormula {
351 sheet: String,
352 sheet_id: SheetId,
353 lease: StagedFormulaLease,
354 ast_id: Option<AstNodeId>,
355 plan: Option<DependencyPlanRow>,
356}
357
358struct PreparedTargetSourcePackage {
359 sheet: String,
360 sheet_id: SheetId,
361 lease: StagedPackageLease,
362 source_report: crate::engine::FormulaCompressedSourceReport,
363 replay_records: Vec<crate::engine::DeferredReplayFormula>,
364 disposition: crate::engine::FormulaReplayDisposition,
365 placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
366 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
367 direct_families: usize,
368 direct_cells: u64,
369 direct_fragments: u64,
370 direct_complete_families: u64,
371 direct_complete_cells: u64,
372 direct_partition_families: u64,
373 direct_partition_cells: u64,
374 anchor_parses: u64,
375 anchor_asts: u64,
376 anchor_analyses: u64,
377}
378
379impl PreparedTargetSourcePackage {
380 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
381 self.replay_records.iter().filter(|record| {
382 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
383 return true;
384 };
385 let coord = crate::engine::SourceCoord { row, col };
386 let disposition = match record.family {
387 Some(family) => self.disposition.shared_disposition(family, coord),
388 None => self.disposition.ordinary_disposition(coord).0,
389 };
390 !matches!(
391 disposition,
392 crate::engine::FormulaReplayCoordinateDisposition::Direct
393 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
394 )
395 })
396 }
397}
398
399fn producer_dirty_to_span_dirty(
400 dirty: ProducerDirtyDomain,
401 span_ref: FormulaSpanRef,
402) -> DirtyDomain {
403 match dirty {
404 ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
405 ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
406 ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
407 }
408}
409type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
410type StagedFormulaBatches = Vec<(String, StagedSheet)>;
411#[derive(Debug)]
412pub(crate) struct SourceFamilyPreparationError {
413 reason: String,
414 parse_attempted: bool,
415 ast_created: bool,
416 analysis_created: bool,
417}
418
419impl SourceFamilyPreparationError {
420 fn contract(reason: &'static str) -> Self {
421 Self {
422 reason: reason.to_string(),
423 parse_attempted: false,
424 ast_created: false,
425 analysis_created: false,
426 }
427 }
428
429 fn parse(reason: &'static str) -> Self {
430 Self {
431 reason: reason.to_string(),
432 parse_attempted: true,
433 ast_created: false,
434 analysis_created: false,
435 }
436 }
437
438 fn ast(reason: impl Into<String>) -> Self {
439 Self {
440 reason: reason.into(),
441 parse_attempted: true,
442 ast_created: true,
443 analysis_created: false,
444 }
445 }
446
447 fn analysis(reason: impl Into<String>) -> Self {
448 Self {
449 reason: reason.into(),
450 parse_attempted: true,
451 ast_created: true,
452 analysis_created: true,
453 }
454 }
455}
456
457struct PreparedPartitionShadow {
458 fragment_count: u64,
459 direct_cells: u64,
460 fallback_cells: u64,
461 function_semantics_used: bool,
462}
463
464pub(crate) fn classify_mixed_topology_incomplete(
465 observed: &MixedTopologyCompileStats,
466) -> crate::engine::EvaluationIncompleteReason {
467 if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
468 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
469 } else if observed.memory_overflow_count > 0 {
470 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
471 } else if observed.edge_overflow_count > 0 {
472 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
473 } else if observed.candidate_overflow_count > 0 {
474 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
475 } else {
476 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
477 }
478}
479
480#[derive(Clone, Debug, Default)]
481struct LegacyIslandPlan {
482 membership: Vec<VertexId>,
483 dirty_vertices: Vec<VertexId>,
484 sheet_ids: Vec<SheetId>,
485 island_id: u64,
486 retained_bytes: usize,
487 boundary_relationships: usize,
488 omitted_relationships: usize,
489 omitted_direct_relationships: usize,
490}
491
492impl LegacyIslandPlan {
493 fn is_empty(&self) -> bool {
494 self.membership.is_empty()
495 }
496}
497
498type FormulaPlaneMixedScheduleBuild = (
499 MixedSchedule,
500 BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
501 u64,
502 Vec<VertexId>,
503 Vec<usize>,
504 LegacyIslandPlan,
505);
506
507#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub(crate) enum FormulaSpanDemotionFault {
509 #[default]
510 None,
511 AstPreparation,
512 LegacyGraphPreparation,
513 FinalLegacyGraphValidation,
514 FinalAuthorityValidation,
515 AllocationReservation,
516 BeforeFirstMutation,
517}
518
519#[derive(Debug)]
520pub(crate) enum FormulaSpanDemotionError {
521 StaleAuthority,
522 InvalidSpan,
523 CountOverflow,
524 Resource(ExcelError),
525 AstPreparation(ExcelError),
526 LegacyGraph(PreparedLegacyGraphError),
527 Injected(FormulaSpanDemotionFault),
528}
529
530impl std::fmt::Display for FormulaSpanDemotionError {
531 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
532 match self {
533 Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
534 Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
535 Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
536 Self::Resource(error) => write!(f, "{error}"),
537 Self::AstPreparation(error) => {
538 write!(f, "FormulaPlane demotion analysis failed: {error}")
539 }
540 Self::LegacyGraph(error) => {
541 write!(f, "FormulaPlane demotion graph plan failed: {error}")
542 }
543 Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
544 }
545 }
546}
547
548impl std::error::Error for FormulaSpanDemotionError {}
549
550pub(crate) struct PreparedFormulaSpanDemotion {
551 span_refs: Vec<FormulaSpanRef>,
552 expected_plane_epoch: u64,
553 expected_indexes_epoch: u64,
554 expected_indexed_plane_epoch: u64,
555 placement_count: usize,
556 legacy_graph: PreparedLegacyGraphPlan,
557 fault: FormulaSpanDemotionFault,
558}
559
560#[derive(Clone, Copy, Debug, PartialEq, Eq)]
561pub(crate) struct FormulaSpanDemotionReport {
562 pub(crate) spans_demoted: usize,
563 pub(crate) placements_materialized: usize,
564}
565
566type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
567type CompressedReplayBatch = (
568 FormulaIngestBatch,
569 crate::engine::FormulaCompressedSourceBatch,
570);
571type PreparedSourceBatch = (
572 FormulaIngestBatch,
573 crate::engine::FormulaCompressedSourceReport,
574 crate::engine::FormulaCompressedPreparation,
575);
576type PreparedStagedFormulaBatches = (
577 PreparedFormulaBatches,
578 Vec<CompressedReplayBatch>,
579 Vec<PreparedSourceBatch>,
580);
581
582#[doc(hidden)]
585pub struct SourceFormulaIngress<'a, R> {
586 engine: &'a mut Engine<R>,
587}
588
589impl<R> SourceFormulaIngress<'_, R>
590where
591 R: EvaluationContext,
592{
593 pub fn prepare_families(
594 &mut self,
595 sheet_name: &str,
596 families: &[crate::engine::SourceFormulaFamily],
597 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
598 self.engine.observe_function_semantic_epoch()?;
599 Ok(self
600 .engine
601 .prepare_source_formula_families(sheet_name, families))
602 }
603
604 pub fn prepare_eager_proposals(
605 &mut self,
606 sheet_name: &str,
607 families: &[crate::engine::SourceFormulaFamily],
608 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
609 formula_record_count: u64,
610 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
611 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
612 self.engine.observe_function_semantic_epoch()?;
613 let replay = Arc::new(std::sync::Mutex::new(replay));
614 self.engine.prepare_source_formula_proposals(
615 sheet_name,
616 families,
617 partitions,
618 partitions,
619 formula_record_count,
620 replay,
621 &BTreeSet::new(),
622 )
623 }
624
625 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
626 self.engine.stage_deferred_formula_package(package);
627 }
628
629 pub fn ingest_replay_batches(
630 &mut self,
631 batches: Vec<(
632 FormulaIngestBatch,
633 crate::engine::FormulaCompressedSourceBatch,
634 )>,
635 ) -> Result<FormulaIngestReport, ExcelError> {
636 self.engine
637 .ingest_compressed_formula_source_batches(batches)
638 }
639
640 pub fn finish_prepared(
641 &mut self,
642 batches: Vec<(
643 FormulaIngestBatch,
644 crate::engine::FormulaCompressedSourceReport,
645 crate::engine::FormulaCompressedPreparation,
646 )>,
647 ) -> Result<FormulaIngestReport, ExcelError> {
648 self.engine.finish_compressed_formula_sources(batches)
649 }
650}
651
652const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
656
657#[derive(Debug, Clone, PartialEq)]
658pub(crate) enum ComputedWrite {
659 Cell {
660 seq: u64,
661 sheet_id: SheetId,
662 row0: u32,
663 col0: u32,
664 value: OverlayValue,
665 },
666 Rect {
667 seq: u64,
668 sheet_id: SheetId,
669 sr0: u32,
670 sc0: u32,
671 values: Vec<Vec<OverlayValue>>,
672 },
673}
674
675impl ComputedWrite {
676 #[inline]
677 pub(crate) fn seq(&self) -> u64 {
678 match self {
679 ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685pub(crate) struct ComputedWriteBuffer {
686 writes: Vec<ComputedWrite>,
687 next_seq: u64,
688 estimated_bytes: usize,
689}
690
691impl ComputedWriteBuffer {
692 const ENTRY_BASE_BYTES: usize = 32;
693
694 #[inline]
695 pub(crate) fn is_empty(&self) -> bool {
696 self.writes.is_empty()
697 }
698
699 #[inline]
700 pub(crate) fn len(&self) -> usize {
701 self.writes.len()
702 }
703
704 #[inline]
705 pub(crate) fn estimated_bytes(&self) -> usize {
706 self.estimated_bytes
707 }
708
709 #[inline]
710 pub(crate) fn writes(&self) -> &[ComputedWrite] {
711 &self.writes
712 }
713
714 pub(crate) fn push_cell(
715 &mut self,
716 sheet_id: SheetId,
717 row0: u32,
718 col0: u32,
719 value: OverlayValue,
720 ) {
721 let seq = self.next_sequence();
722 self.estimated_bytes = self
723 .estimated_bytes
724 .saturating_add(Self::estimate_value_bytes(&value));
725 self.writes.push(ComputedWrite::Cell {
726 seq,
727 sheet_id,
728 row0,
729 col0,
730 value,
731 });
732 }
733
734 pub(crate) fn push_rect(
735 &mut self,
736 sheet_id: SheetId,
737 sr0: u32,
738 sc0: u32,
739 values: Vec<Vec<OverlayValue>>,
740 ) {
741 let seq = self.next_sequence();
742 let added = values
743 .iter()
744 .flat_map(|row| row.iter())
745 .map(Self::estimate_value_bytes)
746 .fold(0usize, usize::saturating_add);
747 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
748 self.writes.push(ComputedWrite::Rect {
749 seq,
750 sheet_id,
751 sr0,
752 sc0,
753 values,
754 });
755 }
756
757 pub(crate) fn clear(&mut self) {
758 self.writes.clear();
759 self.estimated_bytes = 0;
760 }
761
762 fn take_writes(&mut self) -> Vec<ComputedWrite> {
763 self.estimated_bytes = 0;
764 std::mem::take(&mut self.writes)
765 }
766
767 fn next_sequence(&mut self) -> u64 {
768 let seq = self.next_seq;
769 self.next_seq = self.next_seq.wrapping_add(1);
770 seq
771 }
772
773 #[inline]
774 fn estimate_value_bytes(value: &OverlayValue) -> usize {
775 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
776 }
777}
778
779#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
780struct ComputedWriteChunkKey {
781 sheet_id: SheetId,
782 col0: u32,
783 chunk_idx: usize,
784 chunk_start_row0: u32,
785}
786
787#[derive(Debug, Clone, PartialEq)]
788pub(crate) struct ComputedWriteChunkEntryPlan {
789 pub(crate) row_in_chunk: usize,
790 pub(crate) seq: u64,
791 pub(crate) value: OverlayValue,
792}
793
794#[derive(Debug, Clone, PartialEq, Eq)]
795pub(crate) enum ComputedWriteChunkPlanShape {
796 Point,
797 SparseOffsets {
798 entries: usize,
799 span_len: usize,
800 },
801 DenseRange {
802 start: usize,
803 len: usize,
804 },
805 RunRange {
806 start: usize,
807 len: usize,
808 runs: usize,
809 },
810}
811
812#[derive(Debug, Clone, PartialEq)]
813pub(crate) struct ComputedWriteChunkPlan {
814 pub(crate) sheet_id: SheetId,
815 pub(crate) col0: u32,
816 pub(crate) chunk_idx: usize,
817 pub(crate) chunk_start_row0: u32,
818 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
819 pub(crate) shape: ComputedWriteChunkPlanShape,
820}
821
822#[derive(Debug, Clone, Default, PartialEq)]
823pub(crate) struct ComputedWriteCoalescingPlan {
824 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
825 pub(crate) input_cells: usize,
826 pub(crate) coalesced_cells: usize,
827 pub(crate) overwritten_cells: usize,
828}
829
830impl ComputedWriteCoalescingPlan {
831 #[inline]
832 pub(crate) fn is_empty(&self) -> bool {
833 self.chunks.is_empty()
834 }
835}
836
837impl ComputedWriteChunkPlan {
838 fn from_group(
839 key: ComputedWriteChunkKey,
840 mut entries: Vec<ComputedWriteChunkEntryPlan>,
841 ) -> (Self, usize) {
842 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
843 let input_len = entries.len();
844 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
845 for entry in entries {
846 if let Some(prev) = coalesced.last_mut()
847 && prev.row_in_chunk == entry.row_in_chunk
848 {
849 *prev = entry;
850 continue;
851 }
852 coalesced.push(entry);
853 }
854 let overwritten = input_len.saturating_sub(coalesced.len());
855 let shape = Self::classify_shape(&coalesced);
856 (
857 Self {
858 sheet_id: key.sheet_id,
859 col0: key.col0,
860 chunk_idx: key.chunk_idx,
861 chunk_start_row0: key.chunk_start_row0,
862 entries: coalesced,
863 shape,
864 },
865 overwritten,
866 )
867 }
868
869 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
870 debug_assert!(!entries.is_empty());
871 if entries.len() == 1 {
872 return ComputedWriteChunkPlanShape::Point;
873 }
874
875 let start = entries[0].row_in_chunk;
876 let end = entries[entries.len() - 1].row_in_chunk;
877 let span_len = end.saturating_sub(start).saturating_add(1);
878 if span_len != entries.len() {
879 return ComputedWriteChunkPlanShape::SparseOffsets {
880 entries: entries.len(),
881 span_len,
882 };
883 }
884
885 let runs = Self::run_count(entries);
886 if runs < entries.len() {
887 ComputedWriteChunkPlanShape::RunRange {
888 start,
889 len: entries.len(),
890 runs,
891 }
892 } else {
893 ComputedWriteChunkPlanShape::DenseRange {
894 start,
895 len: entries.len(),
896 }
897 }
898 }
899
900 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
901 let mut runs = 0usize;
902 let mut prev: Option<&OverlayValue> = None;
903 for entry in entries {
904 if prev != Some(&entry.value) {
905 runs = runs.saturating_add(1);
906 prev = Some(&entry.value);
907 }
908 }
909 runs
910 }
911}
912
913pub struct Engine<R> {
914 pub(crate) graph: DependencyGraph,
915 resolver: R,
916 pub config: EvalConfig,
917 workbook_load_limits: crate::engine::WorkbookLoadLimits,
918 clock: crate::timezone::SnapshotClock,
923 thread_pool: Option<Arc<rayon::ThreadPool>>,
924 pub recalc_epoch: u64,
925 snapshot_id: std::sync::atomic::AtomicU64,
926 topology_epoch: u64,
927 legacy_island_structural_summaries_trusted: bool,
930 cached_static_schedule: Option<CachedScheduleEntry>,
931 cached_mixed_topology: Option<CachedMixedTopology>,
932 mixed_topology_cache_builds: u64,
933 mixed_topology_cache_hits: u64,
934 mixed_topology_cache_overflows: u64,
935 mixed_topology_cache_skip_streak: u64,
936 spill_mgr: ShimSpillManager,
937 arrow_sheets: SheetStore,
939 has_edited: bool,
941 overlay_compactions: u64,
943
944 computed_overlay_bytes_estimate: usize,
946 computed_overlay_mirroring_disabled: bool,
947 pub(crate) force_materialize_range_views: bool,
950 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
952 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
954 lookup_index_cache: LookupIndexCache,
955 source_cache: Arc<std::sync::RwLock<SourceCache>>,
956 source_formula_token: Arc<()>,
958 recalc_plan_token: Arc<()>,
960 staged_formulas: StagedFormulaMap,
962 staged_formula_index: StagedFormulaIndex,
964 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
966 row_visibility_mask_cache: std::sync::RwLock<
968 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
969 >,
970 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
972 last_formula_ingest_report: Option<FormulaIngestReport>,
974 formula_ingest_report_total: FormulaIngestReport,
976 formula_plane_cycle_member_span_demotions: u64,
986 formula_plane_capacity_bailouts: u64,
991 formula_plane_structural_span_candidates: u64,
993 active_cancel_flag: Option<crate::engine::CancelToken>,
995 active_evaluation_deadline: Option<Instant>,
997
998 action_depth: u32,
1003
1004 last_virtual_dep_telemetry: VirtualDepTelemetry,
1006 virtual_dep_fallback_activations: u64,
1007
1008 last_cycle_telemetry: CycleTelemetry,
1010
1011 next_evaluation_resource_request_id: u64,
1013 evaluation_resource_request_depth: usize,
1014 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1015 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1016 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1017 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1018 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1019 evaluation_resource_config_diagnostic:
1020 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1021 active_resource_ledger: Option<ResourceLedger>,
1022
1023 pending_iterative_redirty: Vec<VertexId>,
1037
1038 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1048
1049 function_semantic_epoch_seen: u64,
1052 function_provider_revision_seen: Option<u64>,
1055
1056 #[cfg(test)]
1057 last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1058 #[cfg(test)]
1059 evaluation_request_begin_count_for_test: u64,
1060 #[cfg(any(test, feature = "test-support"))]
1061 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1062 #[cfg(test)]
1063 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1064 #[cfg(test)]
1065 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1066 #[cfg(test)]
1067 inject_target_semantic_stale_once_for_test: bool,
1068 #[cfg(test)]
1069 force_virtual_dep_changes_remaining_for_test: usize,
1070 #[cfg(test)]
1071 fail_evaluation_commit_preflight_once_for_test: bool,
1072 #[cfg(test)]
1073 force_source_family_fallback: bool,
1074 #[cfg(test)]
1075 rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1076 #[cfg(test)]
1077 fragmented_commit_fault_for_test:
1078 Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1079 #[cfg(test)]
1080 formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1081 #[cfg(test)]
1082 target_preparation_fault_for_test:
1083 Option<crate::engine::target_preparation::TargetPreparationFault>,
1084 #[cfg(test)]
1085 force_non_cycle_schedule_fallback_for_test: bool,
1086 #[cfg(test)]
1087 mixed_topology_index_builds_for_test: u64,
1088 #[cfg(test)]
1089 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1090 #[cfg(test)]
1091 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1092}
1093
1094fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1097 fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1098 let node_type = match &ast.node_type {
1099 ASTNodeType::Literal(_) if !in_array => {
1100 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1101 *next = next.saturating_add(1);
1102 ASTNodeType::Literal(value)
1103 }
1104 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1105 ASTNodeType::Omitted => ASTNodeType::Omitted,
1106 ASTNodeType::Reference {
1107 original,
1108 reference,
1109 } => ASTNodeType::Reference {
1110 original: original.clone(),
1111 reference: reference.clone(),
1112 },
1113 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1114 op: op.clone(),
1115 expr: Box::new(visit(expr, binding, next, in_array)),
1116 },
1117 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1118 op: op.clone(),
1119 left: Box::new(visit(left, binding, next, in_array)),
1120 right: Box::new(visit(right, binding, next, in_array)),
1121 },
1122 ASTNodeType::Function { name, args } => ASTNodeType::Function {
1123 name: name.clone(),
1124 args: args
1125 .iter()
1126 .map(|arg| visit(arg, binding, next, in_array))
1127 .collect(),
1128 },
1129 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1130 callee: Box::new(visit(callee, binding, next, in_array)),
1131 args: args
1132 .iter()
1133 .map(|arg| visit(arg, binding, next, in_array))
1134 .collect(),
1135 },
1136 ASTNodeType::Array(rows) => ASTNodeType::Array(
1137 rows.iter()
1138 .map(|row| {
1139 row.iter()
1140 .map(|cell| visit(cell, binding, next, true))
1141 .collect()
1142 })
1143 .collect(),
1144 ),
1145 };
1146 ASTNode::new(node_type, ast.source_token.clone())
1147 }
1148
1149 let mut next = 0;
1150 visit(ast, binding, &mut next, false)
1151}
1152
1153impl<R: EvaluationContext> Engine<R> {
1156 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1157 self.graph.ingest_pipeline(&self.resolver)
1158 }
1159}
1160
1161pub struct EngineAction<'a, R>
1162where
1163 R: EvaluationContext,
1164{
1165 engine: &'a mut Engine<R>,
1166 name: String,
1167 log: Option<*mut crate::engine::ChangeLog>,
1170 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1173 atomic_policy: bool,
1175}
1176
1177impl<'a, R> EngineAction<'a, R>
1178where
1179 R: EvaluationContext,
1180{
1181 #[inline]
1182 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1183 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1184 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1185 crate::reference::CellRef::new(sheet_id, coord)
1186 }
1187
1188 #[inline]
1189 pub fn name(&self) -> &str {
1190 &self.name
1191 }
1192
1193 #[inline]
1194 pub fn set_cell_value(
1195 &mut self,
1196 sheet: &str,
1197 row: u32,
1198 col: u32,
1199 value: LiteralValue,
1200 ) -> Result<(), crate::engine::EditorError> {
1201 if self.log.is_some() {
1202 let old_value = self.engine.read_cell_value(sheet, row, col);
1203 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1204 let addr = self.addr_for(sheet, row, col);
1205 let Some(log_ptr) = self.log else {
1206 return Err(crate::engine::EditorError::TransactionFailed {
1207 reason: "action_with_logger: missing ChangeLog".to_string(),
1208 });
1209 };
1210
1211 let old_comp = if self.arrow_undo.is_some() {
1214 self.engine.read_computed_overlay_cell(sheet, row, col)
1215 } else {
1216 None
1217 };
1218
1219 if self.engine.graph_admission_enabled() {
1220 let admission =
1221 self.engine
1222 .graph
1223 .preview_value_mutation(addr.sheet_id, row, col)?;
1224 self.engine.preflight_graph_admission(admission)?;
1225 }
1226 self.engine.demote_span_containing_cell_for_write(
1227 addr.sheet_id,
1228 addr.coord.row(),
1229 addr.coord.col(),
1230 )?;
1231 if old_formula.is_none() {
1232 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1233 }
1234
1235 let delta_old_sem = if old_formula.is_some() {
1236 None
1237 } else {
1238 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1239 };
1240
1241 let start_len = unsafe { (&*log_ptr).len() };
1242
1243 let log = unsafe { &mut *log_ptr };
1245 self.engine.edit_with_logger(log, |editor| {
1246 editor.set_cell_value_with_old_state(
1247 addr,
1248 value.clone(),
1249 old_value.clone(),
1250 old_formula.clone(),
1251 );
1252 })?;
1253 self.engine
1254 .record_formula_plane_structural_change(StructuralScope::Cell {
1255 sheet: addr.sheet_id,
1256 row: addr.coord.row(),
1257 col: addr.coord.col(),
1258 });
1259
1260 if let Some(undo_ptr) = self.arrow_undo {
1261 let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1263 let undo = unsafe { &mut *undo_ptr };
1264 self.engine
1265 .record_spill_ops_into_arrow_undo(undo, new_events);
1266
1267 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1269 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1270 let row0 = row.saturating_sub(1);
1271 let col0 = col.saturating_sub(1);
1272 let delta_new_sem = Some(value.clone());
1273 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1274 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1275 }
1276 Ok(())
1277 } else {
1278 self.engine
1279 .set_cell_value(sheet, row, col, value)
1280 .map_err(crate::engine::EditorError::from)
1281 }
1282 }
1283
1284 #[inline]
1285 pub fn set_cell_formula(
1286 &mut self,
1287 sheet: &str,
1288 row: u32,
1289 col: u32,
1290 ast: ASTNode,
1291 ) -> Result<(), crate::engine::EditorError> {
1292 if self.log.is_some() {
1293 let old_value = self.engine.read_cell_value(sheet, row, col);
1294 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1295 let addr = self.addr_for(sheet, row, col);
1296 let Some(log_ptr) = self.log else {
1297 return Err(crate::engine::EditorError::TransactionFailed {
1298 reason: "action_with_logger: missing ChangeLog".to_string(),
1299 });
1300 };
1301
1302 let admitted_formula = if self.engine.graph_admission_enabled() {
1303 let placement =
1304 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1305 let ingested = self.engine.ingest_pipeline().ingest_formula(
1306 FormulaAstInput::Tree(ast.clone()),
1307 placement,
1308 None,
1309 )?;
1310 let admission = self.engine.graph.preview_formula_mutations(&[(
1311 addr.sheet_id,
1312 row,
1313 col,
1314 ingested.dep_plan.clone(),
1315 )])?;
1316 self.engine.preflight_graph_admission(admission)?;
1317 Some((ingested.ast_id, ingested.dep_plan))
1318 } else {
1319 None
1320 };
1321 self.engine.demote_span_containing_cell_for_write(
1322 addr.sheet_id,
1323 addr.coord.row(),
1324 addr.coord.col(),
1325 )?;
1326 if old_formula.is_none() {
1327 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328 }
1329 let delta_old = if self.arrow_undo.is_some() {
1330 if old_formula.is_some() {
1331 None
1332 } else {
1333 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334 }
1335 } else {
1336 None
1337 };
1338 let start_len = unsafe { (&*log_ptr).len() };
1339
1340 let log = unsafe { &mut *log_ptr };
1342 self.engine.edit_with_logger(log, |editor| {
1343 if let Some((ast_id, plan)) = admitted_formula {
1344 editor.set_cell_formula_with_prepared_plan(
1345 addr,
1346 ast.clone(),
1347 old_value,
1348 old_formula,
1349 ast_id,
1350 plan,
1351 );
1352 } else {
1353 editor.set_cell_formula_with_old_state(
1354 addr,
1355 ast.clone(),
1356 old_value,
1357 old_formula,
1358 );
1359 }
1360 })?;
1361 self.engine
1362 .record_formula_plane_structural_change(StructuralScope::Cell {
1363 sheet: addr.sheet_id,
1364 row: addr.coord.row(),
1365 col: addr.coord.col(),
1366 });
1367
1368 if let Some(undo_ptr) = self.arrow_undo {
1369 let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1370 let undo = unsafe { &mut *undo_ptr };
1371 self.engine
1372 .record_spill_ops_into_arrow_undo(undo, new_events);
1373 let delta_new: Option<LiteralValue> = None;
1374 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1375 let row0 = row.saturating_sub(1);
1376 let col0 = col.saturating_sub(1);
1377 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1378 }
1379 Ok(())
1380 } else {
1381 self.engine
1382 .set_cell_formula(sheet, row, col, ast)
1383 .map_err(crate::engine::EditorError::from)
1384 }
1385 }
1386
1387 #[inline]
1388 pub fn set_row_hidden(
1389 &mut self,
1390 sheet: &str,
1391 row_1based: u32,
1392 hidden: bool,
1393 source: RowVisibilitySource,
1394 ) -> Result<(), crate::engine::EditorError> {
1395 if self.log.is_some() {
1396 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1397 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1398 let old_hidden = self
1399 .engine
1400 .row_visibility
1401 .get(&sheet_id)
1402 .map(|state| state.is_row_hidden(row0, Some(source)))
1403 .unwrap_or(false);
1404 if old_hidden == hidden {
1405 return Ok(());
1406 }
1407
1408 let _ = self
1409 .engine
1410 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1411
1412 let Some(log_ptr) = self.log else {
1413 return Err(crate::engine::EditorError::TransactionFailed {
1414 reason: "action_with_logger: missing ChangeLog".to_string(),
1415 });
1416 };
1417 unsafe { &mut *log_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1418 sheet_id,
1419 row0,
1420 source,
1421 old_hidden,
1422 new_hidden: hidden,
1423 });
1424
1425 Ok(())
1426 } else {
1427 self.engine
1428 .set_row_hidden(sheet, row_1based, hidden, source)
1429 }
1430 }
1431
1432 #[inline]
1433 pub fn set_rows_hidden(
1434 &mut self,
1435 sheet: &str,
1436 start_row_1based: u32,
1437 end_row_1based: u32,
1438 hidden: bool,
1439 source: RowVisibilitySource,
1440 ) -> Result<(), crate::engine::EditorError> {
1441 if self.log.is_some() {
1442 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1443 let (start_row0, end_row0) =
1444 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1445
1446 let Some(log_ptr) = self.log else {
1447 return Err(crate::engine::EditorError::TransactionFailed {
1448 reason: "action_with_logger: missing ChangeLog".to_string(),
1449 });
1450 };
1451 let log = unsafe { &mut *log_ptr };
1452
1453 for row0 in start_row0..=end_row0 {
1454 let old_hidden = self
1455 .engine
1456 .row_visibility
1457 .get(&sheet_id)
1458 .map(|state| state.is_row_hidden(row0, Some(source)))
1459 .unwrap_or(false);
1460 if old_hidden == hidden {
1461 continue;
1462 }
1463
1464 let _ = self
1465 .engine
1466 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1467
1468 log.record(crate::engine::ChangeEvent::SetRowVisibility {
1469 sheet_id,
1470 row0,
1471 source,
1472 old_hidden,
1473 new_hidden: hidden,
1474 });
1475 }
1476
1477 Ok(())
1478 } else {
1479 self.engine
1480 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1481 }
1482 }
1483
1484 #[inline]
1485 pub fn insert_rows(
1486 &mut self,
1487 sheet: &str,
1488 before: u32,
1489 count: u32,
1490 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1491 if count == 0 {
1492 return Ok(crate::engine::ShiftSummary::default());
1493 }
1494 if self.log.is_some() {
1495 let Some(log_ptr) = self.log else {
1496 return Err(crate::engine::EditorError::TransactionFailed {
1497 reason: "action_atomic: missing ChangeLog".to_string(),
1498 });
1499 };
1500
1501 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1502 let before0 = before.saturating_sub(1);
1503 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1504 self.engine
1508 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1509
1510 let summary = {
1512 let log = unsafe { &mut *log_ptr };
1513 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1514 Ok(crate::engine::ShiftSummary::default());
1515 self.engine.edit_with_logger(log, |editor| {
1516 out = editor.insert_rows(sheet_id, before0, count);
1517 })?;
1518 out?
1519 };
1520
1521 self.engine.ensure_arrow_sheet(sheet);
1523 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1524 asheet.insert_rows(before0 as usize, count as usize);
1525 }
1526 self.engine
1527 .shift_row_visibility_insert(sheet_id, before0, count);
1528 self.engine.mark_moved_formula_vertices_dirty(&summary);
1529 self.engine
1530 .clear_computed_overlay_after_row(sheet, before0 as usize);
1531 self.engine
1532 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1533 self.engine.mark_topology_edited();
1534 if let Some(undo_ptr) = self.arrow_undo {
1535 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1536 }
1537 Ok(summary)
1538 } else {
1539 self.engine.insert_rows(sheet, before, count)
1540 }
1541 }
1542
1543 #[inline]
1544 pub fn delete_rows(
1545 &mut self,
1546 sheet: &str,
1547 start: u32,
1548 count: u32,
1549 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1550 if count == 0 {
1551 return Ok(crate::engine::ShiftSummary::default());
1552 }
1553 if self.atomic_policy {
1554 return Err(crate::engine::EditorError::TransactionUnsupported {
1555 reason:
1556 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1557 .to_string(),
1558 });
1559 }
1560 self.engine.delete_rows(sheet, start, count)
1561 }
1562
1563 #[inline]
1564 pub fn insert_columns(
1565 &mut self,
1566 sheet: &str,
1567 before: u32,
1568 count: u32,
1569 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1570 if count == 0 {
1571 return Ok(crate::engine::ShiftSummary::default());
1572 }
1573 if self.log.is_some() {
1574 let Some(log_ptr) = self.log else {
1575 return Err(crate::engine::EditorError::TransactionFailed {
1576 reason: "action_atomic: missing ChangeLog".to_string(),
1577 });
1578 };
1579
1580 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1581 let before0 = before.saturating_sub(1);
1582 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1583 self.engine
1584 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1585
1586 let summary = {
1587 let log = unsafe { &mut *log_ptr };
1588 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1589 Ok(crate::engine::ShiftSummary::default());
1590 self.engine.edit_with_logger(log, |editor| {
1591 out = editor.insert_columns(sheet_id, before0, count);
1592 })?;
1593 out?
1594 };
1595
1596 self.engine.ensure_arrow_sheet(sheet);
1597 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1598 asheet.insert_columns(before0 as usize, count as usize);
1599 }
1600 self.engine.mark_moved_formula_vertices_dirty(&summary);
1601 self.engine
1602 .clear_computed_overlay_after_col(sheet, before0 as usize);
1603 self.engine
1604 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1605 self.engine.mark_topology_edited();
1606 if let Some(undo_ptr) = self.arrow_undo {
1607 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1608 }
1609 Ok(summary)
1610 } else {
1611 self.engine.insert_columns(sheet, before, count)
1612 }
1613 }
1614
1615 #[inline]
1616 pub fn delete_columns(
1617 &mut self,
1618 sheet: &str,
1619 start: u32,
1620 count: u32,
1621 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1622 if count == 0 {
1623 return Ok(crate::engine::ShiftSummary::default());
1624 }
1625 if self.atomic_policy {
1626 return Err(crate::engine::EditorError::TransactionUnsupported {
1627 reason:
1628 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1629 .to_string(),
1630 });
1631 }
1632 self.engine.delete_columns(sheet, start, count)
1633 }
1634
1635 #[inline]
1640 pub fn action<T>(
1641 &mut self,
1642 name: impl AsRef<str>,
1643 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1644 ) -> Result<T, crate::engine::EditorError> {
1645 self.engine.action(name, f)
1646 }
1647}
1648
1649struct ActionDepthGuard<'a, R> {
1650 engine: *mut Engine<R>,
1651 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1652}
1653
1654impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1655 fn drop(&mut self) {
1656 unsafe {
1659 let e = &mut *self.engine;
1660 e.action_depth = e.action_depth.saturating_sub(1);
1661 }
1662 }
1663}
1664
1665#[derive(Default)]
1666struct SourceCache {
1667 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1668 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1669}
1670
1671#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1672struct VisibilityMaskCacheKey {
1673 sheet_id: SheetId,
1674 start_row0: u32,
1675 end_row0: u32,
1676 mode: VisibilityMaskMode,
1677 version: u64,
1678}
1679
1680#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1681enum StructuralScope {
1682 Cell { sheet: SheetId, row: u32, col: u32 },
1683 Region(Region),
1684 Sheet(SheetId),
1685 RemovedSheet(SheetId),
1686 OpaqueGlobal,
1687 AllSheets,
1688}
1689
1690struct SourceCacheSession {
1691 cache: Arc<std::sync::RwLock<SourceCache>>,
1692}
1693
1694impl Drop for SourceCacheSession {
1695 fn drop(&mut self) {
1696 if let Ok(mut g) = self.cache.write() {
1697 *g = SourceCache::default();
1698 }
1699 }
1700}
1701
1702#[derive(Debug)]
1703#[non_exhaustive]
1704pub struct EvalResult {
1705 pub computed_vertices: usize,
1706 pub cycle_errors: usize,
1707 pub elapsed: std::time::Duration,
1708}
1709
1710#[derive(Clone, Debug, PartialEq, Eq)]
1711#[non_exhaustive]
1712pub struct TableMetadata {
1713 pub name: String,
1714 pub sheet: String,
1715 pub start_row: u32,
1716 pub start_col: u32,
1717 pub end_row: u32,
1718 pub end_col: u32,
1719 pub header_row: bool,
1720 pub headers: Vec<String>,
1721 pub totals_row: bool,
1722}
1723
1724#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1729#[non_exhaustive]
1730pub struct EngineBaselineStats {
1731 pub graph_vertex_count: usize,
1732 pub graph_formula_vertex_count: usize,
1733 pub graph_edge_count: usize,
1734 pub dirty_vertex_count: usize,
1735 pub evaluation_vertex_count: usize,
1736 pub formula_ast_root_count: usize,
1737 pub formula_ast_node_count: usize,
1738 pub staged_formula_count: usize,
1739 pub formula_plane_active_span_count: usize,
1740 pub formula_plane_producer_result_entries: usize,
1741 pub formula_plane_consumer_read_entries: usize,
1742 pub formula_plane_mixed_topology_cache_builds: u64,
1743 pub formula_plane_mixed_topology_cache_hits: u64,
1744 pub formula_plane_mixed_topology_cache_overflows: u64,
1745 pub formula_plane_dirty_pending_events: usize,
1746 pub formula_plane_dirty_region_events_recorded: u64,
1747 pub formula_plane_dirty_span_region_events_recorded: u64,
1748 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1749 pub formula_plane_dirty_global_invalidations: u64,
1750 pub formula_plane_structural_span_candidates: u64,
1751 pub formula_plane_cycle_member_span_demotions: u64,
1754}
1755
1756#[derive(Debug, Clone, Default)]
1757#[non_exhaustive]
1758pub struct VirtualDepTelemetry {
1759 pub candidate_vertices_total: usize,
1760 pub vdeps_vertices_total: usize,
1761 pub vdeps_edges_total: usize,
1762 pub builder_elapsed_ms_total: u128,
1763 pub schedule_virtual_passes: usize,
1764 pub schedule_static_passes: usize,
1765 pub schedule_cache_hits: usize,
1766 pub schedule_cache_misses: usize,
1767 pub reused_schedule_vertices_total: usize,
1768 pub replan_iterations: usize,
1769 pub changed_vdeps_total: usize,
1770 pub bailout_reason: Option<&'static str>,
1771 pub fallback_mode_activations: u64,
1772}
1773
1774#[derive(Debug, Clone, Default, PartialEq)]
1783#[non_exhaustive]
1784pub struct CycleTelemetry {
1785 pub static_sccs: usize,
1787 pub phantom_sccs: usize,
1789 pub live_cycles_witnessed: usize,
1791 pub circ_cells_stamped: usize,
1793 pub settle_passes_total: usize,
1796 pub max_passes_single_scc: usize,
1798 pub iterated_sccs: usize,
1801 pub converged_sccs: usize,
1804 pub capped_sccs: usize,
1810 pub max_abs_delta_at_stop: f64,
1814 pub nan_converged: usize,
1817 pub elapsed_ms: u128,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct ScheduleBuildMeta {
1823 candidate_vertices: usize,
1824 vdeps_vertices: usize,
1825 vdeps_edges: usize,
1826 builder_elapsed_ms: u128,
1827 used_virtual_schedule: bool,
1828 schedule_cache_hit: bool,
1829 schedule_cache_eligible: bool,
1830}
1831
1832#[derive(Debug, Clone)]
1833struct CachedScheduleEntry {
1834 topology_epoch: u64,
1835 candidate_vertices: Vec<VertexId>,
1836 schedule: crate::engine::scheduler::Schedule,
1837}
1838
1839#[derive(Clone, Debug, PartialEq, Eq)]
1840struct MixedTopologyCacheKey {
1841 engine_topology_epoch: u64,
1842 graph_topology_revision: u64,
1843 authority_indexes_epoch: u64,
1844 legacy_island_revision: u64,
1848 boundary_index_revision: u64,
1849 function_semantic_epoch: u64,
1850 function_provider_revision: Option<u64>,
1851 max_candidates: usize,
1852 max_edges: usize,
1853 max_memory_bytes: usize,
1854}
1855
1856#[derive(Debug)]
1857struct CachedMixedTopology {
1858 key: MixedTopologyCacheKey,
1859 topology: MixedTopology,
1860 producer_results: FormulaProducerResultIndex,
1861 consumer_reads: FormulaConsumerReadIndex,
1862 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1863 plane_epoch: u64,
1864 island: LegacyIslandPlan,
1865}
1866
1867#[derive(Debug)]
1868struct SkippedMixedTopology {
1869 reason: MixedScheduleFallbackReason,
1870 observed: MixedTopologyCompileStats,
1871 producer_results: FormulaProducerResultIndex,
1872 consumer_reads: FormulaConsumerReadIndex,
1873 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1874 plane_epoch: u64,
1875 island: LegacyIslandPlan,
1876}
1877
1878#[derive(Debug)]
1879enum FormulaPlaneTopologyCompileResult {
1880 Cached(CachedMixedTopology),
1881 CacheSkipped(SkippedMixedTopology),
1882}
1883
1884#[derive(Clone, Copy, Debug)]
1885struct ExactDemandScratchEstimates {
1886 paged: u64,
1887 runs: u64,
1888 native: u64,
1889 repeated: u64,
1890}
1891
1892fn exact_demand_scratch_estimates(
1893 read_count: usize,
1894 closure_base: u64,
1895) -> ExactDemandScratchEstimates {
1896 let read_count = read_count as u64;
1897 let page_count = read_count.div_ceil(128);
1898 let run_count = read_count.div_ceil(256);
1899 let pointer_bytes = std::mem::size_of::<usize>() as u64;
1900 let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
1901 + std::mem::size_of::<Vec<usize>>()
1902 + 12 * std::mem::size_of::<usize>()) as u64;
1903 let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
1904 page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
1905 );
1906 let runs_memory = read_count
1907 .saturating_mul(pointer_bytes)
1908 .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
1909 ExactDemandScratchEstimates {
1910 paged: closure_base.saturating_add(paged_memory),
1911 runs: closure_base.saturating_add(runs_memory),
1912 native: closure_base
1913 .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
1914 repeated: closure_base.saturating_add(512),
1915 }
1916}
1917
1918fn native_exact_demand_allowed(
1919 policy: Option<crate::engine::DiskScratchPolicy>,
1920 target_supports_native: bool,
1921) -> bool {
1922 target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
1923}
1924
1925fn select_exact_demand_strategy(
1926 estimates: ExactDemandScratchEstimates,
1927 native_allowed: bool,
1928 mut can_reserve: impl FnMut(u64) -> bool,
1929) -> (FormulaPlaneTopologyStrategy, u64) {
1930 if can_reserve(estimates.paged) {
1931 (
1932 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1933 estimates.paged,
1934 )
1935 } else if can_reserve(estimates.runs) {
1936 (
1937 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1938 estimates.runs,
1939 )
1940 } else if native_allowed && can_reserve(estimates.native) {
1941 (
1942 FormulaPlaneTopologyStrategy::ExactNativeScratch,
1943 estimates.native,
1944 )
1945 } else {
1946 (
1947 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1948 estimates.repeated,
1949 )
1950 }
1951}
1952
1953#[cfg(test)]
1954mod exact_demand_strategy_tests {
1955 use super::*;
1956
1957 #[test]
1958 fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
1959 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
1960 assert!(estimates.paged > estimates.runs);
1961 assert!(estimates.runs > estimates.native);
1962 assert!(estimates.native > estimates.repeated);
1963 for (limit, native_allowed, expected) in [
1964 (
1965 estimates.paged,
1966 true,
1967 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1968 ),
1969 (
1970 estimates.runs,
1971 true,
1972 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1973 ),
1974 (
1975 estimates.native,
1976 true,
1977 FormulaPlaneTopologyStrategy::ExactNativeScratch,
1978 ),
1979 (
1980 estimates.repeated,
1981 true,
1982 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1983 ),
1984 ] {
1985 assert_eq!(
1986 select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
1987 expected
1988 );
1989 }
1990 }
1991
1992 #[cfg(not(target_arch = "wasm32"))]
1993 #[test]
1994 fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
1995 let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
1996 let path = scratch.path.clone();
1997 let auxiliary_path = scratch.auxiliary_path.clone();
1998 assert!(path.exists());
1999 assert!(auxiliary_path.exists());
2000 drop(scratch);
2001 assert!(!path.exists());
2002 assert!(!auxiliary_path.exists());
2003 }
2004
2005 #[test]
2006 fn native_policy_and_wasm_target_support_gate_native_rung() {
2007 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2008 assert!(!native_exact_demand_allowed(
2009 Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2010 true,
2011 ));
2012 assert!(!native_exact_demand_allowed(
2013 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2014 false,
2015 ));
2016 let native_allowed = native_exact_demand_allowed(
2017 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2018 !cfg!(target_arch = "wasm32"),
2019 );
2020 let expected = if cfg!(target_arch = "wasm32") {
2021 FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2022 } else {
2023 FormulaPlaneTopologyStrategy::ExactNativeScratch
2024 };
2025 assert_eq!(
2026 select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2027 bytes <= estimates.native
2028 })
2029 .0,
2030 expected
2031 );
2032 }
2033}
2034
2035#[cfg(not(target_arch = "wasm32"))]
2036struct NativeTopologyScratch {
2037 path: std::path::PathBuf,
2038 file: Option<std::fs::File>,
2039 auxiliary_path: std::path::PathBuf,
2040 auxiliary_file: Option<std::fs::File>,
2041}
2042
2043#[cfg(not(target_arch = "wasm32"))]
2044impl NativeTopologyScratch {
2045 fn create(request_id: u64) -> std::io::Result<Self> {
2046 use std::fs::OpenOptions;
2047 for attempt in 0..32_u64 {
2048 let path = std::env::temp_dir().join(format!(
2049 "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2050 std::process::id()
2051 ));
2052 match OpenOptions::new()
2053 .read(true)
2054 .write(true)
2055 .create_new(true)
2056 .open(&path)
2057 {
2058 Ok(file) => {
2059 let auxiliary_path = path.with_extension("aux.tmp");
2060 match OpenOptions::new()
2061 .read(true)
2062 .write(true)
2063 .create_new(true)
2064 .open(&auxiliary_path)
2065 {
2066 Ok(auxiliary_file) => {
2067 return Ok(Self {
2068 path,
2069 file: Some(file),
2070 auxiliary_path,
2071 auxiliary_file: Some(auxiliary_file),
2072 });
2073 }
2074 Err(error) => {
2075 drop(file);
2076 let _ = std::fs::remove_file(&path);
2077 if error.kind() == std::io::ErrorKind::AlreadyExists {
2078 continue;
2079 }
2080 return Err(error);
2081 }
2082 }
2083 }
2084 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2085 Err(error) => return Err(error),
2086 }
2087 }
2088 Err(std::io::Error::new(
2089 std::io::ErrorKind::AlreadyExists,
2090 "could not allocate unique topology scratch file",
2091 ))
2092 }
2093}
2094
2095#[cfg(not(target_arch = "wasm32"))]
2096impl Drop for NativeTopologyScratch {
2097 fn drop(&mut self) {
2098 drop(self.file.take());
2099 drop(self.auxiliary_file.take());
2100 let _ = std::fs::remove_file(&self.path);
2101 let _ = std::fs::remove_file(&self.auxiliary_path);
2102 }
2103}
2104
2105fn estimated_span_bindings_bytes(
2106 bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2107) -> Option<usize> {
2108 const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2109 bindings.len().checked_mul(
2110 std::mem::size_of::<FormulaSpanId>()
2111 .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2112 .checked_add(TREE_ENTRY_OVERHEAD)?,
2113 )
2114}
2115
2116type ScheduleBuildOutput = (
2117 crate::engine::scheduler::Schedule,
2118 FxHashMap<VertexId, Vec<VertexId>>,
2119 ScheduleBuildMeta,
2120);
2121
2122#[derive(Debug)]
2124pub struct RecalcPlan {
2125 key: RecalcPlanKey,
2126 kind: RecalcPlanKind,
2127}
2128
2129#[derive(Debug)]
2130struct RecalcPlanKey {
2131 engine_token: Arc<()>,
2132 revisions: PlanningRevisionSnapshot,
2133}
2134
2135#[derive(Clone, Debug, PartialEq, Eq)]
2136struct PlanningRevisionSnapshot {
2137 engine_topology_epoch: u64,
2138 graph_topology_revision: u64,
2139 authority: u64,
2140 authority_indexes: u64,
2141 authority_indexed_plane: u64,
2142 staged: u64,
2143 symbols: u64,
2144 semantic: u64,
2145 provider: Option<u64>,
2146 formula_plane_mode: FormulaPlaneMode,
2147 deterministic_mode: crate::engine::DeterministicMode,
2148 budgets: crate::engine::EvaluationBudgets,
2149 span_refs: Vec<FormulaSpanRef>,
2150}
2151
2152#[derive(Debug)]
2153enum RecalcPlanKind {
2154 CompatibilityFull {
2155 schedule: crate::engine::Schedule,
2156 has_dynamic_refs: bool,
2157 },
2158 Target {
2159 targets: Vec<crate::engine::EvaluationTarget>,
2160 scope: crate::engine::PrepareScope,
2161 topology: RecalcTopology,
2162 dynamic_policy: DynamicPlanPolicy,
2163 },
2164}
2165
2166#[derive(Debug)]
2167enum RecalcTopology {
2168 RunLocalRecipe,
2169 Workbook,
2170}
2171
2172#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2173enum DynamicPlanPolicy {
2174 BoundedTargetReplan,
2175}
2176
2177impl RecalcPlan {
2178 pub fn layer_count(&self) -> usize {
2181 match &self.kind {
2182 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2183 RecalcPlanKind::Target { .. } => 0,
2184 }
2185 }
2186
2187 pub fn has_dynamic_refs(&self) -> bool {
2188 match &self.kind {
2189 RecalcPlanKind::CompatibilityFull {
2190 has_dynamic_refs, ..
2191 } => *has_dynamic_refs,
2192 RecalcPlanKind::Target { .. } => false,
2193 }
2194 }
2195
2196 #[cfg(test)]
2197 pub(crate) fn force_stale_reasons_for_test(
2198 &mut self,
2199 reasons: &[formualizer_common::PlanStaleReason],
2200 ) {
2201 use formualizer_common::PlanStaleReason;
2202 for reason in reasons {
2203 match reason {
2204 PlanStaleReason::Engine => {
2205 self.key.engine_token = Arc::new(());
2206 }
2207 PlanStaleReason::Provider => {
2208 self.key.revisions.provider = Some(
2209 self.key
2210 .revisions
2211 .provider
2212 .unwrap_or_default()
2213 .wrapping_add(1),
2214 );
2215 }
2216 PlanStaleReason::Semantic => {
2217 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2218 }
2219 PlanStaleReason::Budget => {
2220 let current = self.key.revisions.budgets.work.max_work_units;
2221 self.key.revisions.budgets.work.max_work_units =
2222 Some(current.unwrap_or_default().wrapping_add(1));
2223 }
2224 PlanStaleReason::Staged => {
2225 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2226 }
2227 PlanStaleReason::Symbols => {
2228 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2229 }
2230 PlanStaleReason::Authority => {
2231 self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2232 }
2233 PlanStaleReason::SpanGeneration => {
2234 self.key.revisions.span_refs.push(FormulaSpanRef {
2235 id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2236 generation: u32::MAX,
2237 version: u32::MAX,
2238 });
2239 }
2240 PlanStaleReason::Graph => {
2241 self.key.revisions.graph_topology_revision =
2242 self.key.revisions.graph_topology_revision.wrapping_add(1);
2243 }
2244 _ => {}
2245 }
2246 }
2247 }
2248}
2249
2250#[cfg(test)]
2251pub(crate) mod criteria_mask_test_hooks {
2252 use std::cell::Cell;
2253
2254 thread_local! {
2255 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2256 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2257 }
2258
2259 pub fn reset_text_segment_counters() {
2260 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2261 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2262 }
2263
2264 pub fn text_segment_counters() -> (usize, usize) {
2265 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2266 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2267 (a, b)
2268 }
2269
2270 pub(crate) fn inc_total() {
2271 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2272 }
2273 pub(crate) fn inc_all_null() {
2274 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2275 }
2276}
2277
2278#[cfg(test)]
2279pub(crate) mod visibility_mask_test_hooks {
2280 use std::cell::Cell;
2281
2282 thread_local! {
2283 static HITS: Cell<usize> = const { Cell::new(0) };
2284 static MISSES: Cell<usize> = const { Cell::new(0) };
2285 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2286 }
2287
2288 pub fn reset() {
2289 HITS.with(|c| c.set(0));
2290 MISSES.with(|c| c.set(0));
2291 EVICTIONS.with(|c| c.set(0));
2292 }
2293
2294 pub fn counters() -> (usize, usize, usize) {
2295 let hits = HITS.with(|c| c.get());
2296 let misses = MISSES.with(|c| c.get());
2297 let evictions = EVICTIONS.with(|c| c.get());
2298 (hits, misses, evictions)
2299 }
2300
2301 pub(crate) fn inc_hit() {
2302 HITS.with(|c| c.set(c.get() + 1));
2303 }
2304
2305 pub(crate) fn inc_miss() {
2306 MISSES.with(|c| c.set(c.get() + 1));
2307 }
2308
2309 pub(crate) fn inc_eviction() {
2310 EVICTIONS.with(|c| c.set(c.get() + 1));
2311 }
2312}
2313
2314fn compute_criteria_mask(
2315 view: &RangeView<'_>,
2316 col_in_view: usize,
2317 pred: &crate::args::CriteriaPredicate,
2318) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2319 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2320 use arrow::compute::kernels::comparison::{ilike, nilike};
2321 use arrow_array::{
2322 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2323 };
2324
2325 fn apply_numeric_pred(
2327 chunk: &Float64Array,
2328 pred: &crate::args::CriteriaPredicate,
2329 ) -> Option<BooleanArray> {
2330 match pred {
2331 crate::args::CriteriaPredicate::Gt(n) => {
2332 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2333 }
2334 crate::args::CriteriaPredicate::Ge(n) => {
2335 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2336 }
2337 crate::args::CriteriaPredicate::Lt(n) => {
2338 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2339 }
2340 crate::args::CriteriaPredicate::Le(n) => {
2341 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2342 }
2343 crate::args::CriteriaPredicate::Eq(v) => match v {
2344 formualizer_common::LiteralValue::Number(x) => {
2345 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2346 }
2347 formualizer_common::LiteralValue::Int(i) => {
2348 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2349 }
2350 _ => None,
2351 },
2352 crate::args::CriteriaPredicate::Ne(v) => match v {
2353 formualizer_common::LiteralValue::Number(x) => {
2354 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2355 }
2356 formualizer_common::LiteralValue::Int(i) => {
2357 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2358 }
2359 _ => None,
2360 },
2361 _ => None,
2362 }
2363 }
2364
2365 let is_numeric_pred = matches!(
2367 pred,
2368 crate::args::CriteriaPredicate::Gt(_)
2369 | crate::args::CriteriaPredicate::Ge(_)
2370 | crate::args::CriteriaPredicate::Lt(_)
2371 | crate::args::CriteriaPredicate::Le(_)
2372 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2373 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2374 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2375 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2376 );
2377
2378 if is_numeric_pred {
2382 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2383 for res in view.numbers_slices() {
2384 let (_rs, _rl, cols_seg) = res.ok()?;
2385 if col_in_view < cols_seg.len() {
2386 let chunk = cols_seg[col_in_view].as_ref();
2387 let mask = apply_numeric_pred(chunk, pred)?;
2388 bool_parts.push(mask);
2389 }
2390 }
2391
2392 if bool_parts.is_empty() {
2393 return None;
2394 } else if bool_parts.len() == 1 {
2395 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2396 } else {
2397 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2399 .iter()
2400 .map(|a| a as &dyn arrow_array::Array)
2401 .collect();
2402 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2403 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2404 return Some(std::sync::Arc::new(ba));
2405 }
2406 }
2407
2408 let (text_kind, text_pat, empty_special) = match pred {
2411 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2412 (0u8, t.to_lowercase(), t.is_empty())
2413 }
2414 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2415 (1u8, t.to_lowercase(), false)
2416 }
2417 crate::args::CriteriaPredicate::TextLike {
2418 pattern,
2419 case_insensitive,
2420 } => {
2421 let p = if *case_insensitive {
2422 pattern.to_lowercase()
2423 } else {
2424 pattern.clone()
2425 };
2426 (2u8, p.replace('*', "%").replace('?', "_"), false)
2427 }
2428 _ => return None,
2429 };
2430
2431 let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2432 let pat = StringArray::new_scalar(text_pat);
2433 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2434
2435 for res in view.iter_row_chunks() {
2436 let cs = res.ok()?;
2437 if cs.row_len == 0 {
2438 continue;
2439 }
2440 #[cfg(test)]
2441 criteria_mask_test_hooks::inc_total();
2442
2443 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2444 if col_in_view >= slices.len() {
2445 return None;
2446 }
2447
2448 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2449 let seg = match seg_opt {
2450 Some(s) => s,
2451 None => {
2452 #[cfg(test)]
2453 criteria_mask_test_hooks::inc_all_null();
2454 if (text_kind == 0 && empty_special) || ne_matches_blank {
2455 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2457 bb.append_n(cs.row_len, true);
2458 bool_parts.push(bb.finish());
2459 } else {
2460 bool_parts.push(BooleanArray::new_null(cs.row_len));
2462 }
2463 continue;
2464 }
2465 };
2466
2467 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2468 let mut m = match text_kind {
2469 0 => ilike(seg_sa, &pat).ok()?,
2470 1 => nilike(seg_sa, &pat).ok()?,
2471 2 => ilike(seg_sa, &pat).ok()?,
2472 _ => return None,
2473 };
2474
2475 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2481 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2483 for i in 0..seg_sa.len() {
2484 bb.append_value(seg_sa.is_null(i));
2485 }
2486 let nulls = bb.finish();
2487 m = boolean::or_kleene(&m, &nulls).ok()?;
2488 }
2489
2490 bool_parts.push(m);
2491 }
2492
2493 if bool_parts.is_empty() {
2494 None
2495 } else if bool_parts.len() == 1 {
2496 Some(std::sync::Arc::new(bool_parts.remove(0)))
2497 } else {
2498 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2499 .iter()
2500 .map(|a| a as &dyn arrow_array::Array)
2501 .collect();
2502 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2503 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2504 Some(std::sync::Arc::new(ba))
2505 }
2506}
2507
2508#[derive(Debug, Clone)]
2509pub struct LayerInfo {
2510 pub vertex_count: usize,
2511 pub parallel_eligible: bool,
2512 pub sample_cells: Vec<String>, }
2514
2515#[derive(Debug, Clone)]
2516pub struct EvalPlan {
2517 pub total_vertices_to_evaluate: usize,
2518 pub layers: Vec<LayerInfo>,
2519 pub cycles_detected: usize,
2520 pub dirty_count: usize,
2521 pub volatile_count: usize,
2522 pub parallel_enabled: bool,
2523 pub estimated_parallel_layers: usize,
2524 pub target_cells: Vec<String>,
2525}
2526
2527impl<R> Engine<R>
2528where
2529 R: EvaluationContext,
2530{
2531 pub fn new(resolver: R, config: EvalConfig) -> Self {
2538 if let Err(msg) = config.cycle.validate() {
2539 panic!("invalid CycleConfig: {msg}");
2540 }
2541 crate::builtins::load_builtins();
2542 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2543 &config.evaluation_budgets,
2544 config.max_vertices,
2545 config.max_memory_mb,
2546 config.max_eval_time,
2547 );
2548
2549 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2550 #[cfg(feature = "system-clock")]
2551 {
2552 Arc::new(crate::timezone::SystemClock::new(
2553 crate::timezone::TimeZoneSpec::default(),
2554 ))
2555 }
2556 #[cfg(not(feature = "system-clock"))]
2557 {
2558 Arc::new(crate::timezone::FixedClock::new(
2559 chrono::DateTime::UNIX_EPOCH,
2560 crate::timezone::TimeZoneSpec::Utc,
2561 ))
2562 }
2563 });
2564
2565 let thread_pool = if config.enable_parallel {
2567 let mut builder = ThreadPoolBuilder::new();
2568 if let Some(max_threads) = config.max_threads {
2569 builder = builder.num_threads(max_threads);
2570 }
2571
2572 match builder.build() {
2573 Ok(pool) => Some(Arc::new(pool)),
2574 Err(_) => {
2575 None
2577 }
2578 }
2579 } else {
2580 None
2581 };
2582
2583 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2585 let function_provider_revision_seen = resolver.planning_semantic_revision();
2586 let mut engine = Self {
2587 graph: DependencyGraph::new_with_config(config.clone()),
2588 resolver,
2589 config,
2590 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2591 clock: crate::timezone::SnapshotClock::new(clock),
2592 thread_pool,
2593 recalc_epoch: 0,
2594 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2595 topology_epoch: 0,
2596 legacy_island_structural_summaries_trusted: true,
2597 cached_static_schedule: None,
2598 cached_mixed_topology: None,
2599 mixed_topology_cache_builds: 0,
2600 mixed_topology_cache_hits: 0,
2601 mixed_topology_cache_overflows: 0,
2602 mixed_topology_cache_skip_streak: 0,
2603 spill_mgr: ShimSpillManager::default(),
2604 arrow_sheets: SheetStore::default(),
2605 has_edited: false,
2606 overlay_compactions: 0,
2607 computed_overlay_bytes_estimate: 0,
2608 computed_overlay_mirroring_disabled: false,
2609 force_materialize_range_views: false,
2610 row_bounds_cache: std::sync::RwLock::new(None),
2611 used_axis_bounds_cache: std::sync::RwLock::new(None),
2612 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2613 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2614 source_formula_token: Arc::new(()),
2615 recalc_plan_token: Arc::new(()),
2616 staged_formulas: std::collections::HashMap::new(),
2617 staged_formula_index: StagedFormulaIndex::default(),
2618 row_visibility: FxHashMap::default(),
2619 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2620 formula_parse_diagnostics: Vec::new(),
2621 last_formula_ingest_report: None,
2622 formula_ingest_report_total: FormulaIngestReport::default(),
2623 formula_plane_cycle_member_span_demotions: 0,
2624 formula_plane_capacity_bailouts: 0,
2625 formula_plane_structural_span_candidates: 0,
2626 active_cancel_flag: None,
2627 active_evaluation_deadline: None,
2628 action_depth: 0,
2629 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2630 virtual_dep_fallback_activations: 0,
2631 last_cycle_telemetry: CycleTelemetry::default(),
2632 next_evaluation_resource_request_id: 1,
2633 evaluation_resource_request_depth: 0,
2634 active_evaluation_resource_request: None,
2635 last_evaluation_resource_request: None,
2636 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2637 evaluation_resource_request_started_at: None,
2638 evaluation_resource_budgets: resolved_resources.budgets,
2639 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2640 active_resource_ledger: None,
2641 pending_iterative_redirty: Vec::new(),
2642 iterative_state_values: FxHashMap::default(),
2643 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2644 function_provider_revision_seen,
2645 #[cfg(test)]
2646 last_formula_plane_span_eval_report: None,
2647 #[cfg(test)]
2648 evaluation_request_begin_count_for_test: 0,
2649 #[cfg(any(test, feature = "test-support"))]
2650 before_prepared_span_commit_hook: None,
2651 #[cfg(test)]
2652 before_target_preparation_commit_hook: None,
2653 #[cfg(test)]
2654 before_target_planning_snapshot_hook: None,
2655 #[cfg(test)]
2656 inject_target_semantic_stale_once_for_test: false,
2657 #[cfg(test)]
2658 force_virtual_dep_changes_remaining_for_test: 0,
2659 #[cfg(test)]
2660 fail_evaluation_commit_preflight_once_for_test: false,
2661 #[cfg(test)]
2662 force_source_family_fallback: false,
2663 #[cfg(test)]
2664 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2665 #[cfg(test)]
2666 fragmented_commit_fault_for_test: None,
2667 #[cfg(test)]
2668 formula_span_demotion_fault_for_test: None,
2669 #[cfg(test)]
2670 target_preparation_fault_for_test: None,
2671 #[cfg(test)]
2672 force_non_cycle_schedule_fallback_for_test: false,
2673 #[cfg(test)]
2674 mixed_topology_index_builds_for_test: 0,
2675 #[cfg(test)]
2676 before_legacy_fallback_final_provider_sample_hook: None,
2677 #[cfg(test)]
2678 after_eager_proposal_commit_hook: None,
2679 };
2680 engine.config.arrow_storage_enabled = true;
2682 engine.config.delta_overlay_enabled = true;
2683 engine.config.write_formula_overlay_enabled = true;
2684 let default_sheet = engine.graph.default_sheet_name().to_string();
2685 engine.ensure_arrow_sheet(&default_sheet);
2686 engine
2687 }
2688
2689 pub fn with_thread_pool(
2694 resolver: R,
2695 config: EvalConfig,
2696 thread_pool: Arc<rayon::ThreadPool>,
2697 ) -> Self {
2698 if let Err(msg) = config.cycle.validate() {
2699 panic!("invalid CycleConfig: {msg}");
2700 }
2701 crate::builtins::load_builtins();
2702 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2703 &config.evaluation_budgets,
2704 config.max_vertices,
2705 config.max_memory_mb,
2706 config.max_eval_time,
2707 );
2708 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2709 #[cfg(feature = "system-clock")]
2710 {
2711 Arc::new(crate::timezone::SystemClock::new(
2712 crate::timezone::TimeZoneSpec::default(),
2713 ))
2714 }
2715 #[cfg(not(feature = "system-clock"))]
2716 {
2717 Arc::new(crate::timezone::FixedClock::new(
2718 chrono::DateTime::UNIX_EPOCH,
2719 crate::timezone::TimeZoneSpec::Utc,
2720 ))
2721 }
2722 });
2723 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2725 let function_provider_revision_seen = resolver.planning_semantic_revision();
2726 let mut engine = Self {
2727 graph: DependencyGraph::new_with_config(config.clone()),
2728 resolver,
2729 config,
2730 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2731 clock: crate::timezone::SnapshotClock::new(clock),
2732 thread_pool: Some(thread_pool),
2733 recalc_epoch: 0,
2734 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2735 topology_epoch: 0,
2736 legacy_island_structural_summaries_trusted: true,
2737 cached_static_schedule: None,
2738 cached_mixed_topology: None,
2739 mixed_topology_cache_builds: 0,
2740 mixed_topology_cache_hits: 0,
2741 mixed_topology_cache_overflows: 0,
2742 mixed_topology_cache_skip_streak: 0,
2743 spill_mgr: ShimSpillManager::default(),
2744 arrow_sheets: SheetStore::default(),
2745 has_edited: false,
2746 overlay_compactions: 0,
2747 computed_overlay_bytes_estimate: 0,
2748 computed_overlay_mirroring_disabled: false,
2749 force_materialize_range_views: false,
2750 row_bounds_cache: std::sync::RwLock::new(None),
2751 used_axis_bounds_cache: std::sync::RwLock::new(None),
2752 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2753 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2754 source_formula_token: Arc::new(()),
2755 recalc_plan_token: Arc::new(()),
2756 staged_formulas: std::collections::HashMap::new(),
2757 staged_formula_index: StagedFormulaIndex::default(),
2758 row_visibility: FxHashMap::default(),
2759 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2760 formula_parse_diagnostics: Vec::new(),
2761 last_formula_ingest_report: None,
2762 formula_ingest_report_total: FormulaIngestReport::default(),
2763 formula_plane_cycle_member_span_demotions: 0,
2764 formula_plane_capacity_bailouts: 0,
2765 formula_plane_structural_span_candidates: 0,
2766 active_cancel_flag: None,
2767 active_evaluation_deadline: None,
2768 action_depth: 0,
2769 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2770 virtual_dep_fallback_activations: 0,
2771 last_cycle_telemetry: CycleTelemetry::default(),
2772 next_evaluation_resource_request_id: 1,
2773 evaluation_resource_request_depth: 0,
2774 active_evaluation_resource_request: None,
2775 last_evaluation_resource_request: None,
2776 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2777 evaluation_resource_request_started_at: None,
2778 evaluation_resource_budgets: resolved_resources.budgets,
2779 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2780 active_resource_ledger: None,
2781 pending_iterative_redirty: Vec::new(),
2782 iterative_state_values: FxHashMap::default(),
2783 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2784 function_provider_revision_seen,
2785 #[cfg(test)]
2786 last_formula_plane_span_eval_report: None,
2787 #[cfg(test)]
2788 evaluation_request_begin_count_for_test: 0,
2789 #[cfg(any(test, feature = "test-support"))]
2790 before_prepared_span_commit_hook: None,
2791 #[cfg(test)]
2792 before_target_preparation_commit_hook: None,
2793 #[cfg(test)]
2794 before_target_planning_snapshot_hook: None,
2795 #[cfg(test)]
2796 inject_target_semantic_stale_once_for_test: false,
2797 #[cfg(test)]
2798 force_virtual_dep_changes_remaining_for_test: 0,
2799 #[cfg(test)]
2800 fail_evaluation_commit_preflight_once_for_test: false,
2801 #[cfg(test)]
2802 force_source_family_fallback: false,
2803 #[cfg(test)]
2804 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2805 #[cfg(test)]
2806 fragmented_commit_fault_for_test: None,
2807 #[cfg(test)]
2808 formula_span_demotion_fault_for_test: None,
2809 #[cfg(test)]
2810 target_preparation_fault_for_test: None,
2811 #[cfg(test)]
2812 force_non_cycle_schedule_fallback_for_test: false,
2813 #[cfg(test)]
2814 mixed_topology_index_builds_for_test: 0,
2815 #[cfg(test)]
2816 before_legacy_fallback_final_provider_sample_hook: None,
2817 #[cfg(test)]
2818 after_eager_proposal_commit_hook: None,
2819 };
2820 engine.config.arrow_storage_enabled = true;
2822 engine.config.delta_overlay_enabled = true;
2823 engine.config.write_formula_overlay_enabled = true;
2824 let default_sheet = engine.graph.default_sheet_name().to_string();
2825 engine.ensure_arrow_sheet(&default_sheet);
2826 engine
2827 }
2828
2829 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
2830 &self.workbook_load_limits
2831 }
2832
2833 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
2834 self.workbook_load_limits = limits;
2835 }
2836
2837 fn clear_source_cache(&self) {
2838 if let Ok(mut g) = self.source_cache.write() {
2839 *g = SourceCache::default();
2840 }
2841 }
2842
2843 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
2844 &self.last_virtual_dep_telemetry
2845 }
2846
2847 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
2851 &self.last_cycle_telemetry
2852 }
2853
2854 pub fn last_evaluation_resource_request_stats(
2856 &self,
2857 ) -> Option<&EvaluationResourceRequestStats> {
2858 self.last_evaluation_resource_request.as_ref()
2859 }
2860
2861 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
2863 self.evaluation_resource_baseline
2864 }
2865
2866 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
2867 &self.evaluation_resource_budgets
2868 }
2869
2870 pub fn evaluation_resource_config_diagnostic(
2872 &self,
2873 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
2874 self.evaluation_resource_config_diagnostic.as_ref()
2875 }
2876
2877 pub fn reset_evaluation_resource_telemetry(&mut self) {
2879 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
2880 self.last_evaluation_resource_request = None;
2881 }
2882
2883 fn duration_ns(duration: std::time::Duration) -> u64 {
2884 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
2885 }
2886
2887 fn observe_evaluation_resource_request<T>(
2888 &mut self,
2889 kind: EvaluationRequestKind,
2890 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
2891 ) -> Result<T, ExcelError> {
2892 let outermost = self.evaluation_resource_request_depth == 0;
2893 if outermost {
2894 let request_id = self.next_evaluation_resource_request_id;
2895 self.next_evaluation_resource_request_id = request_id
2896 .checked_add(1)
2897 .expect("evaluation resource request ID exhausted");
2898 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
2899 request_id,
2900 kind,
2901 self.config.formula_plane_mode,
2902 self.staged_formula_count(),
2903 ));
2904 self.evaluation_resource_baseline.record_started(request_id);
2905 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
2906 self.active_resource_ledger = Some(ResourceLedger::new(
2907 Some(request_id),
2908 self.evaluation_resource_budgets.clone(),
2909 ));
2910 }
2911 self.evaluation_resource_request_depth =
2912 self.evaluation_resource_request_depth.saturating_add(1);
2913 let result = if outermost {
2914 self.resource_checkpoint(0).and_then(|()| evaluate(self))
2915 } else {
2916 evaluate(self)
2917 };
2918 self.evaluation_resource_request_depth =
2919 self.evaluation_resource_request_depth.saturating_sub(1);
2920
2921 if outermost {
2922 let total_ns = self
2923 .evaluation_resource_request_started_at
2924 .take()
2925 .map(|start| Self::duration_ns(start.elapsed()))
2926 .unwrap_or(0);
2927 let mut stats = self
2928 .active_evaluation_resource_request
2929 .take()
2930 .expect("outer evaluation resource request has active stats");
2931 let mut ledger = self
2932 .active_resource_ledger
2933 .take()
2934 .expect("outer evaluation resource request has active ledger");
2935 ledger.release_all_scratch();
2936 stats.ledger.update(ledger.snapshot());
2937 stats.outcome = match &result {
2938 Ok(_) => EvaluationRequestOutcome::Success,
2939 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
2940 EvaluationRequestOutcome::Cancelled
2941 }
2942 Err(_) => EvaluationRequestOutcome::Error,
2943 };
2944 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
2945 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
2946 FormulaDirtyLeaseOutcome::RetainedOnCancellation
2947 } else {
2948 FormulaDirtyLeaseOutcome::RetainedOnError
2949 };
2950 }
2951 stats.phases.total_ns = total_ns;
2952 let attributed = stats
2953 .phases
2954 .staged_prepare_ns
2955 .saturating_add(stats.phases.topology_ns)
2956 .saturating_add(stats.phases.materialization_ns);
2957 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
2958 self.evaluation_resource_baseline.record_finished(&stats);
2959 self.last_evaluation_resource_request = Some(stats);
2960 }
2961 result
2962 }
2963
2964 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
2965 self.evaluation_resource_budgets = budgets.clone();
2966 self.config.evaluation_budgets = budgets.clone();
2967 self.graph.set_evaluation_budgets(budgets);
2968 }
2969
2970 #[cfg(test)]
2971 pub(crate) fn set_evaluation_budgets_for_test(
2972 &mut self,
2973 budgets: crate::engine::EvaluationBudgets,
2974 ) {
2975 self.set_evaluation_resource_budgets(budgets);
2976 }
2977
2978 fn resource_loop_checkpoint(
2979 ledger: &mut Option<ResourceLedger>,
2980 work_units: u64,
2981 ) -> Result<(), ExcelError> {
2982 ledger
2983 .as_mut()
2984 .map_or(Ok(()), |ledger| {
2985 ledger
2986 .charge_work(work_units)
2987 .and_then(|()| ledger.checkpoint_deadline())
2988 })
2989 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
2990 }
2991
2992 fn preflight_evaluation_commit_window(
2993 &mut self,
2994 bounded_writes: usize,
2995 ) -> Result<crate::instant::FzInstant, ExcelError> {
2996 #[cfg(test)]
2997 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
2998 return Err(crate::engine::ResourceLedgerError::Exhausted(
2999 formualizer_common::ResourceExhaustionDetail {
3000 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3001 limit: 0,
3002 observed: 1,
3003 request_id: self
3004 .active_evaluation_resource_request
3005 .as_ref()
3006 .map(|stats| stats.request_id),
3007 },
3008 )
3009 .into_excel_error());
3010 }
3011 let estimate = std::time::Duration::from_nanos(
3012 u64::try_from(bounded_writes)
3013 .unwrap_or(u64::MAX)
3014 .saturating_mul(100),
3015 );
3016 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3017 ledger
3018 .preflight_commit_window(estimate)
3019 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3020 }
3021 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3022 stats.evaluation_commit_preflight_count =
3023 stats.evaluation_commit_preflight_count.saturating_add(1);
3024 stats.evaluation_commit_estimated_ns = stats
3025 .evaluation_commit_estimated_ns
3026 .saturating_add(Self::duration_ns(estimate));
3027 }
3028 Ok(crate::instant::FzInstant::now())
3029 }
3030
3031 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3032 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3033 stats.evaluation_commit_actual_ns = stats
3034 .evaluation_commit_actual_ns
3035 .saturating_add(Self::duration_ns(started.elapsed()));
3036 }
3037 }
3038
3039 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3040 if self
3041 .active_cancel_flag
3042 .as_ref()
3043 .is_some_and(|cancel| cancel.is_cancelled())
3044 {
3045 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3046 }
3047 if self
3048 .active_evaluation_deadline
3049 .is_some_and(|deadline| Instant::now() >= deadline)
3050 {
3051 return Err(crate::engine::ResourceLedgerError::Exhausted(
3052 formualizer_common::ResourceExhaustionDetail {
3053 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3054 limit: 0,
3055 observed: 1,
3056 request_id: self
3057 .active_evaluation_resource_request
3058 .as_ref()
3059 .map(|request| request.request_id),
3060 },
3061 )
3062 .into_excel_error()
3063 .with_message(message));
3064 }
3065 Ok(())
3066 }
3067
3068 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3069 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3070 return Ok(());
3071 };
3072 ledger
3073 .charge_work(work_units)
3074 .and_then(|()| ledger.checkpoint_deadline())
3075 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3076 }
3077
3078 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3079 if work_units == 0 {
3080 return self.resource_checkpoint(0);
3081 }
3082 while work_units > 0 {
3083 let chunk = work_units.min(256);
3084 self.resource_checkpoint(chunk)?;
3085 work_units -= chunk;
3086 }
3087 Ok(())
3088 }
3089
3090 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3091 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3092 ledger.observe_scratch(bytes);
3095 }
3096 Ok(())
3097 }
3098
3099 fn release_request_scratch(&mut self, bytes: u64) {
3100 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3101 let released = ledger.release_scratch(bytes);
3102 debug_assert!(
3103 released.is_ok(),
3104 "request scratch release exceeded the outstanding reservation"
3105 );
3106 }
3107 }
3108
3109 fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3110 self.active_resource_ledger
3111 .as_ref()
3112 .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3113 }
3114
3115 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3116 self.active_resource_ledger
3117 .as_mut()
3118 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3119 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3120 }
3121
3122 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3123 self.active_resource_ledger
3124 .as_mut()
3125 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3126 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3127 }
3128
3129 fn with_request_scratch<T>(
3130 &mut self,
3131 bytes: u64,
3132 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3133 ) -> Result<T, ExcelError> {
3134 self.reserve_request_scratch(bytes)?;
3135 let result = work(self);
3136 self.release_request_scratch(bytes);
3137 result
3138 }
3139
3140 fn observe_staged_preparation(
3141 &mut self,
3142 selected: usize,
3143 retained: usize,
3144 elapsed: std::time::Duration,
3145 ) {
3146 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3147 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3148 stats.staged_retained = retained as u64;
3149 stats.phases.staged_prepare_ns = stats
3150 .phases
3151 .staged_prepare_ns
3152 .saturating_add(Self::duration_ns(elapsed));
3153 }
3154 }
3155
3156 fn graph_admission_enabled(&self) -> bool {
3157 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3158 }
3159
3160 fn preflight_graph_admission(
3161 &mut self,
3162 usage: crate::engine::resource_ledger::GraphAdmission,
3163 ) -> Result<(), ExcelError> {
3164 let request_id = self
3165 .active_evaluation_resource_request
3166 .as_ref()
3167 .map(|stats| stats.request_id);
3168 crate::engine::resource_ledger::preflight_graph_admission(
3169 &self.evaluation_resource_budgets,
3170 usage,
3171 request_id,
3172 )
3173 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3174 }
3175
3176 fn prepared_legacy_admission(
3177 &mut self,
3178 plan: &PreparedLegacyGraphPlan,
3179 materialization_cells: u64,
3180 ) -> Result<(), ExcelError> {
3181 if !self.graph_admission_enabled() {
3182 return Ok(());
3183 }
3184 let stats = self.graph.baseline_stats();
3185 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3186 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3187 })?;
3188 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3189 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3190 })?;
3191 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3192 final_vertices: stats
3193 .graph_vertex_count
3194 .checked_add(plan.new_vertex_count())
3195 .ok_or_else(|| {
3196 ExcelError::new(ExcelErrorKind::NImpl)
3197 .with_message("graph vertex count overflow")
3198 })?,
3199 final_edges: stats
3200 .graph_edge_count
3201 .checked_sub(removed_edges)
3202 .and_then(|count| count.checked_add(added_edges))
3203 .ok_or_else(|| {
3204 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3205 })?,
3206 materialization_cells,
3207 added_vertices: plan.new_vertex_count(),
3208 added_edges,
3209 })
3210 }
3211
3212 fn observe_target_admission_failure(
3213 &mut self,
3214 reason: formualizer_common::ResourceExhaustionReason,
3215 ) {
3216 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3217 stats.target_admission_failure = Some(reason);
3218 }
3219 }
3220
3221 fn observe_target_preparation_report(
3222 &mut self,
3223 report: &crate::engine::PreparedTargetGraphReport,
3224 ) {
3225 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3226 let index = match reason {
3227 crate::engine::OpaqueReason::DynamicReference => 0,
3228 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3229 crate::engine::OpaqueReason::UnknownFunction => 2,
3230 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3231 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3232 crate::engine::OpaqueReason::UnresolvedName => 5,
3233 crate::engine::OpaqueReason::UnresolvedTable => 6,
3234 crate::engine::OpaqueReason::FormulaName => 7,
3235 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3236 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3237 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3238 };
3239 1u64 << index
3240 };
3241 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3242 stats.staged_selected = report.selected_staged_cells as u64;
3243 stats.staged_retained = report.retained_staged_cells as u64;
3244 stats.target_requested = report.requested_targets as u64;
3245 stats.target_normalized_regions = report.normalized_regions as u64;
3246 stats.target_scope_level = match &report.widened_scope {
3247 crate::engine::PrepareScope::Exact => 0,
3248 crate::engine::PrepareScope::Sheets(_) => 1,
3249 crate::engine::PrepareScope::Workbook => 2,
3250 };
3251 stats.target_widening_reason_bits = report
3252 .widening_reasons
3253 .iter()
3254 .copied()
3255 .fold(0, |bits, reason| bits | reason_bit(reason));
3256 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3257 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3258 stats.target_commit_estimated_work = report.estimated_commit_work;
3259 stats.target_commit_actual_work = report.actual_commit_work;
3260 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3261 stats.phases.staged_prepare_ns = stats
3262 .phases
3263 .staged_prepare_ns
3264 .saturating_add(Self::duration_ns(report.commit_window));
3265 }
3266 }
3267
3268 fn observe_topology(
3269 &mut self,
3270 outcome: FormulaPlaneTopologyCacheOutcome,
3271 strategy: FormulaPlaneTopologyStrategy,
3272 observed: &MixedTopologyCompileStats,
3273 elapsed: std::time::Duration,
3274 ) {
3275 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3276 if outcome.severity() > stats.topology.cache_outcome.severity() {
3277 stats.topology.cache_outcome = outcome;
3278 }
3279 if strategy.severity() > stats.topology.strategy.severity() {
3280 stats.topology.strategy = strategy;
3281 }
3282 match outcome {
3283 FormulaPlaneTopologyCacheOutcome::Hit => {
3284 stats.topology.cache_hit_events =
3285 stats.topology.cache_hit_events.saturating_add(1);
3286 }
3287 FormulaPlaneTopologyCacheOutcome::Built => {
3288 stats.topology.cache_build_events =
3289 stats.topology.cache_build_events.saturating_add(1);
3290 }
3291 FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3292 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3293 stats.topology.cache_build_events =
3294 stats.topology.cache_build_events.saturating_add(1);
3295 stats.topology.cache_skip_events =
3296 stats.topology.cache_skip_events.saturating_add(1);
3297 }
3298 FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3299 }
3300 if matches!(
3301 outcome,
3302 FormulaPlaneTopologyCacheOutcome::Built
3303 | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3304 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3305 ) {
3306 stats.topology.producers_observed = stats
3307 .topology
3308 .producers_observed
3309 .saturating_add(observed.producers as u64);
3310 stats.topology.candidates_observed = stats
3311 .topology
3312 .candidates_observed
3313 .saturating_add(observed.candidates as u64);
3314 stats.topology.edges_observed = stats
3315 .topology
3316 .edges_observed
3317 .saturating_add(observed.relationships as u64);
3318 stats.topology.candidate_cap_hits = stats
3319 .topology
3320 .candidate_cap_hits
3321 .saturating_add(observed.candidate_overflow_count as u64);
3322 stats.topology.edge_cap_hits = stats
3323 .topology
3324 .edge_cap_hits
3325 .saturating_add(observed.edge_overflow_count as u64);
3326 stats.topology.byte_cap_hits = stats
3327 .topology
3328 .byte_cap_hits
3329 .saturating_add(observed.memory_overflow_count as u64);
3330 stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3331 Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3332 } else if stats.topology.edge_cap_hits > 0 {
3333 Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3334 } else if stats.topology.candidate_cap_hits > 0 {
3335 Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3336 } else {
3337 None
3338 };
3339 stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3340 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3341 } else if stats.topology.edge_cap_hits > 0 {
3342 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3343 } else if stats.topology.candidate_cap_hits > 0 {
3344 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3345 } else {
3346 None
3347 };
3348 }
3349 stats.topology.retained_bytes_observed = stats
3350 .topology
3351 .retained_bytes_observed
3352 .max(observed.estimated_memory_bytes as u64);
3353 stats.phases.topology_ns = stats
3354 .phases
3355 .topology_ns
3356 .saturating_add(Self::duration_ns(elapsed));
3357 }
3358 }
3359
3360 fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3361 if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3362 && strategy.severity() > stats.topology.strategy.severity()
3363 {
3364 stats.topology.strategy = strategy;
3365 }
3366 }
3367
3368 fn observe_formula_plane_route(
3369 &mut self,
3370 plan: &LegacyIslandPlan,
3371 phase: FormulaPlaneRoutePhase,
3372 reason: FormulaPlaneRouteTransitionReason,
3373 demotion_generation: usize,
3374 replan_generation: usize,
3375 ) {
3376 if plan.is_empty() {
3377 return;
3378 }
3379 let mut sheet_ids = [0_u16;
3380 crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3381 let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3382 sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3383 let authority = self.graph.formula_authority();
3384 let event = FormulaPlaneRouteEvent {
3385 island_id: plan.island_id,
3386 phase,
3387 route: FormulaPlaneRoute::ContractedLegacyIsland,
3388 sheet_ids,
3389 sheet_count: sheet_count as u8,
3390 authority_epoch: authority.plane.epoch().0,
3391 index_epoch: authority.indexes_epoch(),
3392 transition_reason: reason,
3393 demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3394 replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3395 };
3396 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3397 let index = usize::from(stats.topology.route_event_count);
3398 if index < stats.topology.route_events.len() {
3399 stats.topology.route_events[index] = event;
3400 stats.topology.route_event_count =
3401 stats.topology.route_event_count.saturating_add(1);
3402 } else {
3403 stats.topology.route_events_dropped =
3404 stats.topology.route_events_dropped.saturating_add(1);
3405 }
3406 stats.topology.route_event_bytes_observed = stats
3407 .topology
3408 .route_event_bytes_observed
3409 .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3410 stats.topology.island_membership_vertices = plan.membership.len() as u64;
3411 stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3412 stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3413 stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3414 }
3415 }
3416
3417 fn observe_materialization(
3418 &mut self,
3419 placements: usize,
3420 cycle: bool,
3421 elapsed: std::time::Duration,
3422 ) {
3423 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3424 if cycle {
3425 stats.cycle_materialized_cells = stats
3426 .cycle_materialized_cells
3427 .saturating_add(placements as u64);
3428 } else {
3429 stats.fallback_materialized_cells = stats
3430 .fallback_materialized_cells
3431 .saturating_add(placements as u64);
3432 stats.topology.strategy =
3433 FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3434 }
3435 stats.phases.materialization_ns = stats
3436 .phases
3437 .materialization_ns
3438 .saturating_add(Self::duration_ns(elapsed));
3439 }
3440 }
3441
3442 fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3443 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3444 stats.dirty_lease = if empty {
3445 FormulaDirtyLeaseOutcome::Empty
3446 } else {
3447 FormulaDirtyLeaseOutcome::Acquired
3448 };
3449 }
3450 }
3451
3452 fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3453 let empty = lease.is_empty();
3454 let _ = self.graph.ack_formula_dirty(lease);
3455 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3456 stats.dirty_lease = if empty {
3457 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3458 } else {
3459 FormulaDirtyLeaseOutcome::Acknowledged
3460 };
3461 }
3462 }
3463
3464 fn checked_ack_formula_dirty_observed(
3465 &mut self,
3466 lease: FormulaDirtyLease,
3467 ) -> Result<(), ExcelError> {
3468 self.resource_checkpoint(0)?;
3469 self.ack_formula_dirty_observed(lease);
3470 Ok(())
3471 }
3472
3473 fn checked_ack_formula_dirty_sublease_observed(
3474 &mut self,
3475 lease: FormulaDirtyLease,
3476 owned_events: &[usize],
3477 ) -> Result<(), ExcelError> {
3478 self.resource_checkpoint(owned_events.len() as u64)?;
3479 let empty = lease.is_empty();
3480 let partial = owned_events.len() < lease.len();
3481 let sublease = lease.sublease(owned_events.iter().copied());
3482 if !self.graph.ack_formula_dirty_sublease(sublease) {
3483 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3484 .with_message("FormulaPlane dirty sublease became stale"));
3485 }
3486 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3487 stats.dirty_lease = if empty {
3488 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3489 } else if partial {
3490 FormulaDirtyLeaseOutcome::AcknowledgedPartial
3491 } else {
3492 FormulaDirtyLeaseOutcome::Acknowledged
3493 };
3494 }
3495 Ok(())
3496 }
3497
3498 fn begin_evaluation_request(&mut self) {
3502 #[cfg(test)]
3503 {
3504 self.evaluation_request_begin_count_for_test = self
3505 .evaluation_request_begin_count_for_test
3506 .saturating_add(1);
3507 }
3508 self.last_cycle_telemetry = CycleTelemetry::default();
3509 self.pending_iterative_redirty.clear();
3512 self.clock.refresh();
3516 }
3517
3518 fn redirty_for_next_recalc(&mut self) {
3525 self.graph.redirty_volatiles();
3526 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3527 self.iterative_state_values.clear();
3533 for &vertex in &pending {
3534 if !self.graph.vertex_exists(vertex) {
3535 continue;
3536 }
3537 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3538 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3539 if let Some(value) =
3540 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
3541 && !matches!(value, LiteralValue::Empty)
3542 {
3543 self.iterative_state_values.insert(vertex, value);
3544 }
3545 }
3546 }
3547 if !pending.is_empty() {
3548 self.graph.redirty_iterative_members(&pending);
3549 }
3550 }
3551
3552 pub fn virtual_dep_fallback_activations(&self) -> u64 {
3553 self.virtual_dep_fallback_activations
3554 }
3555
3556 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3557 self.lookup_index_cache.report()
3558 }
3559
3560 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3561 let start_row = view.start_row();
3562 let end_row = view.end_row();
3563 let start_col = view.start_col();
3564 let end_col = view.end_col();
3565 for row in start_row..=end_row {
3566 let Ok(row_u32) = u32::try_from(row) else {
3567 return true;
3568 };
3569 for col in start_col..=end_col {
3570 let Ok(col_u32) = u32::try_from(col) else {
3571 return true;
3572 };
3573 let cell_ref = self
3574 .graph
3575 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3576 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3577 && self.graph.is_volatile(*vertex_id)
3578 {
3579 return true;
3580 }
3581 }
3582 }
3583 false
3584 }
3585
3586 fn build_lookup_index_impl(
3587 &self,
3588 view: &RangeView<'_>,
3589 axis: LookupAxis,
3590 ) -> Option<Arc<LookupIndex>> {
3591 let (rows, cols) = view.dims();
3592 if rows == 0 || cols == 0 {
3593 self.lookup_index_cache.note_skipped_tiny();
3594 return None;
3595 }
3596 let len = match axis {
3597 LookupAxis::ColumnInView(col) => {
3598 if col >= cols {
3599 self.lookup_index_cache.note_skipped_tiny();
3600 return None;
3601 }
3602 rows
3603 }
3604 LookupAxis::RowInView(row) => {
3605 if row >= rows {
3606 self.lookup_index_cache.note_skipped_tiny();
3607 return None;
3608 }
3609 cols
3610 }
3611 };
3612 if len < 64 {
3613 self.lookup_index_cache.note_skipped_tiny();
3614 return None;
3615 }
3616
3617 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3618 let key = LookupIndexKey {
3619 sheet_id,
3620 start_row: u32::try_from(view.start_row()).ok()?,
3621 start_col: u32::try_from(view.start_col()).ok()?,
3622 end_row: u32::try_from(view.end_row()).ok()?,
3623 end_col: u32::try_from(view.end_col()).ok()?,
3624 axis,
3625 snapshot_id: self.data_snapshot_id(),
3626 };
3627 if let Some(index) = self.lookup_index_cache.get(&key) {
3628 return Some(index);
3629 }
3630 if self
3631 .lookup_index_cache
3632 .would_exceed_cap(estimate_bytes(len, 0))
3633 {
3634 self.lookup_index_cache.note_skipped_cap();
3635 return None;
3636 }
3637 if !self.lookup_index_cache.should_build(key) {
3638 return None;
3639 }
3640 if self.lookup_index_cache.is_known_volatile(&key) {
3641 self.lookup_index_cache.note_skipped_volatile();
3642 return None;
3643 }
3644 if self.lookup_view_contains_volatile(view, sheet_id) {
3645 self.lookup_index_cache.note_volatile_key(key);
3646 self.lookup_index_cache.note_skipped_volatile();
3647 return None;
3648 }
3649 match LookupIndex::build(view, axis, self.config.date_system).ok()? {
3650 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3651 BuildOutcome::ErrorInLookupAxis => {
3652 self.lookup_index_cache.note_skipped_error();
3653 None
3654 }
3655 BuildOutcome::Degenerate => {
3656 self.lookup_index_cache.note_skipped_tiny();
3657 None
3658 }
3659 }
3660 }
3661
3662 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
3663 if !self.config.enable_virtual_dep_telemetry {
3664 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
3665 fallback_mode_activations: self.virtual_dep_fallback_activations,
3666 ..VirtualDepTelemetry::default()
3667 };
3668 }
3669 }
3670
3671 fn source_cache_session(&self) -> SourceCacheSession {
3672 self.clear_source_cache();
3673 SourceCacheSession {
3674 cache: self.source_cache.clone(),
3675 }
3676 }
3677
3678 fn resolve_source_scalar_cached(
3679 &self,
3680 name: &str,
3681 version: Option<u64>,
3682 ) -> Result<LiteralValue, ExcelError> {
3683 let key = (name.to_string(), version);
3684 if let Ok(mut g) = self.source_cache.write() {
3685 if let Some(v) = g.scalars.get(&key) {
3686 return Ok(v.clone());
3687 }
3688
3689 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
3690 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3691 ExcelError::new(ExcelErrorKind::Ref)
3692 .with_message(format!("Unresolved source scalar: {name}"))
3693 } else {
3694 err
3695 }
3696 })?;
3697 g.scalars.insert(key, v.clone());
3698 Ok(v)
3699 } else {
3700 self.resolver.resolve_source_scalar(name).map_err(|err| {
3701 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3702 ExcelError::new(ExcelErrorKind::Ref)
3703 .with_message(format!("Unresolved source scalar: {name}"))
3704 } else {
3705 err
3706 }
3707 })
3708 }
3709 }
3710
3711 fn resolve_source_table_cached(
3712 &self,
3713 name: &str,
3714 version: Option<u64>,
3715 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
3716 let key = (name.to_string(), version);
3717 if let Ok(mut g) = self.source_cache.write() {
3718 if let Some(t) = g.tables.get(&key) {
3719 return Ok(t.clone());
3720 }
3721
3722 let t = self.resolver.resolve_source_table(name).map_err(|err| {
3723 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3724 ExcelError::new(ExcelErrorKind::Ref)
3725 .with_message(format!("Unresolved source table: {name}"))
3726 } else {
3727 err
3728 }
3729 })?;
3730 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
3731 g.tables.insert(key, t.clone());
3732 Ok(t)
3733 } else {
3734 self.resolver
3735 .resolve_source_table(name)
3736 .map_err(|err| {
3737 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3738 ExcelError::new(ExcelErrorKind::Ref)
3739 .with_message(format!("Unresolved source table: {name}"))
3740 } else {
3741 err
3742 }
3743 })
3744 .map(Arc::from)
3745 }
3746 }
3747
3748 fn source_table_to_range_view(
3749 &self,
3750 table: &dyn crate::traits::Table,
3751 spec: &Option<formualizer_parse::parser::TableSpecifier>,
3752 ) -> Result<RangeView<'static>, ExcelError> {
3753 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
3754
3755 let owned = match spec {
3756 Some(TableSpecifier::Column(c)) => {
3757 let c = c.trim();
3758 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
3759 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3760 "Complex structured references not yet supported".to_string(),
3761 ));
3762 }
3763 table.get_column(c)?.materialise().into_owned()
3764 }
3765 Some(TableSpecifier::ColumnRange(start, end)) => {
3766 let cols = table.columns();
3767 let start = start.trim();
3768 let end = end.trim();
3769 let start_key = start.to_lowercase();
3770 let end_key = end.to_lowercase();
3771 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
3772 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
3773 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
3774 if si > ei {
3775 std::mem::swap(&mut si, &mut ei);
3776 }
3777 let h = table.data_height();
3778 let w = ei - si + 1;
3779 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
3780 for (offset, ci) in (si..=ei).enumerate() {
3781 let cname = &cols[ci];
3782 let col_range = table.get_column(cname)?;
3783 let (rh, _) = col_range.dimensions();
3784 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
3785 row[offset] = col_range.get(r, 0)?;
3786 }
3787 }
3788 rows
3789 } else {
3790 return Err(ExcelError::new(ExcelErrorKind::Ref)
3791 .with_message("Column range refers to unknown column(s)".to_string()));
3792 }
3793 }
3794 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
3795 | Some(TableSpecifier::Headers) => table
3796 .headers_row()
3797 .map(|r| r.materialise().into_owned())
3798 .unwrap_or_default(),
3799 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
3800 | Some(TableSpecifier::Totals) => table
3801 .totals_row()
3802 .map(|r| r.materialise().into_owned())
3803 .unwrap_or_default(),
3804 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
3805 table
3806 .data_body()
3807 .map(|r| r.materialise().into_owned())
3808 .unwrap_or_default()
3809 }
3810 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
3811 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
3812 if let Some(h) = table.headers_row() {
3813 out.extend(h.iter_rows());
3814 }
3815 if let Some(body) = table.data_body() {
3816 out.extend(body.iter_rows());
3817 }
3818 if let Some(tr) = table.totals_row() {
3819 out.extend(tr.iter_rows());
3820 }
3821 out
3822 }
3823 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
3824 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3825 "@ (This Row) requires table-aware context; not yet supported".to_string(),
3826 ));
3827 }
3828 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
3829 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3830 .with_message("Complex structured references not yet supported".to_string()));
3831 }
3832 None => {
3833 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3834 .with_message("Table reference without specifier is unsupported".to_string()));
3835 }
3836 };
3837
3838 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
3839 }
3840
3841 pub fn default_sheet_id(&self) -> SheetId {
3842 self.graph.default_sheet_id()
3843 }
3844
3845 pub fn default_sheet_name(&self) -> &str {
3846 self.graph.default_sheet_name()
3847 }
3848
3849 pub fn set_workbook_seed(&mut self, seed: u64) {
3851 self.config.workbook_seed = seed;
3852 }
3853
3854 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
3856 self.config.volatile_level = level;
3857 }
3858
3859 pub fn set_deterministic_mode(
3861 &mut self,
3862 mode: crate::engine::DeterministicMode,
3863 ) -> Result<(), ExcelError> {
3864 let clock = mode.build_clock()?;
3865 self.config.deterministic_mode = mode;
3866 self.clock = crate::timezone::SnapshotClock::new(clock);
3867 Ok(())
3868 }
3869
3870 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
3878 self.clock = crate::timezone::SnapshotClock::new(clock);
3879 }
3880
3881 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
3882 self.config.deterministic_mode.validate()
3883 }
3884
3885 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
3886 self.graph.sheet_id(name)
3887 }
3888
3889 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
3890 self.add_sheet(name)
3891 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
3892 }
3893
3894 pub fn sheet_name(&self, id: SheetId) -> &str {
3895 self.graph.sheet_name(id)
3896 }
3897
3898 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
3899 let id = self.graph.add_sheet(name)?;
3900 self.ensure_arrow_sheet(name);
3901 self.mark_topology_edited();
3903 Ok(id)
3904 }
3905
3906 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
3907 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
3908 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
3909 })?;
3910 if new_name.is_empty() || new_name.len() > 255 {
3911 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
3912 }
3913 if self.graph.sheet_id(new_name).is_some() {
3914 return Err(ExcelError::new(ExcelErrorKind::Value)
3915 .with_message(format!("Sheet '{new_name}' already exists")));
3916 }
3917 self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
3920 .map_err(Self::editor_error_to_excel)?;
3921 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
3922
3923 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
3924 let mut copied_sheet = source_sheet;
3925 copied_sheet.name = Arc::<str>::from(new_name);
3926 self.arrow_sheets.sheets.push(copied_sheet);
3927 } else {
3928 self.ensure_arrow_sheet(new_name);
3929 }
3930
3931 let duplicated_formulas = self
3932 .graph
3933 .formula_vertices()
3934 .into_iter()
3935 .filter(|vertex| {
3936 self.graph
3937 .get_cell_ref(*vertex)
3938 .is_some_and(|cell| cell.sheet_id == new_id)
3939 })
3940 .collect::<Vec<_>>();
3941 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
3942 self.mark_topology_edited();
3943 Ok(new_id)
3944 }
3945
3946 fn ensure_arrow_sheet(&mut self, name: &str) {
3947 if self.arrow_sheets.sheet(name).is_some() {
3948 return;
3949 }
3950 self.arrow_sheets
3951 .sheets
3952 .push(crate::arrow_store::ArrowSheet {
3953 name: std::sync::Arc::<str>::from(name),
3954 date_system: self.config.date_system,
3955 columns: Vec::new(),
3956 nrows: 0,
3957 chunk_starts: Vec::new(),
3958 chunk_rows: 32 * 1024,
3959 });
3960 }
3961
3962 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
3963 let name = self.graph.sheet_name(sheet_id).to_string();
3964 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
3968 .map_err(Self::editor_error_to_excel)?;
3969 self.graph.remove_sheet(sheet_id)?;
3970 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
3971 self.clear_all_computed_overlays();
3975 self.mark_all_formula_vertices_dirty();
3976 self.clear_staged_formulas_for_sheet(&name);
3977 if self.row_visibility.remove(&sheet_id).is_some() {
3978 self.invalidate_row_visibility_mask_cache();
3979 }
3980 self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
3981 self.mark_topology_edited();
3982 Ok(())
3983 }
3984
3985 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
3987 if let Some(asheet) = self
3988 .arrow_sheets
3989 .sheets
3990 .iter_mut()
3991 .find(|s| s.name.as_ref() == target_name)
3992 {
3993 asheet.name = std::sync::Arc::<str>::from(new_name);
3994 return true;
3995 }
3996 false
3997 }
3998
3999 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4000 let old_name = self.graph.sheet_name(sheet_id).to_string();
4001
4002 self.rename_sheet_in_arrow_store(&old_name, new_name);
4005
4006 match self.graph.rename_sheet(sheet_id, new_name) {
4008 Ok(_) => {
4009 self.rename_staged_formula_sheet(&old_name, new_name);
4010 let sheet_vertices: Vec<VertexId> =
4012 self.graph.vertices_in_sheet(sheet_id).collect();
4013 for v_id in sheet_vertices {
4014 self.graph.mark_vertex_dirty(v_id);
4015 }
4016 self.mark_topology_edited();
4018 Ok(())
4019 }
4020 Err(e) => {
4021 self.rename_sheet_in_arrow_store(new_name, &old_name);
4023 Err(e)
4024 }
4025 }
4026 }
4027
4028 pub fn named_ranges_iter(
4029 &self,
4030 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4031 self.graph.named_ranges_iter()
4032 }
4033
4034 pub fn sheet_named_ranges_iter(
4035 &self,
4036 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4037 self.graph.sheet_named_ranges_iter()
4038 }
4039
4040 pub fn resolve_name_entry(
4041 &self,
4042 name: &str,
4043 current_sheet: SheetId,
4044 ) -> Option<&crate::engine::named_range::NamedRange> {
4045 self.graph.resolve_name_entry(name, current_sheet)
4046 }
4047
4048 pub(crate) fn name_query_scope(
4058 &self,
4059 scope_sheet: Option<&str>,
4060 ) -> Result<NameScope, ExcelError> {
4061 match scope_sheet {
4062 None => Ok(NameScope::Workbook),
4063 Some(sheet) => self
4064 .graph
4065 .sheet_id(sheet)
4066 .map(NameScope::Sheet)
4067 .ok_or_else(|| {
4068 ExcelError::new(ExcelErrorKind::Ref)
4069 .with_message(format!("name scope sheet not found: {sheet}"))
4070 }),
4071 }
4072 }
4073
4074 fn resolve_sheet_locator(
4087 &self,
4088 locator: &crate::reference::SharedSheetLocator<'_>,
4089 context_sheet: SheetId,
4090 ) -> Result<SheetId, ExcelError> {
4091 self.graph
4092 .sheet_reg()
4093 .resolve_locator(locator, context_sheet)
4094 }
4095
4096 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4097 let Ok(scope) = self.name_query_scope(scope_sheet) else {
4098 return false;
4099 };
4100 self.graph
4101 .resolve_name_entry_in_scope(name, scope)
4102 .is_some()
4103 }
4104
4105 pub fn resolved_name_value(
4109 &self,
4110 name: &str,
4111 scope_sheet: Option<&str>,
4112 ) -> Option<LiteralValue> {
4113 let scope = self.name_query_scope(scope_sheet).ok()?;
4114 let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4115 self.graph.get_value(entry.vertex)
4116 }
4117
4118 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4119 let entry = self.graph.resolve_table_entry(name)?;
4120 Some(TableMetadata {
4121 name: entry.name.clone(),
4122 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4123 start_row: entry.range.start.coord.row() + 1,
4124 start_col: entry.range.start.coord.col() + 1,
4125 end_row: entry.range.end.coord.row() + 1,
4126 end_col: entry.range.end.coord.col() + 1,
4127 header_row: entry.header_row,
4128 headers: entry.headers.clone(),
4129 totals_row: entry.totals_row,
4130 })
4131 }
4132
4133 pub fn tables(&self) -> Vec<TableMetadata> {
4135 self.graph
4136 .table_names()
4137 .into_iter()
4138 .filter_map(|name| self.table_metadata(&name))
4139 .collect()
4140 }
4141
4142 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4143 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4144
4145 for (name, named) in self.graph.named_ranges_iter() {
4146 out.push(crate::engine::named_range::NamedRangeSnapshot {
4147 name: name.clone(),
4148 scope: NameScope::Workbook,
4149 definition: named.definition.clone(),
4150 });
4151 }
4152
4153 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4154 out.push(crate::engine::named_range::NamedRangeSnapshot {
4155 name: name.clone(),
4156 scope: NameScope::Sheet(*sheet_id),
4157 definition: named.definition.clone(),
4158 });
4159 }
4160
4161 out.sort_by(|a, b| {
4162 let a_scope = match a.scope {
4163 NameScope::Workbook => (0u8, 0u32),
4164 NameScope::Sheet(id) => (1u8, u32::from(id)),
4165 };
4166 let b_scope = match b.scope {
4167 NameScope::Workbook => (0u8, 0u32),
4168 NameScope::Sheet(id) => (1u8, u32::from(id)),
4169 };
4170 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4171 });
4172
4173 out
4174 }
4175
4176 pub fn named_ranges_snapshot_for_sheet(
4177 &self,
4178 sheet_id: SheetId,
4179 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4180 self.named_ranges_snapshot()
4181 .into_iter()
4182 .filter(|entry| match entry.scope {
4183 NameScope::Workbook => true,
4184 NameScope::Sheet(id) => id == sheet_id,
4185 })
4186 .collect()
4187 }
4188
4189 pub fn define_name(
4190 &mut self,
4191 name: &str,
4192 definition: NamedDefinition,
4193 scope: NameScope,
4194 ) -> Result<(), ExcelError> {
4195 self.graph.validate_define_name(name, scope)?;
4201 let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4202 if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4203 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4204 "formula-name shadowing with active FormulaPlane dependents is not supported",
4205 ));
4206 }
4207 let prepared = self
4208 .prepare_name_dependent_span_demotion([name])
4209 .map_err(Self::editor_error_to_excel)?;
4210 let demoted = self
4211 .commit_name_dependent_span_demotion(prepared)
4212 .map_err(Self::editor_error_to_excel)?;
4213 self.graph.define_name(name, definition, scope)?;
4214 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4215 if !demoted {
4216 self.mark_topology_edited();
4217 }
4218 Ok(())
4219 }
4220
4221 pub fn update_name(
4222 &mut self,
4223 name: &str,
4224 definition: NamedDefinition,
4225 scope: NameScope,
4226 ) -> Result<(), ExcelError> {
4227 self.graph.validate_existing_name(name, scope)?;
4233 let prepared = self
4234 .prepare_name_dependent_span_demotion([name])
4235 .map_err(Self::editor_error_to_excel)?;
4236 let demoted = self
4237 .commit_name_dependent_span_demotion(prepared)
4238 .map_err(Self::editor_error_to_excel)?;
4239 self.graph.update_name(name, definition, scope)?;
4240 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4241 if !demoted {
4242 self.mark_topology_edited();
4243 }
4244 Ok(())
4245 }
4246
4247 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4248 self.graph.validate_existing_name(name, scope)?;
4252 let prepared = self
4253 .prepare_name_dependent_span_demotion([name])
4254 .map_err(Self::editor_error_to_excel)?;
4255 let demoted = self
4256 .commit_name_dependent_span_demotion(prepared)
4257 .map_err(Self::editor_error_to_excel)?;
4258 self.graph.delete_name(name, scope)?;
4259 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4260 if !demoted {
4261 self.mark_topology_edited();
4262 }
4263 Ok(())
4264 }
4265
4266 fn exact_name_dependent_span_refs<'a>(
4267 &self,
4268 names: impl IntoIterator<Item = &'a str>,
4269 ) -> Vec<FormulaSpanRef> {
4270 let authority = self.graph.formula_authority();
4271 let mut refs = names
4272 .into_iter()
4273 .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4274 .collect::<Vec<_>>();
4275 refs.sort_unstable_by_key(|span_ref| {
4276 (span_ref.id.0, span_ref.generation, span_ref.version)
4277 });
4278 refs.dedup();
4279 refs
4280 }
4281
4282 fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4283 events
4284 .iter()
4285 .filter_map(|event| match event {
4286 ChangeEvent::DefineName { name, .. }
4287 | ChangeEvent::UpdateName { name, .. }
4288 | ChangeEvent::DeleteName { name, .. }
4289 | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4290 _ => None,
4291 })
4292 .collect()
4293 }
4294
4295 fn prepare_name_dependent_span_demotion<'a>(
4296 &mut self,
4297 names: impl IntoIterator<Item = &'a str>,
4298 ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4299 {
4300 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4301 return Ok(None);
4302 }
4303 let refs = self.exact_name_dependent_span_refs(names);
4304 if refs.is_empty() {
4305 return Ok(None);
4306 }
4307
4308 let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4312 let clean_cells = {
4313 let authority = self.graph.formula_authority();
4314 refs.iter()
4315 .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4316 .flat_map(|span| span.domain.iter())
4317 .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4318 .map(|coord| {
4319 CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4320 })
4321 .collect::<Vec<_>>()
4322 };
4323 self.prepare_formula_span_demotion(&refs)
4324 .map(|prepared| Some((prepared, clean_cells)))
4325 .map_err(|error| match error {
4326 FormulaSpanDemotionError::Resource(error) => error.into(),
4327 error => crate::engine::EditorError::TransactionFailed {
4328 reason: format!(
4329 "FormulaPlane name-dependent demotion preparation failed: {error}"
4330 ),
4331 },
4332 })
4333 }
4334
4335 fn commit_name_dependent_span_demotion(
4336 &mut self,
4337 prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4338 ) -> Result<bool, crate::engine::EditorError> {
4339 let Some((prepared, clean_cells)) = prepared else {
4340 return Ok(false);
4341 };
4342 self.commit_prepared_formula_span_demotion(prepared)
4343 .map_err(|error| crate::engine::EditorError::TransactionFailed {
4344 reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4345 })?;
4346 for cell in clean_cells {
4347 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4348 self.graph.set_dirty(vertex_id, false);
4349 }
4350 }
4351 Ok(true)
4352 }
4353
4354 pub fn define_table(
4355 &mut self,
4356 name: &str,
4357 range: crate::reference::RangeRef,
4358 header_row: bool,
4359 headers: Vec<String>,
4360 totals_row: bool,
4361 ) -> Result<(), ExcelError> {
4362 self.graph
4363 .define_table(name, range, header_row, headers, totals_row)?;
4364 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4365 self.mark_topology_edited();
4366 Ok(())
4367 }
4368
4369 pub fn define_source_scalar(
4370 &mut self,
4371 name: &str,
4372 version: Option<u64>,
4373 ) -> Result<(), ExcelError> {
4374 self.graph.define_source_scalar(name, version)?;
4375 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4376 self.mark_topology_edited();
4377 Ok(())
4378 }
4379
4380 pub fn define_source_table(
4381 &mut self,
4382 name: &str,
4383 version: Option<u64>,
4384 ) -> Result<(), ExcelError> {
4385 self.graph.define_source_table(name, version)?;
4386 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4387 self.mark_topology_edited();
4388 Ok(())
4389 }
4390
4391 pub fn set_source_scalar_version(
4392 &mut self,
4393 name: &str,
4394 version: Option<u64>,
4395 ) -> Result<(), ExcelError> {
4396 self.graph.set_source_scalar_version(name, version)?;
4397 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4398 self.graph
4399 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4400 }
4401 Ok(())
4402 }
4403
4404 pub fn set_source_table_version(
4405 &mut self,
4406 name: &str,
4407 version: Option<u64>,
4408 ) -> Result<(), ExcelError> {
4409 self.graph.set_source_table_version(name, version)?;
4410 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4411 self.graph
4412 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4413 }
4414 Ok(())
4415 }
4416
4417 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4418 self.graph.invalidate_source(name)?;
4419 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4420 self.graph
4421 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4422 }
4423 Ok(())
4424 }
4425
4426 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4427 self.graph.get_value(vertex)
4428 }
4429
4430 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4431 self.graph.get_cell_value(sheet, row, col)
4432 }
4433
4434 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4435 self.graph.get_vertex_for_cell(cell)
4436 }
4437
4438 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4439 self.graph.get_evaluation_vertices()
4440 }
4441
4442 pub fn baseline_stats(&self) -> EngineBaselineStats {
4444 let graph = self.graph.baseline_stats();
4445 let formula_authority = self.graph.formula_authority();
4446 let formula_dirty = self.graph.formula_dirty_stats();
4447 EngineBaselineStats {
4448 graph_vertex_count: graph.graph_vertex_count,
4449 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4450 graph_edge_count: graph.graph_edge_count,
4451 dirty_vertex_count: graph.dirty_vertex_count,
4452 evaluation_vertex_count: graph.evaluation_vertex_count,
4453 formula_ast_root_count: graph.formula_ast_root_count,
4454 formula_ast_node_count: graph.formula_ast_node_count,
4455 staged_formula_count: self.staged_formula_count(),
4456 formula_plane_active_span_count: formula_authority.active_span_count(),
4457 formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4458 formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4459 formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4460 formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4461 formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4462 formula_plane_dirty_pending_events: formula_dirty.pending_events,
4463 formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4464 formula_plane_dirty_span_region_events_recorded: formula_dirty
4465 .span_region_events_recorded,
4466 formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4467 formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4468 formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4469 formula_plane_cycle_member_span_demotions: self
4470 .formula_plane_cycle_member_span_demotions,
4471 }
4472 }
4473
4474 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4476 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4477 }
4478
4479 #[cfg(test)]
4480 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4481 self.used_axis_bounds_cache
4482 .read()
4483 .ok()
4484 .and_then(|guard| {
4485 guard.as_ref().map(|cache| {
4486 (
4487 cache.row_hits.load(Ordering::Relaxed),
4488 cache.row_misses.load(Ordering::Relaxed),
4489 cache.col_hits.load(Ordering::Relaxed),
4490 cache.col_misses.load(Ordering::Relaxed),
4491 )
4492 })
4493 })
4494 .unwrap_or((0, 0, 0, 0))
4495 }
4496
4497 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4498 self.graph.set_first_load_assume_new(enabled);
4499 }
4500
4501 pub fn first_load_assume_new(&self) -> bool {
4502 self.graph.first_load_assume_new()
4503 }
4504
4505 pub fn reset_ensure_touched(&mut self) {
4506 self.graph.reset_ensure_touched();
4507 }
4508
4509 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4510 self.graph.finalize_sheet_index(sheet);
4511 }
4512
4513 pub fn action<T>(
4522 &mut self,
4523 name: impl AsRef<str>,
4524 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4525 ) -> Result<T, crate::engine::EditorError> {
4526 if self.action_depth != 0 {
4527 return Err(crate::engine::EditorError::TransactionFailed {
4528 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4529 .to_string(),
4530 });
4531 }
4532
4533 self.action_depth = 1;
4534 let engine_ptr: *mut Engine<R> = self;
4535 let _guard = ActionDepthGuard {
4536 engine: engine_ptr,
4537 _marker: std::marker::PhantomData,
4538 };
4539
4540 let mut tx = EngineAction {
4541 engine: self,
4542 name: name.as_ref().to_string(),
4543 log: None,
4544 arrow_undo: None,
4545 atomic_policy: false,
4546 };
4547 f(&mut tx)
4548 }
4549
4550 pub fn action_atomic<T>(
4554 &mut self,
4555 name: impl Into<String>,
4556 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4557 ) -> Result<T, crate::engine::EditorError> {
4558 let (v, _j) = self.action_atomic_journal(name, f)?;
4559 Ok(v)
4560 }
4561
4562 pub fn action_atomic_journal<T>(
4564 &mut self,
4565 name: impl Into<String>,
4566 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4567 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4568 if self.action_depth != 0 {
4569 return Err(crate::engine::EditorError::TransactionFailed {
4570 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4571 .to_string(),
4572 });
4573 }
4574
4575 self.action_depth = 1;
4576 let engine_ptr: *mut Engine<R> = self;
4577 let _guard = ActionDepthGuard {
4578 engine: engine_ptr,
4579 _marker: std::marker::PhantomData,
4580 };
4581
4582 let name_str = name.into();
4583 let mut log = crate::engine::ChangeLog::new();
4584 let start_len = log.len();
4585 self.action_atomic_impl(&mut log, start_len, name_str, f)
4586 }
4587
4588 fn action_atomic_impl<T>(
4589 &mut self,
4590 log: &mut crate::engine::ChangeLog,
4591 start_len: usize,
4592 name: String,
4593 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4594 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4595 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4596 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4597
4598 let log_ptr: *mut crate::engine::ChangeLog = log;
4599 let mut tx = EngineAction {
4600 engine: self,
4601 name: name.clone(),
4602 log: Some(log_ptr),
4603 arrow_undo: Some(arrow_ptr),
4604 atomic_policy: true,
4605 };
4606
4607 let res = f(&mut tx);
4608
4609 let graph_events: Vec<crate::engine::ChangeEvent> =
4611 unsafe { (&*log_ptr).events() }[start_len..].to_vec();
4612 let graph_batch = crate::engine::GraphUndoBatch {
4613 events: graph_events,
4614 };
4615 let affected_cells = arrow_undo.ops.len();
4616 let journal = crate::engine::ActionJournal {
4617 name,
4618 graph: graph_batch,
4619 arrow: arrow_undo,
4620 affected_cells,
4621 };
4622
4623 match res {
4624 Ok(v) => {
4625 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4626 for event in &journal.graph.events {
4627 self.record_formula_plane_change_for_event(event);
4628 }
4629 self.mark_data_edited();
4630 }
4631 Ok((v, journal))
4632 }
4633 Err(e) => {
4634 if let Err(rb) = self.rollback_from_action_journal(&journal) {
4635 return Err(crate::engine::EditorError::TransactionFailed {
4636 reason: format!(
4637 "Engine::action_atomic rollback failed after error '{e}': {rb}"
4638 ),
4639 });
4640 }
4641 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4642 for event in &journal.graph.events {
4643 self.record_formula_plane_change_for_event(event);
4644 }
4645 }
4646 Err(e)
4647 }
4648 }
4649 }
4650
4651 pub fn action_with_logger<T>(
4658 &mut self,
4659 log: &mut crate::engine::ChangeLog,
4660 name: impl AsRef<str>,
4661 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4662 ) -> Result<T, crate::engine::EditorError> {
4663 if self.action_depth != 0 {
4664 return Err(crate::engine::EditorError::TransactionFailed {
4665 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4666 .to_string(),
4667 });
4668 }
4669
4670 self.action_depth = 1;
4671 let engine_ptr: *mut Engine<R> = self;
4672 let _guard = ActionDepthGuard {
4673 engine: engine_ptr,
4674 _marker: std::marker::PhantomData,
4675 };
4676
4677 let start_len = log.len();
4678 let name_str = name.as_ref().to_string();
4679 log.begin_compound(name_str.clone());
4680
4681 let res = self.action_atomic_impl(log, start_len, name_str, f);
4684
4685 match res {
4686 Ok((v, _journal)) => {
4687 log.end_compound();
4688 Ok(v)
4689 }
4690 Err(e) => {
4691 log.end_compound();
4693 log.truncate(start_len);
4694 Err(e)
4695 }
4696 }
4697 }
4698
4699 fn rollback_from_action_journal(
4700 &mut self,
4701 journal: &crate::engine::ActionJournal,
4702 ) -> Result<(), crate::engine::EditorError> {
4703 journal.graph.undo(&mut self.graph)?;
4705 self.apply_inverse_row_visibility_events(&journal.graph.events);
4707 self.apply_arrow_undo_batch(&journal.arrow, true);
4709 Ok(())
4710 }
4711
4712 fn rollback_from_change_events(
4713 &mut self,
4714 events: &[crate::engine::ChangeEvent],
4715 ) -> Result<(), crate::engine::EditorError> {
4716 use crate::engine::ChangeEvent;
4717
4718 {
4720 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4721 let mut compound_stack: Vec<usize> = Vec::new();
4722 for ev in events.iter().rev() {
4723 match ev {
4724 ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
4725 ChangeEvent::CompoundStart { depth, .. } => {
4726 if compound_stack.last() == Some(depth) {
4727 compound_stack.pop();
4728 }
4729 }
4730 ChangeEvent::SetRowVisibility { .. } => {
4731 }
4733 _ => {
4734 editor.apply_inverse(ev.clone())?;
4735 }
4736 }
4737 }
4738 }
4739
4740 for ev in events.iter().rev() {
4742 self.apply_inverse_row_visibility_event(ev);
4743 }
4744
4745 for ev in events.iter().rev() {
4747 self.mirror_inverse_change_to_arrow(ev);
4748 }
4749
4750 Ok(())
4751 }
4752
4753 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
4754 let sheet_id = self.graph.sheet_id(sheet)?;
4755 let coord = Coord::from_excel(row, col, true, true);
4756 let cell = CellRef::new(sheet_id, coord);
4757 let vid = self.graph.get_vertex_for_cell(&cell)?;
4758 let ast_id = self.graph.get_formula_id(vid)?;
4759 self.graph
4760 .data_store()
4761 .retrieve_ast(ast_id, self.graph.sheet_reg())
4762 }
4763
4764 pub fn define_name_with_logger(
4765 &mut self,
4766 log: &mut crate::engine::ChangeLog,
4767 name: &str,
4768 definition: NamedDefinition,
4769 scope: NameScope,
4770 ) -> Result<(), crate::engine::EditorError> {
4771 self.graph
4772 .validate_define_name(name, scope)
4773 .map_err(crate::engine::EditorError::Excel)?;
4774 let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4775 if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4776 return Err(crate::engine::EditorError::TransactionUnsupported {
4777 reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
4778 .to_string(),
4779 });
4780 }
4781 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4782 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4783 {
4784 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4785 editor.define_name(name, definition, scope)?;
4786 }
4787 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4788 if !demoted {
4789 self.mark_topology_edited();
4790 }
4791 Ok(())
4792 }
4793
4794 pub fn update_name_with_logger(
4795 &mut self,
4796 log: &mut crate::engine::ChangeLog,
4797 name: &str,
4798 definition: NamedDefinition,
4799 scope: NameScope,
4800 ) -> Result<(), crate::engine::EditorError> {
4801 self.graph
4802 .validate_existing_name(name, scope)
4803 .map_err(crate::engine::EditorError::Excel)?;
4804 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4805 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4806 {
4807 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4808 editor.update_name(name, definition, scope)?;
4809 }
4810 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4811 if !demoted {
4812 self.mark_topology_edited();
4813 }
4814 Ok(())
4815 }
4816
4817 pub fn delete_name_with_logger(
4818 &mut self,
4819 log: &mut crate::engine::ChangeLog,
4820 name: &str,
4821 scope: NameScope,
4822 ) -> Result<(), crate::engine::EditorError> {
4823 self.graph
4824 .validate_existing_name(name, scope)
4825 .map_err(crate::engine::EditorError::Excel)?;
4826 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4827 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4828 {
4829 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4830 editor.delete_name(name, scope)?;
4831 }
4832 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4833 if !demoted {
4834 self.mark_topology_edited();
4835 }
4836 Ok(())
4837 }
4838
4839 pub fn edit_with_logger<T>(
4840 &mut self,
4841 log: &mut crate::engine::ChangeLog,
4842 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
4843 ) -> Result<T, crate::engine::EditorError> {
4844 let start_len = log.len();
4846
4847 struct ArrowSpillReader<'a> {
4850 sheets: &'a crate::arrow_store::SheetStore,
4851 }
4852 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
4853 fn read_cell_value(
4854 &self,
4855 sheet: &str,
4856 row: u32,
4857 col: u32,
4858 ) -> Option<formualizer_common::LiteralValue> {
4859 use formualizer_common::LiteralValue;
4860 let asheet = self.sheets.sheet(sheet)?;
4861 let r0 = row.saturating_sub(1) as usize;
4862 let c0 = col.saturating_sub(1) as usize;
4863 let v = asheet.get_cell_value(r0, c0);
4864 if matches!(v, LiteralValue::Empty) {
4865 None
4866 } else {
4867 Some(v)
4868 }
4869 }
4870 }
4871
4872 let ret = {
4873 let spill_reader = ArrowSpillReader {
4874 sheets: &self.arrow_sheets,
4875 };
4876 let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
4877 &mut self.graph,
4878 log,
4879 &spill_reader,
4880 );
4881 f(&mut editor)
4882 };
4883
4884 let new_events = log.events()[start_len..].to_vec();
4885 if new_events.iter().any(|event| {
4886 matches!(
4887 event,
4888 ChangeEvent::DefineName { .. }
4889 | ChangeEvent::UpdateName { .. }
4890 | ChangeEvent::DeleteName { .. }
4891 )
4892 }) {
4893 self.rollback_from_change_events(&new_events)?;
4894 log.truncate(start_len);
4895 return Err(crate::engine::EditorError::TransactionUnsupported {
4896 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
4897 });
4898 }
4899
4900 for ev in &new_events {
4903 self.mirror_forward_change_to_arrow(ev);
4904 }
4905 for ev in &new_events {
4906 self.record_formula_plane_change_for_event(ev);
4907 }
4908
4909 Ok(ret)
4910 }
4911
4912 pub(crate) fn preflight_replay_admission(
4913 &mut self,
4914 events: &[ChangeEvent],
4915 forward: bool,
4916 ) -> Result<(), crate::engine::EditorError> {
4917 if !self.graph_admission_enabled() {
4918 return Ok(());
4919 }
4920 let mut vertex_delta = 0i128;
4921 let mut edge_delta = 0i128;
4922 let mut added_vertices = 0usize;
4923 let mut added_edges = 0usize;
4924 let mut formula_cells = BTreeSet::new();
4925 for event in events {
4926 match event {
4927 ChangeEvent::AddVertex {
4928 formula,
4929 coord,
4930 sheet_id,
4931 ..
4932 } => {
4933 let delta = if forward { 1 } else { -1 };
4934 vertex_delta += delta;
4935 if forward {
4936 added_vertices = added_vertices.saturating_add(1);
4937 if formula.is_some() {
4938 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4939 }
4940 }
4941 }
4942 ChangeEvent::RemoveVertex {
4943 old_formula,
4944 coord,
4945 sheet_id,
4946 ..
4947 } => {
4948 let delta = if forward { -1 } else { 1 };
4949 vertex_delta += delta;
4950 if !forward {
4951 added_vertices = added_vertices.saturating_add(1);
4952 if old_formula.is_some()
4953 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
4954 {
4955 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4956 }
4957 }
4958 }
4959 ChangeEvent::EdgeAdded { .. } => {
4960 let delta = if forward { 1 } else { -1 };
4961 edge_delta += delta;
4962 if forward {
4963 added_edges = added_edges.saturating_add(1);
4964 }
4965 }
4966 ChangeEvent::EdgeRemoved { .. } => {
4967 let delta = if forward { -1 } else { 1 };
4968 edge_delta += delta;
4969 if !forward {
4970 added_edges = added_edges.saturating_add(1);
4971 }
4972 }
4973 ChangeEvent::SetFormula {
4974 addr, old_formula, ..
4975 } => {
4976 if forward || old_formula.is_some() {
4977 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
4978 }
4979 }
4980 _ => {}
4981 }
4982 }
4983 let stats = self.graph.baseline_stats();
4984 let final_vertices = i128::try_from(stats.graph_vertex_count)
4985 .ok()
4986 .and_then(|count| count.checked_add(vertex_delta))
4987 .and_then(|count| usize::try_from(count).ok())
4988 .ok_or_else(|| {
4989 crate::engine::EditorError::Excel(
4990 ExcelError::new(ExcelErrorKind::NImpl)
4991 .with_message("replay vertex count overflow"),
4992 )
4993 })?;
4994 let final_edges = i128::try_from(stats.graph_edge_count)
4995 .ok()
4996 .and_then(|count| count.checked_add(edge_delta))
4997 .and_then(|count| usize::try_from(count).ok())
4998 .ok_or_else(|| {
4999 crate::engine::EditorError::Excel(
5000 ExcelError::new(ExcelErrorKind::NImpl)
5001 .with_message("replay edge count overflow"),
5002 )
5003 })?;
5004 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5005 final_vertices,
5006 final_edges,
5007 materialization_cells: formula_cells.len() as u64,
5008 added_vertices,
5009 added_edges,
5010 })
5011 .map_err(crate::engine::EditorError::Excel)
5012 }
5013
5014 pub fn undo_logged(
5015 &mut self,
5016 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5017 log: &mut crate::engine::ChangeLog,
5018 ) -> Result<(), crate::engine::EditorError> {
5019 let pending_events = log
5020 .last_group_indices()
5021 .into_iter()
5022 .map(|index| log.events()[index].clone())
5023 .collect::<Vec<_>>();
5024 self.preflight_replay_admission(&pending_events, false)?;
5025 let names = Self::name_event_names(&pending_events);
5026 let prepared =
5027 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5028 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5029 let batch = undo.undo(&mut self.graph, log)?;
5030 for item in batch.iter().rev() {
5031 self.apply_inverse_row_visibility_event(&item.event);
5032 self.apply_inverse_staged_formula_event(&item.event);
5033 }
5034 self.mirror_undo_batch_to_arrow(&batch);
5035 if !batch.is_empty() {
5036 for item in &batch {
5037 self.record_formula_plane_change_for_event(&item.event);
5038 }
5039 if !demoted && !names.is_empty() {
5040 self.mark_topology_edited();
5041 }
5042 }
5043 Ok(())
5044 }
5045
5046 pub fn redo_logged(
5047 &mut self,
5048 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5049 log: &mut crate::engine::ChangeLog,
5050 ) -> Result<(), crate::engine::EditorError> {
5051 let pending_events = undo.pending_redo_events();
5052 self.preflight_replay_admission(&pending_events, true)?;
5053 let names = Self::name_event_names(&pending_events);
5054 let prepared =
5055 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5056 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5057 let batch = undo.redo(&mut self.graph, log)?;
5058 for item in &batch {
5059 self.apply_forward_row_visibility_event(&item.event);
5060 self.apply_forward_staged_formula_event(&item.event);
5061 }
5062 self.mirror_redo_batch_to_arrow(&batch);
5063 if !batch.is_empty() {
5064 for item in &batch {
5065 self.record_formula_plane_change_for_event(&item.event);
5066 }
5067 if !demoted && !names.is_empty() {
5068 self.mark_topology_edited();
5069 }
5070 }
5071 Ok(())
5072 }
5073
5074 pub fn undo_action(
5078 &mut self,
5079 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5080 ) -> Result<(), crate::engine::EditorError> {
5081 let Some(journal) = undo.pop_undo_action() else {
5082 return Ok(());
5083 };
5084 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5085 undo.push_done_action(journal);
5086 return Err(error);
5087 }
5088 let names = Self::name_event_names(&journal.graph.events);
5089 let prepared =
5090 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5091 Ok(prepared) => prepared,
5092 Err(error) => {
5093 undo.push_done_action(journal);
5094 return Err(error);
5095 }
5096 };
5097 let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5098 Ok(demoted) => demoted,
5099 Err(error) => {
5100 undo.push_done_action(journal);
5101 return Err(error);
5102 }
5103 };
5104
5105 journal.graph.undo(&mut self.graph)?;
5106 self.apply_inverse_row_visibility_events(&journal.graph.events);
5107 self.apply_arrow_undo_batch(&journal.arrow, true);
5108 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5109 for event in &journal.graph.events {
5110 self.record_formula_plane_change_for_event(event);
5111 }
5112 if !demoted && !names.is_empty() {
5113 self.mark_topology_edited();
5114 } else {
5115 self.mark_data_edited();
5116 }
5117 }
5118
5119 undo.push_redo_action(journal);
5120 Ok(())
5121 }
5122
5123 pub fn redo_action(
5127 &mut self,
5128 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5129 ) -> Result<(), crate::engine::EditorError> {
5130 let Some(journal) = undo.pop_redo_action() else {
5131 return Ok(());
5132 };
5133 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5134 undo.push_redo_action(journal);
5135 return Err(error);
5136 }
5137 let names = Self::name_event_names(&journal.graph.events);
5138 let prepared =
5139 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5140 Ok(prepared) => prepared,
5141 Err(error) => {
5142 undo.push_redo_action(journal);
5143 return Err(error);
5144 }
5145 };
5146 let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5147 Ok(demoted) => demoted,
5148 Err(error) => {
5149 undo.push_redo_action(journal);
5150 return Err(error);
5151 }
5152 };
5153
5154 journal.graph.redo(&mut self.graph)?;
5155 self.apply_forward_row_visibility_events(&journal.graph.events);
5156 self.apply_arrow_undo_batch(&journal.arrow, false);
5157 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5158 for event in &journal.graph.events {
5159 self.record_formula_plane_change_for_event(event);
5160 }
5161 if !demoted && !names.is_empty() {
5162 self.mark_topology_edited();
5163 } else {
5164 self.mark_data_edited();
5165 }
5166 }
5167
5168 undo.push_done_action(journal);
5169 Ok(())
5170 }
5171
5172 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5173 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5174 let row = addr.coord.row() + 1;
5176 let col = addr.coord.col() + 1;
5177 (sheet, row, col)
5178 }
5179
5180 fn mirror_undo_batch_to_arrow(
5181 &mut self,
5182 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5183 ) {
5184 for item in batch.iter().rev() {
5186 self.mirror_inverse_change_to_arrow(&item.event);
5187 }
5188 }
5189
5190 fn mirror_redo_batch_to_arrow(
5191 &mut self,
5192 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5193 ) {
5194 for item in batch.iter() {
5196 self.mirror_forward_change_to_arrow(&item.event);
5197 }
5198 }
5199
5200 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5201 use crate::engine::ChangeEvent;
5202 use formualizer_common::LiteralValue;
5203
5204 match ev {
5205 ChangeEvent::SetValue {
5206 addr,
5207 old_value,
5208 old_formula,
5209 ..
5210 } => {
5211 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5212 if old_formula.is_some() {
5213 self.clear_delta_overlay_cell(&sheet, row, col);
5214 } else {
5215 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5216 self.mirror_value_to_overlay(&sheet, row, col, &v);
5217 }
5218 }
5219 ChangeEvent::SetFormula {
5220 addr,
5221 old_value,
5222 old_formula,
5223 ..
5224 } => {
5225 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5226 if old_formula.is_some() {
5227 self.clear_delta_overlay_cell(&sheet, row, col);
5228 } else {
5229 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5230 self.mirror_value_to_overlay(&sheet, row, col, &v);
5231 }
5232 }
5233 ChangeEvent::SpillCommitted { old, new, .. } => {
5234 self.mirror_spill_snapshot(new, true);
5236 if let Some(snap) = old {
5237 self.mirror_spill_snapshot(snap, false);
5238 }
5239 }
5240 ChangeEvent::SpillCleared { old, .. } => {
5241 self.mirror_spill_snapshot(old, false);
5243 }
5244 ChangeEvent::SetRowVisibility { .. } => {
5245 }
5247 _ => {}
5248 }
5249 }
5250
5251 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5252 use crate::engine::ChangeEvent;
5253
5254 match ev {
5255 ChangeEvent::SetValue { addr, new, .. } => {
5256 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5257 self.mirror_value_to_overlay(&sheet, row, col, new);
5258 }
5259 ChangeEvent::SetFormula { addr, .. } => {
5260 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5261 self.clear_delta_overlay_cell(&sheet, row, col);
5262 }
5264 ChangeEvent::SpillCommitted { old, new, .. } => {
5265 if let Some(snap) = old {
5266 self.mirror_spill_snapshot(snap, true);
5267 }
5268 self.mirror_spill_snapshot(new, false);
5269 }
5270 ChangeEvent::SpillCleared { old, .. } => {
5271 self.mirror_spill_snapshot(old, true);
5272 }
5273 ChangeEvent::SetRowVisibility { .. } => {
5274 }
5276 _ => {
5277 }
5279 }
5280 }
5281
5282 fn mirror_spill_snapshot(
5283 &mut self,
5284 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5285 clear_only: bool,
5286 ) {
5287 use formualizer_common::LiteralValue;
5288
5289 let mut i = 0usize;
5290 for row in &snap.values {
5291 for v in row {
5292 if let Some(cell) = snap.target_cells.get(i) {
5293 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5294 let out = if clear_only {
5295 LiteralValue::Empty
5296 } else {
5297 v.clone()
5298 };
5299 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5300 }
5301 i += 1;
5302 }
5303 }
5304 if clear_only {
5306 for cell in snap.target_cells.iter().skip(i) {
5307 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5308 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5309 }
5310 }
5311 }
5312
5313 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5314 self.graph.set_default_sheet_by_name(name);
5315 }
5316
5317 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5318 self.graph.set_default_sheet_by_id(id);
5319 }
5320
5321 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5322 self.graph.set_sheet_index_mode(mode);
5323 }
5324
5325 fn clear_cached_static_schedule(&mut self) {
5326 self.cached_static_schedule = None;
5327 }
5328
5329 pub fn mark_data_edited(&mut self) {
5332 self.snapshot_id
5333 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5334 self.has_edited = true;
5335 }
5336
5337 pub fn mark_topology_edited(&mut self) {
5339 self.snapshot_id
5340 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5341 self.topology_epoch = self.topology_epoch.wrapping_add(1);
5342 self.graph.bump_topology_revision();
5343 self.clear_cached_static_schedule();
5344 self.cached_mixed_topology = None;
5345 if let Some(ledger) = self.active_resource_ledger.as_mut() {
5346 let released = ledger.account_mixed_cache(0);
5347 debug_assert!(released.is_ok());
5348 }
5349 self.has_edited = true;
5350 }
5351
5352 fn mark_all_formula_vertices_dirty(&mut self) {
5353 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5354 for vertex in vertices {
5355 self.graph.mark_vertex_dirty(vertex);
5356 }
5357 }
5358
5359 fn mark_moved_formula_vertices_dirty(
5360 &mut self,
5361 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5362 ) {
5363 for vertex in &summary.vertices_moved {
5364 if self.graph.get_formula_id(*vertex).is_some() {
5365 self.graph.mark_vertex_dirty(*vertex);
5366 }
5367 }
5368 }
5369
5370 pub fn sheet_store(&self) -> &SheetStore {
5372 &self.arrow_sheets
5373 }
5374
5375 pub(crate) fn has_row_visibility_state(&self) -> bool {
5378 !self.row_visibility.is_empty()
5379 }
5380
5381 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5383 &mut self.arrow_sheets
5384 }
5385
5386 pub fn has_staged_formulas(&self) -> bool {
5387 !self.staged_formulas.is_empty()
5388 }
5389
5390 pub fn staged_formula_count(&self) -> usize {
5391 self.staged_formulas.values().map(StagedSheet::len).sum()
5392 }
5393
5394 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5396 self.staged_formulas
5397 .entry(sheet.to_string())
5398 .or_default()
5399 .stage(row, col, text);
5400 self.staged_formula_index.stage(sheet, row, col);
5401 }
5402
5403 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5404 for (row, col, _) in &staged.entries {
5405 self.staged_formula_index.remove(sheet, *row, *col);
5406 }
5407 if staged.deferred_package.is_some() {
5408 self.staged_formula_index.set_package(sheet, None);
5409 }
5410 }
5411
5412 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5413 self.staged_formulas.insert(sheet, staged);
5414 }
5415
5416 #[doc(hidden)]
5417 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5418 SourceFormulaIngress { engine: self }
5419 }
5420
5421 #[doc(hidden)]
5422 #[doc(hidden)]
5425 #[cfg(any(test, feature = "test-support"))]
5426 pub fn set_before_prepared_span_commit_hook(
5427 &mut self,
5428 hook: impl FnOnce() + Send + Sync + 'static,
5429 ) {
5430 self.before_prepared_span_commit_hook = Some(Box::new(hook));
5431 }
5432
5433 #[doc(hidden)]
5434 #[cfg(test)]
5436 pub(crate) fn set_before_target_preparation_commit_hook(
5437 &mut self,
5438 hook: impl FnOnce() + Send + Sync + 'static,
5439 ) {
5440 self.before_target_preparation_commit_hook = Some(Box::new(hook));
5441 }
5442
5443 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5444 let sheet = package.sheet_name.clone();
5445 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5446 debug_assert!(staged.deferred_package.is_none());
5447 staged.deferred_package = Some(package);
5448 staged.reconcile_attached_deferred_package();
5449 self.staged_formula_index
5450 .set_package(&sheet, staged.deferred_package.as_ref());
5451 }
5452
5453 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
5454 let mut removed = None;
5455 let mut remove_sheet = false;
5456 let mut had_package = false;
5457 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
5458 had_package = entries.deferred_package.is_some();
5459 removed = entries.remove(row, col);
5460 remove_sheet = entries.is_empty();
5461 }
5462 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
5463 if !ordinary_removed && had_package {
5464 self.staged_formula_index.touch_package(sheet);
5465 }
5466 if remove_sheet {
5467 self.staged_formulas.remove(sheet);
5468 self.staged_formula_index.set_package(sheet, None);
5469 }
5470 removed
5471 }
5472
5473 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
5474 if self.staged_formulas.remove(sheet).is_some() {
5475 self.staged_formula_index.clear_sheet(sheet);
5476 }
5477 }
5478
5479 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
5480 let Some(entries) = self.staged_formulas.remove(old) else {
5481 return;
5482 };
5483 self.staged_formula_index.clear_sheet(old);
5484 let (formulas, mut package) = entries.into_parts();
5485 for (row, col, text) in formulas {
5486 self.stage_formula_text(new, row, col, text);
5487 }
5488 if let Some(package) = package.as_mut() {
5489 package.sheet_name = new.to_string();
5490 }
5491 if let Some(package) = package {
5492 self.stage_deferred_formula_package(package);
5493 }
5494 }
5495
5496 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
5498 self.staged_formulas
5499 .get(sheet)
5500 .and_then(|v| v.get(row, col))
5501 }
5502
5503 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
5504 &self.formula_parse_diagnostics
5505 }
5506
5507 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
5508 std::mem::take(&mut self.formula_parse_diagnostics)
5509 }
5510
5511 pub fn clear_formula_parse_diagnostics(&mut self) {
5512 self.formula_parse_diagnostics.clear();
5513 }
5514
5515 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
5516 self.last_formula_ingest_report.as_ref()
5517 }
5518
5519 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
5520 &self.formula_ingest_report_total
5521 }
5522
5523 #[cfg(test)]
5524 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
5525 &mut self,
5526 hook: impl FnOnce() + Send + Sync + 'static,
5527 ) {
5528 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
5529 }
5530
5531 #[cfg(test)]
5532 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
5533 self.inject_target_semantic_stale_once_for_test = true;
5534 }
5535
5536 #[cfg(test)]
5537 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
5538 self.force_virtual_dep_changes_remaining_for_test = rounds;
5539 }
5540
5541 #[cfg(test)]
5542 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
5543 self.fail_evaluation_commit_preflight_once_for_test = true;
5544 }
5545
5546 #[cfg(test)]
5547 pub(crate) fn set_target_preparation_fault_for_test(
5548 &mut self,
5549 fault: crate::engine::target_preparation::TargetPreparationFault,
5550 ) {
5551 self.target_preparation_fault_for_test = Some(fault);
5552 }
5553
5554 #[cfg(test)]
5555 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
5556 self.staged_formula_index.revision()
5557 }
5558
5559 #[cfg(test)]
5560 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
5561 let ordinary_storage = self
5562 .staged_formulas
5563 .values()
5564 .map(|sheet| sheet.entries.len())
5565 .sum::<usize>();
5566 let package_storage = self
5567 .staged_formulas
5568 .values()
5569 .filter(|sheet| sheet.deferred_package.is_some())
5570 .count();
5571 ordinary_storage == self.staged_formula_index.ordinary_count()
5572 && package_storage == self.staged_formula_index.package_count()
5573 && self.staged_formulas.iter().all(|(name, sheet)| {
5574 sheet.entries.iter().all(|(row, col, _)| {
5575 let leases = self
5576 .staged_formula_index
5577 .leases_in_region(name, *row, *col, *row, *col);
5578 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
5579 })
5580 })
5581 }
5582
5583 #[cfg(test)]
5584 pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
5585 self.last_formula_plane_span_eval_report.as_ref()
5586 }
5587
5588 #[cfg(test)]
5589 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
5590 self.evaluation_request_begin_count_for_test
5591 }
5592
5593 #[cfg(test)]
5594 pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
5595 self.force_source_family_fallback = force;
5596 }
5597
5598 #[cfg(test)]
5599 pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
5600 self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
5601 }
5602
5603 #[cfg(test)]
5604 pub(crate) fn set_fragmented_commit_fault_for_test(
5605 &mut self,
5606 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5607 ) {
5608 self.fragmented_commit_fault_for_test = Some(fault);
5609 }
5610
5611 #[cfg(test)]
5612 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
5613 &mut self,
5614 hook: impl FnOnce() + Send + Sync + 'static,
5615 ) {
5616 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
5617 }
5618
5619 #[cfg(test)]
5620 pub(crate) fn set_after_eager_proposal_commit_hook(
5621 &mut self,
5622 hook: impl FnOnce() + Send + Sync + 'static,
5623 ) {
5624 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
5625 }
5626
5627 #[cfg(test)]
5628 pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
5629 self.graph.formula_authority().indexes_epoch()
5630 }
5631
5632 #[cfg(test)]
5633 pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
5634 self.formula_plane_capacity_bailouts
5635 }
5636
5637 #[cfg(test)]
5638 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
5639 self.topology_epoch
5640 }
5641
5642 #[cfg(test)]
5643 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
5644 self.graph.topology_revision()
5645 }
5646
5647 #[cfg(test)]
5648 pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
5649 self.cached_mixed_topology.is_some()
5650 }
5651
5652 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
5653 self.formula_ingest_report_total.mode = report.mode;
5654 self.formula_ingest_report_total.accumulate(&report);
5655 self.last_formula_ingest_report = Some(report);
5656 }
5657
5658 fn prepare_source_formula_family(
5659 &mut self,
5660 sheet_id: SheetId,
5661 family: &crate::engine::SourceFormulaFamily,
5662 allow_function_closure: bool,
5663 planning_provider: Option<&dyn crate::traits::FunctionProvider>,
5664 ) -> Result<
5665 (
5666 crate::formula_plane::placement::PreparedAnchorOncePlacement,
5667 bool,
5668 ),
5669 SourceFamilyPreparationError,
5670 > {
5671 let transport = family
5672 .validated_complete_domain(&self.workbook_load_limits)
5673 .map_err(SourceFamilyPreparationError::contract)?;
5674 let domain = match transport {
5675 crate::engine::PlacementDomainTransport::RowRun {
5676 row_start,
5677 row_end,
5678 col,
5679 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5680 crate::engine::PlacementDomainTransport::ColRun {
5681 row,
5682 col_start,
5683 col_end,
5684 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5685 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5686 sheet_id,
5687 rect.start.row,
5688 rect.end.row,
5689 rect.start.col,
5690 rect.end.col,
5691 ),
5692 };
5693 let formula = if family.anchor_text.starts_with('=') {
5694 family.anchor_text.to_string()
5695 } else {
5696 format!("={}", family.anchor_text)
5697 };
5698 let ast = formualizer_parse::parser::parse(&formula)
5699 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5700 let relocation = if allow_function_closure {
5701 validate_anchor_once_shadow_relocation(
5702 &ast,
5703 family.anchor_coord0.row,
5704 family.anchor_coord0.col,
5705 &domain,
5706 planning_provider.unwrap_or(&self.resolver),
5707 )
5708 } else {
5709 validate_anchor_once_syntax(
5710 &ast,
5711 family.anchor_coord0.row,
5712 family.anchor_coord0.col,
5713 &domain,
5714 )
5715 };
5716 relocation.map_err(SourceFamilyPreparationError::ast)?;
5717 let ast_id = self.intern_formula_ast(&ast);
5718 let placement = CellRef::new(
5719 sheet_id,
5720 Coord::from_excel(
5721 family.anchor_coord0.row + 1,
5722 family.anchor_coord0.col + 1,
5723 true,
5724 true,
5725 ),
5726 );
5727 let ingested = {
5728 let mut pipeline = match planning_provider {
5729 Some(provider) => self.graph.ingest_pipeline(provider),
5730 None => self.ingest_pipeline(),
5731 };
5732 if allow_function_closure {
5733 pipeline = pipeline.enable_function_semantics();
5734 }
5735 pipeline.ingest_formula(
5736 FormulaAstInput::RawArena(ast_id),
5737 placement,
5738 Some(family.anchor_text.clone()),
5739 )
5740 }
5741 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5742 let function_semantics_used = ingested
5743 .labels
5744 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5745 let candidate = FormulaPlacementCandidate::new(
5746 sheet_id,
5747 family.anchor_coord0.row,
5748 family.anchor_coord0.col,
5749 ingested.ast_id,
5750 Some(family.anchor_text.clone()),
5751 );
5752 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5753 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
5754 prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
5755 .map(|prepared| (prepared, function_semantics_used))
5756 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
5757 }
5758
5759 fn placement_domain_from_transport(
5760 sheet_id: SheetId,
5761 transport: crate::engine::PlacementDomainTransport,
5762 ) -> PlacementDomain {
5763 match transport {
5764 crate::engine::PlacementDomainTransport::RowRun {
5765 row_start,
5766 row_end,
5767 col,
5768 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5769 crate::engine::PlacementDomainTransport::ColRun {
5770 row,
5771 col_start,
5772 col_end,
5773 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5774 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5775 sheet_id,
5776 rect.start.row,
5777 rect.end.row,
5778 rect.start.col,
5779 rect.end.col,
5780 ),
5781 }
5782 }
5783
5784 #[cfg(test)]
5785 pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
5786 &mut self,
5787 sheet_name: &str,
5788 source_id: crate::engine::SourceFamilyId,
5789 expected: crate::engine::PartitionLegacyMember,
5790 replay: crate::engine::DeferredReplayFormula,
5791 ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
5792 {
5793 let sheet_id = self.graph.sheet_id_mut(sheet_name);
5794 self.graph
5795 .analyze_fragmented_exact_replay_record(
5796 &self.resolver,
5797 sheet_id,
5798 source_id,
5799 expected,
5800 replay,
5801 )
5802 .map_err(|error| format!("{error:?}"))
5803 }
5804
5805 pub(crate) fn prepare_fragmented_source_transaction(
5806 &mut self,
5807 source: &crate::engine::PartitionedSourceFormulaFamily,
5808 disposition: &crate::engine::FormulaReplayDisposition,
5809 prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5810 legacy_formulas: Vec<
5811 crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
5812 >,
5813 ) -> Result<
5814 crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5815 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
5816 > {
5817 let transaction = self.graph.prepare_fragmented_source_transaction(
5818 &self.source_formula_token,
5819 source,
5820 disposition,
5821 prepared,
5822 legacy_formulas,
5823 )?;
5824 self.prepared_legacy_admission(
5825 transaction.legacy_graph(),
5826 transaction.materialization_cells(),
5827 )
5828 .map_err(
5829 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
5830 )?;
5831 Ok(transaction)
5832 }
5833
5834 pub(crate) fn commit_fragmented_source_transaction(
5835 &mut self,
5836 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5837 disposition: &crate::engine::FormulaReplayDisposition,
5838 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5839 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5840 let provider_revision = self.resolver.planning_semantic_revision();
5841 let epoch = registry_guard.epoch();
5842 let resolver = &self.resolver;
5843 let decision = self.graph.commit_fragmented_source_transaction(
5844 prepared,
5845 &self.source_formula_token,
5846 disposition,
5847 epoch,
5848 provider_revision,
5849 || resolver.planning_semantic_revision(),
5850 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5851 );
5852 drop(registry_guard);
5853 decision
5854 }
5855
5856 #[cfg(test)]
5857 pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
5858 &mut self,
5859 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5860 disposition: &crate::engine::FormulaReplayDisposition,
5861 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5862 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5863 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5864 let provider_revision = self.resolver.planning_semantic_revision();
5865 let epoch = registry_guard.epoch();
5866 let resolver = &self.resolver;
5867 let decision = self.graph.commit_fragmented_source_transaction(
5868 prepared,
5869 &self.source_formula_token,
5870 disposition,
5871 epoch,
5872 provider_revision,
5873 || resolver.planning_semantic_revision(),
5874 fault,
5875 );
5876 drop(registry_guard);
5877 decision
5878 }
5879
5880 #[cfg(test)]
5881 pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
5882 &mut self,
5883 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5884 disposition: &crate::engine::FormulaReplayDisposition,
5885 before_second_sample: impl FnOnce(),
5886 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5887 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5888 let provider_revision = self.resolver.planning_semantic_revision();
5889 let epoch = registry_guard.epoch();
5890 let resolver = &self.resolver;
5891 let decision = self.graph.commit_fragmented_source_transaction(
5892 prepared,
5893 &self.source_formula_token,
5894 disposition,
5895 epoch,
5896 provider_revision,
5897 || {
5898 before_second_sample();
5899 resolver.planning_semantic_revision()
5900 },
5901 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5902 );
5903 drop(registry_guard);
5904 decision
5905 }
5906
5907 #[cfg(test)]
5908 pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
5909 &mut self,
5910 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5911 disposition: &crate::engine::FormulaReplayDisposition,
5912 function_semantic_epoch: u64,
5913 function_provider_revision: Option<u64>,
5914 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5915 self.graph.commit_fragmented_source_transaction(
5916 prepared,
5917 &self.source_formula_token,
5918 disposition,
5919 function_semantic_epoch,
5920 function_provider_revision,
5921 || function_provider_revision,
5922 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5923 )
5924 }
5925
5926 pub(crate) fn analyze_partitioned_source_family_for_transaction(
5927 &mut self,
5928 sheet_name: &str,
5929 family: &crate::engine::PartitionedSourceFormulaFamily,
5930 ) -> Result<
5931 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5932 SourceFamilyPreparationError,
5933 > {
5934 family
5935 .validate(&self.workbook_load_limits)
5936 .map_err(SourceFamilyPreparationError::contract)?;
5937 let formula = if family.template_text.starts_with('=') {
5938 family.template_text.to_string()
5939 } else {
5940 format!("={}", family.template_text)
5941 };
5942 let ast = formualizer_parse::parser::parse(&formula)
5943 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5944 let mut requests = Vec::new();
5945 Self::collect_planning_function_requests(&ast, &mut requests);
5946 let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
5947 &self.resolver,
5948 requests,
5949 )
5950 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5951 let sheet_id = self.graph.sheet_id_mut(sheet_name);
5952 let domains: Vec<_> = family
5953 .fragments
5954 .iter()
5955 .copied()
5956 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
5957 .collect();
5958 for domain in &domains {
5959 validate_anchor_once_shadow_relocation(
5960 &ast,
5961 family.template_origin0.row,
5962 family.template_origin0.col,
5963 domain,
5964 &snapshot,
5965 )
5966 .map_err(SourceFamilyPreparationError::ast)?;
5967 }
5968 let ast_id = self.intern_formula_ast(&ast);
5969 let placement = CellRef::new(
5970 sheet_id,
5971 Coord::from_excel(
5972 family.template_origin0.row + 1,
5973 family.template_origin0.col + 1,
5974 true,
5975 true,
5976 ),
5977 );
5978 let ingested = self
5979 .graph
5980 .ingest_pipeline(&snapshot)
5981 .enable_function_semantics()
5982 .ingest_formula(
5983 FormulaAstInput::RawArena(ast_id),
5984 placement,
5985 Some(family.template_text.clone()),
5986 )
5987 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5988 let function_semantics_used = ingested
5989 .labels
5990 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5991 let name_assumptions = self
5992 .graph
5993 .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
5994 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5995 let candidate = FormulaPlacementCandidate::new(
5996 sheet_id,
5997 family.template_origin0.row,
5998 family.template_origin0.col,
5999 ingested.ast_id,
6000 Some(family.template_text.clone()),
6001 );
6002 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6003 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6004 let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
6005 total
6006 .checked_add(domain.cell_count())
6007 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
6008 })?;
6009 let mut placements = Vec::with_capacity(domains.len());
6010 for domain in domains {
6011 let member_count = domain.cell_count();
6012 placements.push(
6013 prepare_anchor_once_fragment(
6014 candidate.clone(),
6015 analysis.clone(),
6016 domain,
6017 member_count,
6018 direct_cells,
6019 )
6020 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
6021 );
6022 }
6023 let fallback_cells = family.legacy_members.shared_member_count() as u64;
6024 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6025 return Err(SourceFamilyPreparationError::analysis(
6026 "PartitionMemberCountMismatch",
6027 ));
6028 }
6029 Ok(
6030 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
6031 engine_token: Arc::clone(&self.source_formula_token),
6032 sheet_id,
6033 sheet_name: Arc::from(sheet_name),
6034 source: family.clone(),
6035 placements,
6036 function_semantic_epoch: snapshot.epoch(),
6037 function_provider_revision: snapshot.provider_revision(),
6038 function_semantics_used,
6039 name_assumptions,
6040 direct_cells,
6041 },
6042 )
6043 }
6044
6045 fn prepare_partitioned_source_formula_family_shadow(
6046 &mut self,
6047 sheet_id: SheetId,
6048 family: &crate::engine::PartitionedSourceFormulaFamily,
6049 ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
6050 family
6051 .validate(&self.workbook_load_limits)
6052 .map_err(SourceFamilyPreparationError::contract)?;
6053 let formula = if family.template_text.starts_with('=') {
6054 family.template_text.to_string()
6055 } else {
6056 format!("={}", family.template_text)
6057 };
6058 let ast = formualizer_parse::parser::parse(&formula)
6059 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6060 let domains: Vec<_> = family
6061 .fragments
6062 .iter()
6063 .copied()
6064 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6065 .collect();
6066 for domain in &domains {
6067 validate_anchor_once_shadow_relocation(
6068 &ast,
6069 family.template_origin0.row,
6070 family.template_origin0.col,
6071 domain,
6072 &self.resolver,
6073 )
6074 .map_err(SourceFamilyPreparationError::ast)?;
6075 }
6076 let ast_id = self.intern_formula_ast(&ast);
6077 let placement = CellRef::new(
6078 sheet_id,
6079 Coord::from_excel(
6080 family.template_origin0.row + 1,
6081 family.template_origin0.col + 1,
6082 true,
6083 true,
6084 ),
6085 );
6086 let ingested = self
6087 .ingest_pipeline()
6088 .enable_function_semantics()
6089 .ingest_formula(
6090 FormulaAstInput::RawArena(ast_id),
6091 placement,
6092 Some(family.template_text.clone()),
6093 )
6094 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6095 let function_semantics_used = ingested
6096 .labels
6097 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6098 let candidate = FormulaPlacementCandidate::new(
6099 sheet_id,
6100 family.template_origin0.row,
6101 family.template_origin0.col,
6102 ingested.ast_id,
6103 Some(family.template_text.clone()),
6104 );
6105 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6106 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6107 let fragment_count = domains.len() as u64;
6108 let mut direct_cells = 0u64;
6109 for domain in domains {
6110 let member_count = domain.cell_count();
6111 validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6112 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6113 direct_cells = direct_cells
6114 .checked_add(member_count)
6115 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6116 }
6117 let fallback_cells = family.legacy_members.shared_member_count() as u64;
6118 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6119 return Err(SourceFamilyPreparationError::analysis(
6120 "PartitionMemberCountMismatch",
6121 ));
6122 }
6123 Ok(PreparedPartitionShadow {
6124 fragment_count,
6125 direct_cells,
6126 fallback_cells,
6127 function_semantics_used,
6128 })
6129 }
6130
6131 fn analyze_partitioned_formula_plane_shadow(
6132 &mut self,
6133 batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6134 ) -> FormulaIngestReport {
6135 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6136 for (sheet_name, families) in batches {
6137 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6138 for family in families {
6139 report.shadow_candidate_cells = report
6140 .shadow_candidate_cells
6141 .saturating_add(family.surviving_member_count);
6142 match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6143 Ok(prepared) => {
6144 let fragment_count = prepared.fragment_count;
6145 let descendants = family.surviving_member_count.saturating_sub(1);
6146 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6147 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6148 report.source_anchor_analyses =
6149 report.source_anchor_analyses.saturating_add(1);
6150 report.source_partition_analyses_reused = report
6151 .source_partition_analyses_reused
6152 .saturating_add(fragment_count.saturating_sub(1));
6153 report.source_partitioned_families_prepared = report
6154 .source_partitioned_families_prepared
6155 .saturating_add(1);
6156 report.source_partition_fragments_prepared = report
6157 .source_partition_fragments_prepared
6158 .saturating_add(fragment_count);
6159 report.source_partition_span_cells_prepared = report
6160 .source_partition_span_cells_prepared
6161 .saturating_add(prepared.direct_cells);
6162 report.source_partition_fallback_cells = report
6163 .source_partition_fallback_cells
6164 .saturating_add(prepared.fallback_cells);
6165 report.shadow_accepted_span_cells = report
6166 .shadow_accepted_span_cells
6167 .saturating_add(prepared.direct_cells);
6168 report.shadow_fallback_cells = report
6169 .shadow_fallback_cells
6170 .saturating_add(prepared.fallback_cells);
6171 report.source_descendant_strings_avoided = report
6172 .source_descendant_strings_avoided
6173 .saturating_add(descendants);
6174 report.source_descendant_events_avoided = report
6175 .source_descendant_events_avoided
6176 .saturating_add(descendants);
6177 report.source_descendant_analyses_avoided = report
6178 .source_descendant_analyses_avoided
6179 .saturating_add(descendants);
6180 report.graph_formula_vertices_avoided_shadow = report
6181 .graph_formula_vertices_avoided_shadow
6182 .saturating_add(prepared.direct_cells);
6183 report.ast_roots_avoided_shadow = report
6184 .ast_roots_avoided_shadow
6185 .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6186 report.edge_rows_avoided_shadow = report
6187 .edge_rows_avoided_shadow
6188 .saturating_add(prepared.direct_cells);
6189 if prepared.function_semantics_used {
6190 report.source_partition_function_semantics =
6191 report.source_partition_function_semantics.saturating_add(1);
6192 }
6193 }
6194 Err(error) => {
6195 if error.parse_attempted {
6196 report.source_anchor_parses =
6197 report.source_anchor_parses.saturating_add(1);
6198 }
6199 if error.ast_created {
6200 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6201 }
6202 if error.analysis_created {
6203 report.source_anchor_analyses =
6204 report.source_anchor_analyses.saturating_add(1);
6205 }
6206 report.shadow_fallback_cells = report
6207 .shadow_fallback_cells
6208 .saturating_add(family.surviving_member_count);
6209 report.source_partitioned_families_rejected = report
6210 .source_partitioned_families_rejected
6211 .saturating_add(1);
6212 *report.fallback_reasons.entry(error.reason).or_default() += 1;
6213 }
6214 }
6215 }
6216 }
6217 report
6218 }
6219
6220 fn analyze_compressed_formula_plane_shadow(
6221 &mut self,
6222 batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6223 ) -> FormulaIngestReport {
6224 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6225 let _active_epoch = self.graph.formula_authority().plane.epoch();
6226
6227 for (sheet_name, families) in batches {
6228 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6229 for family in families {
6230 let descendants = family.member_count.saturating_sub(1);
6231 report.shadow_candidate_cells = report
6232 .shadow_candidate_cells
6233 .saturating_add(family.member_count);
6234
6235 match self.prepare_source_formula_family(sheet_id, family, true, None) {
6236 Ok(_) => {
6237 report.source_descendant_strings_avoided = report
6238 .source_descendant_strings_avoided
6239 .saturating_add(descendants);
6240 report.source_descendant_events_avoided = report
6241 .source_descendant_events_avoided
6242 .saturating_add(descendants);
6243 report.source_descendant_analyses_avoided = report
6244 .source_descendant_analyses_avoided
6245 .saturating_add(descendants);
6246 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6247 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6248 report.source_anchor_analyses =
6249 report.source_anchor_analyses.saturating_add(1);
6250 report.shadow_accepted_span_cells = report
6251 .shadow_accepted_span_cells
6252 .saturating_add(family.member_count);
6253 report.source_compressed_families_prepared =
6254 report.source_compressed_families_prepared.saturating_add(1);
6255 report.source_compressed_cells_prepared = report
6256 .source_compressed_cells_prepared
6257 .saturating_add(family.member_count);
6258 report.graph_formula_vertices_avoided_shadow = report
6259 .graph_formula_vertices_avoided_shadow
6260 .saturating_add(family.member_count);
6261 report.ast_roots_avoided_shadow =
6262 report.ast_roots_avoided_shadow.saturating_add(descendants);
6263 report.edge_rows_avoided_shadow = report
6264 .edge_rows_avoided_shadow
6265 .saturating_add(family.member_count);
6266 }
6267 Err(error) => {
6268 if error.parse_attempted {
6269 report.source_anchor_parses =
6270 report.source_anchor_parses.saturating_add(1);
6271 }
6272 if error.ast_created {
6273 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6274 }
6275 if error.analysis_created {
6276 report.source_anchor_analyses =
6277 report.source_anchor_analyses.saturating_add(1);
6278 }
6279 report.shadow_fallback_cells = report
6280 .shadow_fallback_cells
6281 .saturating_add(family.member_count);
6282 *report.fallback_reasons.entry(error.reason).or_default() += 1;
6283 }
6284 }
6285 }
6286 }
6287 report
6288 }
6289
6290 fn analyze_formula_plane_shadow_candidates(
6291 &mut self,
6292 batches: &[FormulaIngestBatch],
6293 ) -> FormulaIngestReport {
6294 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6295 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6296
6297 let _active_epoch = self.graph.formula_authority().plane.epoch();
6300
6301 let batch_sheet_ids: Vec<SheetId> = batches
6302 .iter()
6303 .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
6304 .collect();
6305 let mut groups: BTreeMap<
6306 (SheetId, u64, u32),
6307 Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
6308 > = BTreeMap::new();
6309 {
6310 let mut pipeline = self.ingest_pipeline();
6311 for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
6312 for record in &batch.formulas {
6313 if record.row == 0 || record.col == 0 {
6314 report.shadow_candidate_cells =
6315 report.shadow_candidate_cells.saturating_add(1);
6316 report.shadow_fallback_cells =
6317 report.shadow_fallback_cells.saturating_add(1);
6318 Self::record_shadow_fallback_reason(
6319 &mut report,
6320 PlacementFallbackReason::UnsupportedShapeOrGaps,
6321 1,
6322 );
6323 continue;
6324 }
6325
6326 let placement = CellRef::new(
6327 sheet_id,
6328 Coord::from_excel(record.row, record.col, true, true),
6329 );
6330 let ingested = match pipeline.ingest_formula(
6331 FormulaAstInput::RawArena(record.ast_id),
6332 placement,
6333 record.formula_text.clone(),
6334 ) {
6335 Ok(ingested) => ingested,
6336 Err(_) => {
6337 report.shadow_candidate_cells =
6338 report.shadow_candidate_cells.saturating_add(1);
6339 report.shadow_fallback_cells =
6340 report.shadow_fallback_cells.saturating_add(1);
6341 Self::record_shadow_fallback_reason(
6342 &mut report,
6343 PlacementFallbackReason::UnsupportedCanonicalTemplate,
6344 1,
6345 );
6346 continue;
6347 }
6348 };
6349 let candidate = FormulaPlacementCandidate::new(
6350 sheet_id,
6351 record.row - 1,
6352 record.col - 1,
6353 ingested.ast_id,
6354 record.formula_text.clone(),
6355 );
6356 let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
6357 Ok(analysis) => analysis,
6358 Err(reason) => {
6359 report.shadow_candidate_cells =
6360 report.shadow_candidate_cells.saturating_add(1);
6361 report.shadow_fallback_cells =
6362 report.shadow_fallback_cells.saturating_add(1);
6363 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6364 continue;
6365 }
6366 };
6367 groups
6368 .entry((
6369 sheet_id,
6370 ingested.parameterized_canonical_hash,
6371 candidate.col,
6372 ))
6373 .or_default()
6374 .push((candidate, analysis));
6375 }
6376 }
6377 }
6378
6379 let mut scratch_plane = FormulaPlane::default();
6380 for entries in groups.into_values() {
6381 let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
6382 for (component, component_analyses) in
6383 Self::split_candidate_components_with_analyses(candidates, analyses)
6384 {
6385 let placement_report = place_candidate_family_with_analyses(
6386 &mut scratch_plane,
6387 component,
6388 component_analyses,
6389 );
6390 let counters = placement_report.counters;
6391 report.shadow_candidate_cells = report
6392 .shadow_candidate_cells
6393 .saturating_add(counters.formula_cells_seen);
6394 report.shadow_accepted_span_cells = report
6395 .shadow_accepted_span_cells
6396 .saturating_add(counters.accepted_span_cells);
6397 report.shadow_fallback_cells = report
6398 .shadow_fallback_cells
6399 .saturating_add(counters.legacy_cells);
6400 report.shadow_templates_interned = report
6401 .shadow_templates_interned
6402 .saturating_add(counters.templates_interned);
6403 report.shadow_spans_created = report
6404 .shadow_spans_created
6405 .saturating_add(counters.spans_created);
6406 report.graph_formula_vertices_avoided_shadow = report
6407 .graph_formula_vertices_avoided_shadow
6408 .saturating_add(counters.formula_vertices_avoided);
6409 report.ast_roots_avoided_shadow = report
6410 .ast_roots_avoided_shadow
6411 .saturating_add(counters.ast_roots_avoided);
6412 report.edge_rows_avoided_shadow = report
6413 .edge_rows_avoided_shadow
6414 .saturating_add(counters.edge_rows_avoided);
6415 for (reason, count) in counters.fallback_reasons {
6416 Self::record_shadow_fallback_reason(&mut report, reason, count);
6417 }
6418 }
6419 }
6420 report
6421 }
6422
6423 fn record_shadow_fallback_reason(
6424 report: &mut FormulaIngestReport,
6425 reason: PlacementFallbackReason,
6426 count: u64,
6427 ) {
6428 *report
6429 .fallback_reasons
6430 .entry(format!("{reason:?}"))
6431 .or_default() += count;
6432 }
6433
6434 fn analyze_formula_plane_authoritative_ingest(
6435 &mut self,
6436 batches: &[FormulaIngestBatch],
6437 ) -> (
6438 FormulaIngestReport,
6439 Vec<FormulaIngestBatch>,
6440 PlannedFormulaMaterialize,
6441 ) {
6442 let existing_span_refs = self
6443 .graph
6444 .formula_authority()
6445 .active_span_refs()
6446 .into_iter()
6447 .collect::<rustc_hash::FxHashSet<_>>();
6448 let mut report =
6449 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6450 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6451
6452 let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
6453 let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
6454 let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
6455
6456 for batch in batches {
6457 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6458 for record in &batch.formulas {
6459 if record.row == 0 || record.col == 0 {
6460 report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
6461 report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
6462 Self::record_shadow_fallback_reason(
6463 &mut report,
6464 PlacementFallbackReason::UnsupportedShapeOrGaps,
6465 1,
6466 );
6467 fallback
6468 .entry(batch.sheet_name.clone())
6469 .or_default()
6470 .push(record.clone());
6471 continue;
6472 }
6473
6474 pending_candidates.push((
6475 batch.sheet_name.clone(),
6476 FormulaPlacementCandidate::new(
6477 sheet_id,
6478 record.row - 1,
6479 record.col - 1,
6480 record.ast_id,
6481 record.formula_text.clone(),
6482 ),
6483 ));
6484 }
6485 }
6486
6487 let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
6488 let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
6489 (0..pending_candidates.len()).map(|_| None).collect();
6490 let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
6491 (0..pending_candidates.len()).map(|_| None).collect();
6492 {
6493 let mut pipeline = self.ingest_pipeline();
6494 for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
6495 let placement = CellRef::new(
6496 candidate.sheet_id,
6497 Coord::from_excel(
6498 candidate.row.saturating_add(1),
6499 candidate.col.saturating_add(1),
6500 true,
6501 true,
6502 ),
6503 );
6504 let ingested = pipeline.ingest_formula(
6505 FormulaAstInput::RawArena(candidate.ast_id),
6506 placement,
6507 candidate.formula_text.clone(),
6508 );
6509 match ingested {
6510 Ok(ingested) => {
6511 candidate.ast_id = ingested.ast_id;
6512 let dep_plan = ingested.dep_plan.clone();
6513 match CandidateAnalysis::from_ingested(candidate, &ingested) {
6514 Ok(analysis) => {
6515 analyses_by_index[idx] = Some(analysis);
6516 plans_by_index[idx] = Some(dep_plan);
6517 }
6518 Err(reason) => {
6519 report.shadow_candidate_cells =
6520 report.shadow_candidate_cells.saturating_add(1);
6521 report.shadow_fallback_cells =
6522 report.shadow_fallback_cells.saturating_add(1);
6523 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6524 planned_fallback
6525 .entry(sheet_name.clone())
6526 .or_default()
6527 .push((
6528 candidate.row.saturating_add(1),
6529 candidate.col.saturating_add(1),
6530 candidate.ast_id,
6531 dep_plan,
6532 ));
6533 }
6534 }
6535 }
6536 Err(_) => {
6537 let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
6538 report.shadow_candidate_cells =
6539 report.shadow_candidate_cells.saturating_add(1);
6540 report.shadow_fallback_cells =
6541 report.shadow_fallback_cells.saturating_add(1);
6542 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6543 fallback.entry(sheet_name.clone()).or_default().push(
6544 FormulaIngestRecord::new(
6545 candidate.row.saturating_add(1),
6546 candidate.col.saturating_add(1),
6547 candidate.ast_id,
6548 candidate.formula_text.clone(),
6549 ),
6550 );
6551 }
6552 }
6553 }
6554 }
6555
6556 for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
6557 if analyses_by_index[idx].is_none() {
6558 continue;
6559 }
6560 let canonical_hash = analyses_by_index[idx]
6561 .as_ref()
6562 .expect("remaining candidate has an analysis")
6563 .parameterized_canonical_hash();
6564 groups
6565 .entry((candidate.sheet_id, canonical_hash, candidate.col))
6566 .or_default()
6567 .push(idx);
6568 }
6569
6570 for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
6571 let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
6572 let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
6573 for idx in &candidate_indices {
6574 if let Some(plan) = plans_by_index[*idx].take() {
6577 let candidate = &pending_candidates[*idx].1;
6578 plans_by_coord
6579 .entry((candidate.row, candidate.col))
6580 .or_default()
6581 .push(plan);
6582 }
6583 }
6584 let candidates: Vec<_> = candidate_indices
6585 .iter()
6586 .map(|idx| pending_candidates[*idx].1.clone())
6587 .collect();
6588 let components = Self::split_shadow_candidate_components(candidates);
6589 let analyzed_components =
6590 if components.len() == 1 && components[0].len() == candidate_indices.len() {
6591 let component = components.into_iter().next().expect("one component");
6592 let component_analyses = candidate_indices
6593 .iter()
6594 .map(|idx| {
6595 analyses_by_index[*idx]
6596 .take()
6597 .expect("candidate analysis must be used once")
6598 })
6599 .collect();
6600 vec![(component, component_analyses)]
6601 } else {
6602 let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6603 for idx in candidate_indices.iter().rev() {
6604 let candidate = &pending_candidates[*idx].1;
6605 indices_by_coord
6606 .entry((candidate.row, candidate.col))
6607 .or_default()
6608 .push(*idx);
6609 }
6610
6611 components
6612 .into_iter()
6613 .map(|component| {
6614 let mut component_analyses = Vec::with_capacity(component.len());
6615 for candidate in &component {
6616 let idx = indices_by_coord
6617 .get_mut(&(candidate.row, candidate.col))
6618 .and_then(Vec::pop)
6619 .expect("component candidate must have a precomputed analysis");
6620 component_analyses.push(
6621 analyses_by_index[idx]
6622 .take()
6623 .expect("candidate analysis must be used once"),
6624 );
6625 }
6626 (component, component_analyses)
6627 })
6628 .collect()
6629 };
6630
6631 for (component, component_analyses) in analyzed_components {
6632 for (component, component_analyses) in
6633 split_candidate_affine_literal_runs(component, component_analyses)
6634 {
6635 let placement_report = {
6636 let authority = self.graph.formula_authority_mut();
6637 place_candidate_family_with_analyses(
6638 &mut authority.plane,
6639 component.clone(),
6640 component_analyses,
6641 )
6642 };
6643 Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
6644
6645 let mut candidate_by_placement: FxHashMap<
6653 crate::formula_plane::runtime::PlacementCoord,
6654 &FormulaPlacementCandidate,
6655 > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
6656 for candidate in &component {
6657 candidate_by_placement
6658 .entry(candidate.placement())
6659 .or_insert(candidate);
6660 }
6661 for result in &placement_report.results {
6662 let FormulaPlacementResult::Legacy { placement, .. } = result else {
6663 continue;
6664 };
6665 if let Some(&candidate) = candidate_by_placement.get(placement) {
6666 let plan = plans_by_coord
6667 .get_mut(&(candidate.row, candidate.col))
6668 .and_then(Vec::pop);
6669 if let Some(plan) = plan {
6670 planned_fallback
6671 .entry(sheet_name.clone())
6672 .or_default()
6673 .push((
6674 candidate.row.saturating_add(1),
6675 candidate.col.saturating_add(1),
6676 candidate.ast_id,
6677 plan,
6678 ));
6679 } else {
6680 fallback.entry(sheet_name.clone()).or_default().push(
6681 FormulaIngestRecord::new(
6682 candidate.row.saturating_add(1),
6683 candidate.col.saturating_add(1),
6684 candidate.ast_id,
6685 candidate.formula_text.clone(),
6686 ),
6687 );
6688 }
6689 }
6690 }
6691 }
6692 }
6693 }
6694
6695 let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
6696 let new_span_refs = self
6697 .graph
6698 .formula_authority()
6699 .active_span_refs()
6700 .into_iter()
6701 .filter(|span_ref| !existing_span_refs.contains(span_ref))
6702 .collect::<Vec<_>>();
6703 self.graph
6704 .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
6705
6706 let fallback_batches = fallback
6707 .into_iter()
6708 .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
6709 .collect();
6710 (report, fallback_batches, planned_fallback)
6711 }
6712
6713 fn accumulate_formula_plane_placement_report(
6714 report: &mut FormulaIngestReport,
6715 placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
6716 ) {
6717 let counters = &placement_report.counters;
6718 report.shadow_candidate_cells = report
6719 .shadow_candidate_cells
6720 .saturating_add(counters.formula_cells_seen);
6721 report.shadow_accepted_span_cells = report
6722 .shadow_accepted_span_cells
6723 .saturating_add(counters.accepted_span_cells);
6724 report.shadow_fallback_cells = report
6725 .shadow_fallback_cells
6726 .saturating_add(counters.legacy_cells);
6727 report.shadow_templates_interned = report
6728 .shadow_templates_interned
6729 .saturating_add(counters.templates_interned);
6730 report.shadow_spans_created = report
6731 .shadow_spans_created
6732 .saturating_add(counters.spans_created);
6733 report.graph_formula_vertices_avoided_shadow = report
6734 .graph_formula_vertices_avoided_shadow
6735 .saturating_add(counters.formula_vertices_avoided);
6736 report.ast_roots_avoided_shadow = report
6737 .ast_roots_avoided_shadow
6738 .saturating_add(counters.ast_roots_avoided);
6739 report.edge_rows_avoided_shadow = report
6740 .edge_rows_avoided_shadow
6741 .saturating_add(counters.edge_rows_avoided);
6742 for (reason, count) in &counters.fallback_reasons {
6743 Self::record_shadow_fallback_reason(report, *reason, *count);
6744 }
6745 }
6746
6747 fn split_candidate_components_with_analyses(
6748 candidates: Vec<FormulaPlacementCandidate>,
6749 mut analyses: Vec<CandidateAnalysis>,
6750 ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
6751 let components = Self::split_shadow_candidate_components(candidates.clone());
6752 let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
6753 for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
6754 analysis_by_coord
6755 .entry((candidate.row, candidate.col))
6756 .or_default()
6757 .push(analysis);
6758 }
6759 components
6760 .into_iter()
6761 .flat_map(|component| {
6762 let mut component_analyses = Vec::with_capacity(component.len());
6763 for candidate in &component {
6764 let analysis = analysis_by_coord
6765 .get_mut(&(candidate.row, candidate.col))
6766 .and_then(Vec::pop)
6767 .expect("component candidate must have a precomputed analysis");
6768 component_analyses.push(analysis);
6769 }
6770 split_candidate_affine_literal_runs(component, component_analyses)
6771 })
6772 .collect()
6773 }
6774
6775 fn split_shadow_candidate_components(
6776 candidates: Vec<FormulaPlacementCandidate>,
6777 ) -> Vec<Vec<FormulaPlacementCandidate>> {
6778 if candidates.len() <= 1 {
6779 return vec![candidates];
6780 }
6781
6782 let is_row_run = candidates.windows(2).all(|w| {
6786 w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
6787 });
6788 let is_col_run = candidates.windows(2).all(|w| {
6789 w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
6790 });
6791 if is_row_run || is_col_run {
6792 return vec![candidates];
6793 }
6794
6795 let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6796 for (idx, candidate) in candidates.iter().enumerate() {
6797 coord_to_indices
6798 .entry((candidate.row, candidate.col))
6799 .or_default()
6800 .push(idx);
6801 }
6802
6803 let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
6804 let mut components = Vec::new();
6805 while let Some(&start) = remaining.iter().next() {
6806 remaining.remove(&start);
6807 let mut queue = VecDeque::from([start]);
6808 let mut component_indices = Vec::new();
6809
6810 while let Some(idx) = queue.pop_front() {
6811 component_indices.push(idx);
6812 let candidate = &candidates[idx];
6813 let mut neighbor_coords = Vec::with_capacity(5);
6814 neighbor_coords.push((candidate.row, candidate.col));
6815 if let Some(row) = candidate.row.checked_sub(1) {
6816 neighbor_coords.push((row, candidate.col));
6817 }
6818 neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
6819 if let Some(col) = candidate.col.checked_sub(1) {
6820 neighbor_coords.push((candidate.row, col));
6821 }
6822 neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
6823
6824 for coord in neighbor_coords {
6825 if let Some(indices) = coord_to_indices.get(&coord) {
6826 for &neighbor in indices {
6827 if remaining.remove(&neighbor) {
6828 queue.push_back(neighbor);
6829 }
6830 }
6831 }
6832 }
6833 }
6834
6835 component_indices.sort_by_key(|idx| {
6836 let candidate = &candidates[*idx];
6837 (candidate.row, candidate.col, *idx)
6838 });
6839 components.push(
6840 component_indices
6841 .into_iter()
6842 .map(|idx| candidates[idx].clone())
6843 .collect(),
6844 );
6845 }
6846
6847 components
6848 }
6849
6850 fn collect_planning_function_requests(
6851 ast: &ASTNode,
6852 requests: &mut Vec<(String, String, usize)>,
6853 ) {
6854 match &ast.node_type {
6855 ASTNodeType::Function { name, args } => {
6856 requests.push((String::new(), name.clone(), args.len()));
6857 for arg in args {
6858 Self::collect_planning_function_requests(arg, requests);
6859 }
6860 }
6861 ASTNodeType::BinaryOp { left, right, .. } => {
6862 Self::collect_planning_function_requests(left, requests);
6863 Self::collect_planning_function_requests(right, requests);
6864 }
6865 ASTNodeType::UnaryOp { expr, .. } => {
6866 Self::collect_planning_function_requests(expr, requests);
6867 }
6868 ASTNodeType::Call { callee, args } => {
6869 Self::collect_planning_function_requests(callee, requests);
6870 for arg in args {
6871 Self::collect_planning_function_requests(arg, requests);
6872 }
6873 }
6874 ASTNodeType::Array(rows) => {
6875 for cell in rows.iter().flatten() {
6876 Self::collect_planning_function_requests(cell, requests);
6877 }
6878 }
6879 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6880 }
6881 }
6882
6883 fn prepared_placement_function_semantics_changed(
6884 &self,
6885 semantic_epoch: u64,
6886 prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
6887 guard: &crate::function_registry::SemanticEpochReadGuard,
6888 ) -> bool {
6889 if semantic_epoch == guard.epoch() {
6890 return false;
6891 }
6892 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6893 let Some(ast) = self
6894 .graph
6895 .data_store()
6896 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6897 else {
6898 return true;
6900 };
6901 let mut requests = Vec::new();
6902 Self::collect_planning_function_requests(&ast, &mut requests);
6903 guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
6904 }
6905
6906 fn prepared_function_semantics_changed(
6907 &self,
6908 preparation: &crate::engine::FormulaCompressedPreparation,
6909 guard: &crate::function_registry::SemanticEpochReadGuard,
6910 ) -> bool {
6911 if preparation.function_semantic_epoch == guard.epoch() {
6912 return false;
6913 }
6914
6915 let mut requests = Vec::new();
6916 for (_, _, prepared) in &preparation.prepared {
6917 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6918 let Some(ast) = self
6919 .graph
6920 .data_store()
6921 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6922 else {
6923 return true;
6924 };
6925 Self::collect_planning_function_requests(&ast, &mut requests);
6926 }
6927 guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
6928 }
6929
6930 pub(crate) fn prepare_source_formula_families(
6931 &mut self,
6932 sheet_name: &str,
6933 families: &[crate::engine::SourceFormulaFamily],
6934 ) -> crate::engine::FormulaCompressedPreparation {
6935 let mut preparation = crate::engine::FormulaCompressedPreparation {
6936 engine_token: Arc::clone(&self.source_formula_token),
6937 function_semantic_epoch: crate::function_registry::semantic_epoch(),
6938 function_provider_revision: None,
6939 function_semantics_used: false,
6940 sheet_name: Arc::from(sheet_name),
6941 prepared: Vec::new(),
6942 rejected: BTreeMap::new(),
6943 fragmented: Vec::new(),
6944 fragmented_sources: BTreeMap::new(),
6945 eager_replay: Vec::new(),
6946 preparation_spool_replays: 0,
6947 clean_rejected_anchor_counts: [0; 3],
6948 fragmented_rejected_anchor_counts: [0; 3],
6949 exact_replay: None,
6950 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6951 };
6952 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6953 return preparation;
6954 }
6955
6956 let mut requests = Vec::new();
6957 for family in families {
6958 let formula = if family.anchor_text.starts_with('=') {
6959 family.anchor_text.to_string()
6960 } else {
6961 format!("={}", family.anchor_text)
6962 };
6963 if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
6964 Self::collect_planning_function_requests(&ast, &mut requests);
6965 }
6966 }
6967 requests.sort();
6968 requests.dedup();
6969 let snapshot =
6970 match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6971 &self.resolver,
6972 requests,
6973 ) {
6974 Ok(snapshot) => snapshot,
6975 Err(error) => {
6976 let reason = match error {
6977 crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
6978 "FunctionSemanticSnapshotUnavailable"
6979 }
6980 crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
6981 "FunctionProviderRevisionUnavailable"
6982 }
6983 };
6984 for family in families {
6985 preparation
6986 .rejected
6987 .insert(family.source_id, reason.to_string());
6988 }
6989 return preparation;
6990 }
6991 };
6992 preparation.function_semantic_epoch = snapshot.epoch();
6993 preparation.function_provider_revision = snapshot.provider_revision();
6994
6995 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6996 for family in families {
6997 #[cfg(test)]
6998 let force_replay = self.force_source_family_fallback
6999 || cfg!(feature = "benchmark_internal")
7000 && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7001 #[cfg(all(feature = "benchmark_internal", not(test)))]
7002 let force_replay =
7003 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7004 #[cfg(any(test, feature = "benchmark_internal"))]
7005 if force_replay {
7006 preparation
7007 .rejected
7008 .insert(family.source_id, "ForcedReplay".to_string());
7009 continue;
7010 }
7011 match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
7012 Ok((prepared, function_semantics_used)) => {
7013 preparation.function_semantics_used |= function_semantics_used;
7014 preparation
7015 .replay_disposition
7016 .set_family_direct(family.source_id);
7017 preparation
7018 .prepared
7019 .push((family.source_id, family.source_order, prepared));
7020 }
7021 Err(error) => {
7022 preparation.clean_rejected_anchor_counts[0] = preparation
7023 .clean_rejected_anchor_counts[0]
7024 .saturating_add(u64::from(error.parse_attempted));
7025 preparation.clean_rejected_anchor_counts[1] = preparation
7026 .clean_rejected_anchor_counts[1]
7027 .saturating_add(u64::from(error.ast_created));
7028 preparation.clean_rejected_anchor_counts[2] = preparation
7029 .clean_rejected_anchor_counts[2]
7030 .saturating_add(u64::from(error.analysis_created));
7031 preparation.rejected.insert(family.source_id, error.reason);
7032 }
7033 }
7034 }
7035 preparation
7036 }
7037
7038 fn prepare_source_formula_proposals(
7039 &mut self,
7040 sheet_name: &str,
7041 families: &[crate::engine::SourceFormulaFamily],
7042 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7043 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7044 formula_record_count: u64,
7045 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
7046 suppressed: &BTreeSet<(u32, u32)>,
7047 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
7048 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
7049 preparation.exact_replay = Some(Arc::clone(&replay));
7050 preparation
7051 .replay_disposition
7052 .extend_suppressed_excel_coords(suppressed.iter().copied());
7053 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7054 return Ok(preparation);
7055 }
7056 #[cfg(feature = "benchmark_internal")]
7057 let benchmark_forced_replay =
7058 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7059 #[cfg(not(feature = "benchmark_internal"))]
7060 let benchmark_forced_replay = false;
7061 let authority_partitions = if benchmark_forced_replay {
7062 for partition in authority_partitions {
7063 preparation
7064 .rejected
7065 .insert(partition.source_id, "ForcedReplay".to_string());
7066 }
7067 &[][..]
7068 } else {
7069 authority_partitions
7070 };
7071 let authority_ids: BTreeSet<_> = authority_partitions
7072 .iter()
7073 .map(|partition| partition.source_id)
7074 .collect();
7075 for partition in replay_partitions {
7076 if !authority_ids.contains(&partition.source_id) {
7077 preparation
7078 .replay_disposition
7079 .register_partition(partition, false)
7080 .map_err(|reason| {
7081 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7082 })?;
7083 }
7084 }
7085 if replay_partitions.is_empty()
7086 && preparation.rejected.is_empty()
7087 && preparation.direct_cell_count() == formula_record_count
7088 {
7089 return Ok(preparation);
7090 }
7091
7092 let mut analyzed = Vec::new();
7093 for source in authority_partitions {
7094 preparation
7095 .fragmented_sources
7096 .insert(source.source_id, source.clone());
7097 match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7098 Ok(prepared) => {
7099 let mut candidate = preparation.replay_disposition.clone();
7100 match candidate.register_partition(source, true) {
7101 Ok(()) => {
7102 preparation.replay_disposition = candidate;
7103 analyzed.push((source.clone(), prepared));
7104 }
7105 Err(reason) => {
7106 preparation
7107 .rejected
7108 .insert(source.source_id, reason.to_string());
7109 preparation
7110 .replay_disposition
7111 .register_partition(source, false)
7112 .map_err(|reason| {
7113 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7114 })?;
7115 }
7116 }
7117 }
7118 Err(error) => {
7119 preparation.fragmented_rejected_anchor_counts[0] = preparation
7120 .fragmented_rejected_anchor_counts[0]
7121 .saturating_add(u64::from(error.parse_attempted));
7122 preparation.fragmented_rejected_anchor_counts[1] = preparation
7123 .fragmented_rejected_anchor_counts[1]
7124 .saturating_add(u64::from(error.ast_created));
7125 preparation.fragmented_rejected_anchor_counts[2] = preparation
7126 .fragmented_rejected_anchor_counts[2]
7127 .saturating_add(u64::from(error.analysis_created));
7128 preparation.rejected.insert(source.source_id, error.reason);
7129 preparation
7130 .replay_disposition
7131 .register_partition(source, false)
7132 .map_err(|reason| {
7133 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7134 })?;
7135 }
7136 }
7137 }
7138
7139 let initial_replay = replay
7140 .lock()
7141 .map_err(|_| {
7142 ExcelError::new(ExcelErrorKind::Value)
7143 .with_message("compressed formula exact replay lock poisoned")
7144 })?
7145 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7146 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7147 preparation.preparation_spool_replays = 1;
7148 let sheet_id = self.graph.sheet_id_mut(sheet_name);
7149 let mut preparation_failed = false;
7150 for (source, prepared_family) in analyzed {
7151 let result = (|| {
7152 let expected: BTreeMap<_, _> = source
7153 .legacy_members
7154 .as_slice()
7155 .iter()
7156 .map(|member| (member.coord, member.kind))
7157 .collect();
7158 let mut actual = BTreeMap::new();
7159 for record in initial_replay
7160 .iter()
7161 .filter(|record| record.partition_owner == Some(source.source_id))
7162 {
7163 let coord = record
7164 .row
7165 .checked_sub(1)
7166 .zip(record.col.checked_sub(1))
7167 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7168 .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7169 let kind = match record.family {
7170 Some(family) if family == source.source_id => {
7171 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7172 }
7173 None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7174 Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7175 };
7176 if actual.insert(coord, (kind, record.clone())).is_some() {
7177 return Err("ExactReplayRecordDuplicate".to_string());
7178 }
7179 }
7180 let actual_kinds: BTreeMap<_, _> = actual
7181 .iter()
7182 .map(|(coord, (kind, _))| (*coord, *kind))
7183 .collect();
7184 if actual_kinds != expected {
7185 return Err("ExactReplayLegacySetMismatch".to_string());
7186 }
7187 let mut legacy = Vec::with_capacity(source.legacy_members.len());
7188 for expected in source.legacy_members.as_slice() {
7189 let record = actual
7190 .remove(&expected.coord)
7191 .map(|(_, record)| record)
7192 .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7193 legacy.push(
7194 self.graph
7195 .analyze_fragmented_exact_replay_record(
7196 &self.resolver,
7197 sheet_id,
7198 source.source_id,
7199 *expected,
7200 record,
7201 )
7202 .map_err(|error| format!("{error:?}"))?,
7203 );
7204 }
7205 Ok::<_, String>((prepared_family, legacy))
7206 })();
7207 match result {
7208 Ok((prepared_family, legacy)) => preparation.fragmented.push(
7209 crate::engine::formula_source::PreparedFragmentedSourceProposal {
7210 source: source.clone(),
7211 prepared: prepared_family,
7212 legacy,
7213 replay: Arc::clone(&replay),
7214 },
7215 ),
7216 Err(reason) => {
7217 preparation_failed = true;
7218 for count in &mut preparation.fragmented_rejected_anchor_counts {
7219 *count = count.saturating_add(1);
7220 }
7221 preparation.rejected.insert(source.source_id, reason);
7222 preparation
7223 .replay_disposition
7224 .force_family_legacy(source.source_id);
7225 }
7226 }
7227 }
7228
7229 preparation.eager_replay = if preparation_failed {
7230 preparation.preparation_spool_replays = 2;
7231 replay
7232 .lock()
7233 .map_err(|_| {
7234 ExcelError::new(ExcelErrorKind::Value)
7235 .with_message("compressed formula exact replay lock poisoned")
7236 })?
7237 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7238 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7239 } else {
7240 initial_replay
7241 };
7242 Ok(preparation)
7243 }
7244
7245 fn formula_batch_from_exact_replay(
7246 &mut self,
7247 sheet_name: &str,
7248 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7249 ) -> Result<FormulaIngestBatch, ExcelError> {
7250 let mut cache = rustc_hash::FxHashMap::default();
7251 let mut formulas = Vec::new();
7252 for record in replayed {
7253 let key = if record.text.starts_with('=') {
7254 record.text
7255 } else {
7256 format!("={}", record.text)
7257 };
7258 let ast_id = if let Some(cached) = cache.get(&key) {
7259 *cached
7260 } else {
7261 let parsed = match formualizer_parse::parser::parse(&key) {
7262 Ok(parsed) => parsed,
7263 Err(error) => {
7264 let Some(parsed) = self.handle_formula_parse_error(
7265 sheet_name,
7266 record.row,
7267 record.col,
7268 &key,
7269 error.to_string(),
7270 )?
7271 else {
7272 continue;
7273 };
7274 parsed
7275 }
7276 };
7277 let ast_id = self.intern_formula_ast(&parsed);
7278 cache.insert(key.clone(), ast_id);
7279 ast_id
7280 };
7281 formulas.push(
7282 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
7283 .with_source_proof(record.source_order, record.family, record.partition_owner),
7284 );
7285 }
7286 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
7287 }
7288
7289 fn replay_prepared_families_exact_records(
7290 &mut self,
7291 preparation: &crate::engine::FormulaCompressedPreparation,
7292 ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
7293 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7294 ExcelError::new(ExcelErrorKind::Value)
7295 .with_message("stale compressed preparation has no exact replay owner")
7296 })?;
7297 let direct: BTreeSet<_> = preparation
7298 .prepared
7299 .iter()
7300 .map(|(family, _, _)| *family)
7301 .collect();
7302 let mut replay_disposition = preparation.replay_disposition.clone();
7303 for family in &direct {
7304 replay_disposition.force_family_legacy(*family);
7305 }
7306 let replayed = replay
7307 .lock()
7308 .map_err(|_| {
7309 ExcelError::new(ExcelErrorKind::Value)
7310 .with_message("compressed formula exact replay lock poisoned")
7311 })?
7312 .replay(&replay_disposition)
7313 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7314 let mut records: Vec<_> = replayed
7315 .into_iter()
7316 .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
7317 .collect();
7318 records.sort_by_key(|record| record.source_order);
7319 let expected: u64 = preparation
7320 .prepared
7321 .iter()
7322 .map(|(_, _, prepared)| prepared.member_count)
7323 .sum();
7324 if records.len() as u64 != expected {
7325 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7326 "compressed formula exact replay incomplete: expected {expected}, got {}",
7327 records.len()
7328 )));
7329 }
7330 Ok(records)
7331 }
7332
7333 fn replay_one_prepared_family_exact(
7334 &mut self,
7335 preparation: &crate::engine::FormulaCompressedPreparation,
7336 family: crate::engine::SourceFamilyId,
7337 member_count: u64,
7338 ) -> Result<FormulaIngestBatch, ExcelError> {
7339 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7340 ExcelError::new(ExcelErrorKind::Value)
7341 .with_message("stale compressed preparation has no exact replay owner")
7342 })?;
7343 let mut replay_disposition = preparation.replay_disposition.clone();
7344 replay_disposition.force_family_legacy(family);
7345 let replayed = replay
7346 .lock()
7347 .map_err(|_| {
7348 ExcelError::new(ExcelErrorKind::Value)
7349 .with_message("compressed formula exact replay lock poisoned")
7350 })?
7351 .replay(&replay_disposition)
7352 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7353 .into_iter()
7354 .filter(|record| record.family == Some(family));
7355 let batch =
7356 self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
7357 if batch.formulas.len() as u64 != member_count {
7358 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7359 "compressed formula exact replay incomplete: expected {member_count}, got {}",
7360 batch.formulas.len()
7361 )));
7362 }
7363 Ok(batch)
7364 }
7365
7366 fn exact_replay_record_is_prepared_partition_legacy(
7367 source: &crate::engine::PartitionedSourceFormulaFamily,
7368 record: &crate::engine::DeferredReplayFormula,
7369 ) -> bool {
7370 if record.partition_owner != Some(source.source_id) {
7371 return false;
7372 }
7373 let Some(coord) = record
7374 .row
7375 .checked_sub(1)
7376 .zip(record.col.checked_sub(1))
7377 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7378 else {
7379 return false;
7380 };
7381 source.legacy_members.as_slice().iter().any(|member| {
7382 member.coord == coord
7383 && match member.kind {
7384 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
7385 record.family == Some(source.source_id)
7386 }
7387 crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
7388 record.family.is_none()
7389 }
7390 }
7391 })
7392 }
7393
7394 fn validate_whole_partition_replay(
7395 source: &crate::engine::PartitionedSourceFormulaFamily,
7396 records: &[crate::engine::DeferredReplayFormula],
7397 ) -> Result<(), ExcelError> {
7398 let mut coordinates = BTreeSet::new();
7399 let mut shared = 0u64;
7400 let mut ordinary = 0u64;
7401 for record in records {
7402 let coord = record
7403 .row
7404 .checked_sub(1)
7405 .zip(record.col.checked_sub(1))
7406 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7407 .ok_or_else(|| {
7408 ExcelError::new(ExcelErrorKind::Value)
7409 .with_message("whole-family replay contains an invalid coordinate")
7410 })?;
7411 if !coordinates.insert(coord) {
7412 return Err(ExcelError::new(ExcelErrorKind::Value)
7413 .with_message("whole-family replay contains a duplicate coordinate"));
7414 }
7415 match (record.family, record.partition_owner) {
7416 (Some(family), Some(owner))
7417 if family == source.source_id && owner == source.source_id =>
7418 {
7419 let direct = source.fragments.iter().any(|fragment| {
7420 let rect = fragment.rect();
7421 coord.row >= rect.start.row
7422 && coord.row <= rect.end.row
7423 && coord.col >= rect.start.col
7424 && coord.col <= rect.end.col
7425 });
7426 let legacy = source.legacy_members.as_slice().iter().any(|member| {
7427 member.coord == coord
7428 && member.kind
7429 == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7430 });
7431 if !direct && !legacy {
7432 return Err(ExcelError::new(ExcelErrorKind::Value)
7433 .with_message("whole-family replay contains an extra shared member"));
7434 }
7435 shared = shared.saturating_add(1);
7436 }
7437 (None, Some(owner)) if owner == source.source_id => {
7438 if !source.legacy_members.as_slice().iter().any(|member| {
7439 member.coord == coord
7440 && member.kind
7441 == crate::engine::PartitionLegacyMemberKind::OrdinaryException
7442 }) {
7443 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7444 "whole-family replay contains an extra ordinary exception",
7445 ));
7446 }
7447 ordinary = ordinary.saturating_add(1);
7448 }
7449 _ => {
7450 return Err(ExcelError::new(ExcelErrorKind::Value)
7451 .with_message("whole-family replay ownership mismatch"));
7452 }
7453 }
7454 }
7455 if shared != source.surviving_member_count
7456 || ordinary != source.reconciliation.ordinary_exceptions
7457 {
7458 return Err(ExcelError::new(ExcelErrorKind::Value)
7459 .with_message("whole-family replay is incomplete"));
7460 }
7461 Ok(())
7462 }
7463
7464 fn prepare_target_combined_legacy_graph(
7465 &self,
7466 packages: &[PreparedTargetSourcePackage],
7467 ordinary: &[PreparedOrdinaryStagedFormula],
7468 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
7469 let mut planned_by_coord = BTreeMap::new();
7470 for package in packages {
7471 for (row, col, ast_id, plan) in &package.legacy {
7472 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
7473 }
7474 }
7475 for formula in ordinary {
7476 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
7477 planned_by_coord.insert(
7478 (formula.sheet_id, formula.lease.row, formula.lease.col),
7479 (ast_id, plan),
7480 );
7481 }
7482 }
7483 let planned = planned_by_coord
7484 .into_iter()
7485 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
7486 .collect::<Vec<_>>();
7487 let formula_count = planned.len();
7488 let graph = self
7489 .graph
7490 .prepare_legacy_graph_plan_multi_sheet(planned)
7491 .map_err(|error| {
7492 ExcelError::new(ExcelErrorKind::Value)
7493 .with_message(format!("target graph preparation failed: {error}"))
7494 })?;
7495 Ok((graph, formula_count))
7496 }
7497
7498 fn materialize_target_package_direct_records(
7499 &mut self,
7500 package: &mut PreparedTargetSourcePackage,
7501 assumptions: &crate::engine::PreparationRevision,
7502 planning_requests: &mut BTreeSet<(String, String, usize)>,
7503 deadline: Option<std::time::Instant>,
7504 ) -> Result<(), ExcelError> {
7505 let existing = package
7506 .legacy
7507 .iter()
7508 .map(|(row, col, _, _)| (*row, *col))
7509 .collect::<BTreeSet<_>>();
7510 let mut missing = BTreeMap::new();
7511 for record in &package.replay_records {
7512 if !existing.contains(&(record.row, record.col)) {
7513 missing.insert((record.row, record.col), record.clone());
7514 }
7515 }
7516 let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
7517 for record in batch.formulas {
7518 self.target_preparation_checkpoint(deadline, 1)?;
7519 let ast = self
7520 .graph
7521 .data_store()
7522 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
7523 .ok_or_else(|| {
7524 ExcelError::new(ExcelErrorKind::Value)
7525 .with_message("target fallback AST is unavailable")
7526 })?;
7527 let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
7528 if let Some(reason) =
7529 Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
7530 {
7531 return Err(Self::preparation_stale(
7532 reason,
7533 "target fallback planning snapshot became stale during materialization",
7534 ));
7535 }
7536 let placement = CellRef::new(
7537 package.sheet_id,
7538 Coord::from_excel(record.row, record.col, true, true),
7539 );
7540 let ingested = self
7541 .graph
7542 .ingest_pipeline(&snapshot)
7543 .enable_function_semantics()
7544 .ingest_formula(
7545 FormulaAstInput::RawArena(record.ast_id),
7546 placement,
7547 record.formula_text,
7548 )?;
7549 package
7550 .legacy
7551 .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7552 }
7553 package.direct_families = 0;
7554 package.direct_cells = 0;
7555 package.direct_fragments = 0;
7556 package.direct_complete_families = 0;
7557 package.direct_complete_cells = 0;
7558 package.direct_partition_families = 0;
7559 package.direct_partition_cells = 0;
7560 Ok(())
7561 }
7562
7563 fn prepare_target_source_package(
7564 &mut self,
7565 sheet: &str,
7566 lease: StagedPackageLease,
7567 deadline: Option<std::time::Instant>,
7568 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
7569 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
7570 ExcelError::new(ExcelErrorKind::Ref)
7571 .with_message(format!("deferred source sheet not found: {sheet}"))
7572 })?;
7573 let (
7574 mut source_report,
7575 families,
7576 partitions,
7577 replay,
7578 invalidated,
7579 suppressed,
7580 reconciliation_replay,
7581 ) = {
7582 let package = self
7583 .staged_formulas
7584 .get(sheet)
7585 .and_then(|staged| staged.deferred_package.as_ref())
7586 .ok_or_else(|| {
7587 ExcelError::new(ExcelErrorKind::Value)
7588 .with_message("staged deferred source package is unavailable")
7589 })?;
7590 if package.sheet_name != sheet {
7591 return Err(ExcelError::new(ExcelErrorKind::Value)
7592 .with_message("deferred formula package sheet mismatch"));
7593 }
7594 (
7595 package.report.clone(),
7596 package.families.clone(),
7597 package.partitioned_families.clone(),
7598 Arc::clone(&package.replay),
7599 package.invalidated.clone(),
7600 package.suppressed.clone(),
7601 package.reconciliation_replay.clone(),
7602 )
7603 };
7604
7605 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
7606 for partition in &partitions {
7607 replay_disposition
7608 .register_partition(partition, false)
7609 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
7610 }
7611 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
7612 self.target_preparation_checkpoint(deadline, 1)?;
7613 let mut replay_records = if let Some(mut records) = reconciliation_replay {
7614 records.retain(|record| {
7615 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
7616 else {
7617 return true;
7618 };
7619 let coord = crate::engine::SourceCoord { row, col };
7620 let disposition = record.family.map_or_else(
7621 || replay_disposition.ordinary_disposition(coord).0,
7622 |family| replay_disposition.shared_disposition(family, coord),
7623 );
7624 !matches!(
7625 disposition,
7626 crate::engine::FormulaReplayCoordinateDisposition::Direct
7627 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
7628 )
7629 });
7630 records
7631 } else {
7632 replay
7633 .lock()
7634 .map_err(|_| {
7635 ExcelError::new(ExcelErrorKind::Value)
7636 .with_message("deferred formula spool lock poisoned")
7637 })?
7638 .replay_partitioned(&replay_disposition, &partitions)
7639 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7640 };
7641 replay_records.sort_by_key(|record| record.source_order);
7642 for chunk in replay_records.chunks(256) {
7643 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
7644 }
7645 if replay_records
7646 .windows(2)
7647 .any(|records| records[0].source_order == records[1].source_order)
7648 {
7649 return Err(ExcelError::new(ExcelErrorKind::Value)
7650 .with_message("duplicate deferred source-order proof"));
7651 }
7652
7653 let mut disposition = replay_disposition;
7654 let mut ordered_placements = Vec::new();
7655 let mut direct_families = 0usize;
7656 let mut direct_cells = 0u64;
7657 let mut direct_fragments = 0u64;
7658 let mut direct_complete_families = 0u64;
7659 let mut direct_complete_cells = 0u64;
7660 let mut direct_partition_families = 0u64;
7661 let mut direct_partition_cells = 0u64;
7662 let mut anchor_parses = 0u64;
7663 let mut anchor_asts = 0u64;
7664 let mut anchor_analyses = 0u64;
7665 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
7666 for family in &families {
7667 self.target_preparation_checkpoint(deadline, 1)?;
7668 if invalidated.contains(&family.source_id) {
7669 continue;
7670 }
7671 match self.prepare_source_formula_family(sheet_id, family, true, None) {
7672 Ok((prepared, _)) => {
7673 anchor_parses = anchor_parses.saturating_add(1);
7674 anchor_asts = anchor_asts.saturating_add(1);
7675 anchor_analyses = anchor_analyses.saturating_add(1);
7676 direct_families = direct_families.saturating_add(1);
7677 direct_cells = direct_cells.saturating_add(prepared.member_count);
7678 direct_complete_families = direct_complete_families.saturating_add(1);
7679 direct_complete_cells =
7680 direct_complete_cells.saturating_add(prepared.member_count);
7681 if self.config.formula_plane_mode
7682 == FormulaPlaneMode::AuthoritativeExperimental
7683 {
7684 disposition.set_family_direct(family.source_id);
7685 }
7686 ordered_placements.push((family.source_order, vec![prepared]));
7687 }
7688 Err(error) => {
7689 anchor_parses =
7690 anchor_parses.saturating_add(u64::from(error.parse_attempted));
7691 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7692 anchor_analyses =
7693 anchor_analyses.saturating_add(u64::from(error.analysis_created));
7694 *source_report
7695 .fallback_reasons
7696 .entry(error.reason)
7697 .or_default() += 1;
7698 }
7699 }
7700 }
7701
7702 for source in &partitions {
7703 self.target_preparation_checkpoint(deadline, 1)?;
7704 if invalidated.contains(&source.source_id) {
7705 continue;
7706 }
7707 let prepared =
7708 match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
7709 Ok(prepared) => {
7710 anchor_parses = anchor_parses.saturating_add(1);
7711 anchor_asts = anchor_asts.saturating_add(1);
7712 anchor_analyses = anchor_analyses.saturating_add(1);
7713 prepared
7714 }
7715 Err(error) => {
7716 anchor_parses =
7717 anchor_parses.saturating_add(u64::from(error.parse_attempted));
7718 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7719 anchor_analyses =
7720 anchor_analyses.saturating_add(u64::from(error.analysis_created));
7721 *source_report
7722 .fallback_reasons
7723 .entry(error.reason)
7724 .or_default() += 1;
7725 continue;
7726 }
7727 };
7728 let fragment_regions: Vec<_> = prepared
7729 .placements
7730 .iter()
7731 .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
7732 .collect();
7733 let crosses_fragment =
7734 prepared
7735 .placements
7736 .iter()
7737 .enumerate()
7738 .any(|(fragment_index, placement)| {
7739 placement
7740 .fragment_dependency_proof()
7741 .1
7742 .dependencies
7743 .iter()
7744 .any(|dependency| {
7745 fragment_regions.iter().enumerate().any(
7746 |(candidate_index, candidate)| {
7747 candidate_index != fragment_index
7748 && dependency.read_region.intersects(candidate)
7749 },
7750 )
7751 })
7752 });
7753 let family_records = replay_records
7754 .iter()
7755 .filter(|record| {
7756 record.family == Some(source.source_id)
7757 || record.partition_owner == Some(source.source_id)
7758 })
7759 .cloned()
7760 .collect::<Vec<_>>();
7761 if crosses_fragment {
7762 *source_report
7763 .fallback_reasons
7764 .entry("CrossFragmentDependency".to_string())
7765 .or_default() += 1;
7766 continue;
7767 }
7768 if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
7769 *source_report
7770 .fallback_reasons
7771 .entry(format!("TargetPartitionReplay:{error}"))
7772 .or_default() += 1;
7773 continue;
7774 }
7775 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
7776 let mut candidate = disposition.clone();
7777 if let Err(reason) = candidate.register_partition(source, true) {
7778 *source_report
7779 .fallback_reasons
7780 .entry(reason.to_string())
7781 .or_default() += 1;
7782 continue;
7783 }
7784 disposition = candidate;
7785 }
7786 direct_families = direct_families.saturating_add(1);
7787 direct_cells = direct_cells.saturating_add(prepared.direct_cells);
7788 direct_partition_families = direct_partition_families.saturating_add(1);
7789 direct_partition_cells =
7790 direct_partition_cells.saturating_add(prepared.direct_cells);
7791 direct_fragments =
7792 direct_fragments.saturating_add(prepared.placements.len() as u64);
7793 ordered_placements.push((source.source_order, prepared.placements));
7794 }
7795 }
7796 ordered_placements.sort_by_key(|(source_order, _)| *source_order);
7797 let placements = ordered_placements
7798 .into_iter()
7799 .flat_map(|(_, placements)| placements)
7800 .collect();
7801
7802 Ok(PreparedTargetSourcePackage {
7803 sheet: sheet.to_string(),
7804 sheet_id,
7805 lease,
7806 source_report,
7807 replay_records,
7808 disposition,
7809 placements,
7810 legacy: Vec::new(),
7811 direct_families,
7812 direct_cells,
7813 direct_fragments,
7814 direct_complete_families,
7815 direct_complete_cells,
7816 direct_partition_families,
7817 direct_partition_cells,
7818 anchor_parses,
7819 anchor_asts,
7820 anchor_analyses,
7821 })
7822 }
7823
7824 fn fallback_planning_snapshot(
7825 &self,
7826 batch: &FormulaIngestBatch,
7827 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7828 let mut requests = Vec::new();
7829 for formula in &batch.formulas {
7830 let ast = self
7831 .graph
7832 .data_store()
7833 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
7834 .ok_or_else(|| {
7835 ExcelError::new(ExcelErrorKind::Value)
7836 .with_message("ordered fallback AST is unavailable")
7837 })?;
7838 Self::collect_planning_function_requests(&ast, &mut requests);
7839 }
7840 requests.sort();
7841 requests.dedup();
7842 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7843 &self.resolver,
7844 requests,
7845 )
7846 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
7847 }
7848
7849 fn prepare_legacy_batch_fallback(
7850 &mut self,
7851 batch: FormulaIngestBatch,
7852 function_provider: &dyn crate::traits::FunctionProvider,
7853 ) -> Result<
7854 (
7855 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
7856 u64,
7857 ),
7858 ExcelError,
7859 > {
7860 let formula_count = batch.formulas.len() as u64;
7861 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7862 let mut planned = Vec::with_capacity(batch.formulas.len());
7863 for record in batch.formulas {
7864 let placement = CellRef::new(
7865 sheet_id,
7866 Coord::from_excel(record.row, record.col, true, true),
7867 );
7868 let ingested = self
7869 .graph
7870 .ingest_pipeline(function_provider)
7871 .enable_function_semantics()
7872 .ingest_formula(
7873 FormulaAstInput::RawArena(record.ast_id),
7874 placement,
7875 record.formula_text,
7876 )
7877 .map_err(|error| {
7878 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
7879 })?;
7880 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7881 }
7882 let plan = self
7883 .graph
7884 .prepare_legacy_graph_plan(sheet_id, planned)
7885 .map_err(|error| {
7886 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
7887 })?;
7888 Ok((plan, formula_count))
7889 }
7890
7891 fn publish_compressed_partial_report(
7892 &mut self,
7893 report: &FormulaIngestReport,
7894 direct_report: &FormulaIngestReport,
7895 ) {
7896 if direct_report.source_family_promoted == 0
7897 && direct_report.graph_formula_cells_materialized == 0
7898 {
7899 return;
7900 }
7901 let mut published = report.clone();
7902 published.accumulate(direct_report);
7903 self.record_formula_ingest_report(published);
7904 }
7905
7906 fn finish_compressed_formula_sources(
7907 &mut self,
7908 batches: Vec<(
7909 FormulaIngestBatch,
7910 crate::engine::FormulaCompressedSourceReport,
7911 crate::engine::FormulaCompressedPreparation,
7912 )>,
7913 ) -> Result<FormulaIngestReport, ExcelError> {
7914 self.observe_function_semantic_epoch()?;
7915 if batches.iter().any(|(_, _, preparation)| {
7916 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7917 }) {
7918 return Err(ExcelError::new(ExcelErrorKind::Value)
7919 .with_message("compressed source preparation belongs to another engine"));
7920 }
7921 if batches.iter().any(|(fallback, _, preparation)| {
7922 preparation.sheet_name.as_ref() != fallback.sheet_name
7923 }) {
7924 return Err(ExcelError::new(ExcelErrorKind::Value)
7925 .with_message("compressed source preparation sheet mismatch"));
7926 }
7927 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7928 let initial_provider_revision = self.resolver.planning_semantic_revision();
7929 let mut fallback_batches = Vec::with_capacity(batches.len());
7930 let mut stale_fallback_batches = Vec::new();
7931 let mut pending_preparations = Vec::new();
7932 for (mut fallback, mut source, mut preparation) in batches {
7933 for formula in fallback.formulas.drain(..) {
7934 let source_order = formula.source_order.ok_or_else(|| {
7935 ExcelError::new(ExcelErrorKind::Value).with_message(
7936 "compressed source supplied formulas without source-order proof",
7937 )
7938 })?;
7939 let text = formula.formula_text.ok_or_else(|| {
7940 ExcelError::new(ExcelErrorKind::Value).with_message(
7941 "ordered compressed fallback formula has no exact source text",
7942 )
7943 })?;
7944 preparation
7945 .eager_replay
7946 .push(crate::engine::DeferredReplayFormula {
7947 source_order,
7948 row: formula.row,
7949 col: formula.col,
7950 text: text.to_string(),
7951 family: formula.source_family,
7952 partition_owner: formula.partition_owner,
7953 });
7954 }
7955 preparation
7956 .eager_replay
7957 .sort_by_key(|record| record.source_order);
7958 source.source_spool_replays = source
7959 .source_spool_replays
7960 .saturating_add(preparation.preparation_spool_replays);
7961 let stale_reason = preparation
7962 .function_semantics_used
7963 .then(|| {
7964 if preparation.function_provider_revision != initial_provider_revision {
7965 Some("FunctionProviderRevisionChanged")
7966 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7967 {
7968 Some("FunctionSemanticEpochChanged")
7969 } else {
7970 None
7971 }
7972 })
7973 .flatten();
7974 let stale_semantics = stale_reason.is_some();
7975 if let Some(reason) = stale_reason {
7976 for (family, _, _) in &preparation.prepared {
7977 preparation.rejected.insert(*family, reason.to_string());
7978 }
7979 }
7980 for reason in preparation.rejected.values() {
7981 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7982 }
7983 let clean_direct_cells: u64 = preparation
7984 .prepared
7985 .iter()
7986 .map(|(_, _, prepared)| prepared.member_count)
7987 .sum();
7988 let fragmented_direct_cells: u64 = preparation
7989 .fragmented
7990 .iter()
7991 .map(|fragment| fragment.source.surviving_member_count)
7992 .sum();
7993 let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
7994 source.replay_families = source.families_seen.saturating_sub(
7995 preparation
7996 .prepared
7997 .len()
7998 .saturating_add(preparation.fragmented.len()) as u64,
7999 );
8000 source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
8001 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
8002 fallback.sheet_name.clone(),
8003 source,
8004 );
8005 if stale_semantics {
8006 stale_fallback_batches.push((fallback, compressed));
8007 } else {
8008 fallback_batches.push((fallback, compressed));
8009 }
8010 pending_preparations.push((preparation, stale_semantics));
8011 }
8012 drop(initial_guard);
8013
8014 let configured_mode = self.config.formula_plane_mode;
8016 self.config.formula_plane_mode = FormulaPlaneMode::Off;
8017 let stale_result =
8018 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
8019 self.config.formula_plane_mode = configured_mode;
8020 let mut report = stale_result?;
8021 report.mode = configured_mode;
8022
8023 self.config.formula_plane_mode = FormulaPlaneMode::Off;
8024 let fallback_result =
8025 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
8026 self.config.formula_plane_mode = configured_mode;
8027 match fallback_result {
8028 Ok(fallback_report) => report.accumulate(&fallback_report),
8029 Err(error) => {
8030 self.record_formula_ingest_report(report);
8031 return Err(error);
8032 }
8033 }
8034
8035 let mut direct_report =
8036 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
8037 for (preparation, _) in &pending_preparations {
8038 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
8039 preparation.clean_rejected_anchor_counts[0]
8040 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
8041 );
8042 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
8043 preparation.clean_rejected_anchor_counts[1]
8044 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
8045 );
8046 direct_report.source_anchor_analyses =
8047 direct_report.source_anchor_analyses.saturating_add(
8048 preparation.clean_rejected_anchor_counts[2]
8049 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
8050 );
8051 let rejected_partitions: Vec<_> = preparation
8052 .rejected
8053 .keys()
8054 .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
8055 .collect();
8056 direct_report.source_partitioned_families_rejected = direct_report
8057 .source_partitioned_families_rejected
8058 .saturating_add(rejected_partitions.len() as u64);
8059 direct_report.source_partition_failures = direct_report
8060 .source_partition_failures
8061 .saturating_add(rejected_partitions.len() as u64);
8062 direct_report.source_partition_fallback_cells = direct_report
8063 .source_partition_fallback_cells
8064 .saturating_add(
8065 rejected_partitions
8066 .iter()
8067 .map(|source| source.surviving_member_count)
8068 .sum::<u64>(),
8069 );
8070 }
8071 loop {
8072 #[cfg(any(test, feature = "test-support"))]
8073 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8074 hook();
8075 }
8076 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8077 let commit_provider_revision = self.resolver.planning_semantic_revision();
8078 let mut newly_stale = Vec::new();
8079 let mut current = Vec::new();
8080 for pending in pending_preparations.drain(..) {
8081 let stale_reason = pending
8082 .0
8083 .function_semantics_used
8084 .then(|| {
8085 if pending.0.function_provider_revision != commit_provider_revision {
8086 Some("FunctionProviderRevisionChanged")
8087 } else if self
8088 .prepared_function_semantics_changed(&pending.0, &commit_guard)
8089 {
8090 Some("FunctionSemanticEpochChanged")
8091 } else {
8092 None
8093 }
8094 })
8095 .flatten();
8096 if let Some(reason) = stale_reason {
8097 newly_stale.push((pending, reason));
8098 } else {
8099 current.push(pending);
8100 }
8101 }
8102 if !newly_stale.is_empty() {
8103 drop(commit_guard);
8104 for ((mut preparation, was_initially_stale), reason) in newly_stale {
8105 for (family, _, _) in &preparation.prepared {
8106 preparation.rejected.insert(*family, reason.to_string());
8107 }
8108 let exact = match self.replay_prepared_families_exact_records(&preparation) {
8109 Ok(exact) => exact,
8110 Err(error) => {
8111 self.publish_compressed_partial_report(&report, &direct_report);
8112 return Err(error);
8113 }
8114 };
8115 preparation.eager_replay.extend(exact);
8116 preparation
8117 .eager_replay
8118 .sort_by_key(|record| record.source_order);
8119 direct_report.source_spool_replays =
8120 direct_report.source_spool_replays.saturating_add(1);
8121 if !was_initially_stale {
8122 *direct_report
8123 .fallback_reasons
8124 .entry(reason.to_string())
8125 .or_default() += preparation.prepared.len() as u64;
8126 }
8127 direct_report.source_family_fallback = direct_report
8128 .source_family_fallback
8129 .saturating_add(preparation.prepared.len() as u64);
8130 direct_report.source_family_fallback_cells =
8131 direct_report.source_family_fallback_cells.saturating_add(
8132 preparation
8133 .prepared
8134 .iter()
8135 .map(|(_, _, prepared)| prepared.member_count)
8136 .sum(),
8137 );
8138 preparation.prepared.clear();
8139 preparation.function_semantics_used = false;
8140 if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8141 current.push((preparation, false));
8142 }
8143 }
8144 pending_preparations = current;
8145 continue;
8146 }
8147 drop(commit_guard);
8148
8149 enum SourceProposal {
8152 KnownFallback {
8153 source_order: crate::engine::SourceFormulaOrder,
8154 records: Vec<crate::engine::DeferredReplayFormula>,
8155 },
8156 Clean(
8157 crate::engine::SourceFamilyId,
8158 crate::engine::SourceFormulaOrder,
8159 crate::formula_plane::placement::PreparedAnchorOncePlacement,
8160 ),
8161 Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8162 }
8163
8164 impl SourceProposal {
8165 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8166 match self {
8167 Self::KnownFallback { source_order, .. } => *source_order,
8168 Self::Clean(_, source_order, _) => *source_order,
8169 Self::Fragment(fragment) => fragment.source.source_order,
8170 }
8171 }
8172 }
8173
8174 for (mut preparation, _) in current {
8175 let mut proposals = Vec::with_capacity(
8176 preparation
8177 .prepared
8178 .len()
8179 .saturating_add(preparation.fragmented.len())
8180 .saturating_add(preparation.eager_replay.len()),
8181 );
8182 proposals.extend(preparation.prepared.drain(..).map(
8183 |(source_id, source_order, prepared)| {
8184 SourceProposal::Clean(source_id, source_order, prepared)
8185 },
8186 ));
8187
8188 let accepted_fragments: BTreeMap<_, _> = preparation
8189 .fragmented
8190 .iter()
8191 .map(|fragment| (fragment.source.source_id, &fragment.source))
8192 .collect();
8193 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8194 for record in preparation.eager_replay.drain(..).filter(|record| {
8195 !record.partition_owner.is_some_and(|owner| {
8196 accepted_fragments.get(&owner).is_some_and(|source| {
8197 Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8198 })
8199 })
8200 }) {
8201 if let Some(owner) = record.partition_owner.or(record.family) {
8202 family_fallbacks.entry(owner).or_default().push(record);
8203 } else {
8204 proposals.push(SourceProposal::KnownFallback {
8205 source_order: record.source_order,
8206 records: vec![record],
8207 });
8208 }
8209 }
8210 for (_, mut records) in family_fallbacks {
8211 records.sort_by_key(|record| record.source_order);
8212 if records
8213 .windows(2)
8214 .any(|window| window[0].source_order == window[1].source_order)
8215 {
8216 self.publish_compressed_partial_report(&report, &direct_report);
8217 return Err(ExcelError::new(ExcelErrorKind::Value)
8218 .with_message("duplicate exact-replay source-order proof"));
8219 }
8220 let Some(source_order) = records.first().map(|record| record.source_order)
8221 else {
8222 self.publish_compressed_partial_report(&report, &direct_report);
8223 return Err(ExcelError::new(ExcelErrorKind::Value)
8224 .with_message("empty exact-replay fallback family"));
8225 };
8226 proposals.push(SourceProposal::KnownFallback {
8227 source_order,
8228 records,
8229 });
8230 }
8231 proposals.extend(
8232 preparation
8233 .fragmented
8234 .drain(..)
8235 .map(SourceProposal::Fragment),
8236 );
8237 proposals.sort_by_key(SourceProposal::source_order);
8238 if proposals
8239 .windows(2)
8240 .any(|window| window[0].source_order() == window[1].source_order())
8241 {
8242 self.publish_compressed_partial_report(&report, &direct_report);
8243 return Err(ExcelError::new(ExcelErrorKind::Value)
8244 .with_message("ambiguous compressed source-order proof"));
8245 }
8246
8247 for proposal in proposals {
8248 match proposal {
8249 SourceProposal::KnownFallback { records, .. } => {
8250 let batch = match self.formula_batch_from_exact_replay(
8251 preparation.sheet_name.as_ref(),
8252 records,
8253 ) {
8254 Ok(batch) => batch,
8255 Err(error) => {
8256 self.publish_compressed_partial_report(&report, &direct_report);
8257 return Err(error);
8258 }
8259 };
8260 if batch.is_empty() {
8261 continue;
8262 }
8263 let snapshot = match self.fallback_planning_snapshot(&batch) {
8264 Ok(snapshot) => snapshot,
8265 Err(error) => {
8266 self.publish_compressed_partial_report(&report, &direct_report);
8267 return Err(error);
8268 }
8269 };
8270 let commit_guard =
8271 crate::function_registry::semantic_epoch_read_guard();
8272 let provider_revision_initial =
8273 self.resolver.planning_semantic_revision();
8274 if (commit_guard.epoch() != snapshot.epoch()
8275 && snapshot.semantic_changes_affect_requests_since_guarded(
8276 &commit_guard,
8277 snapshot.epoch(),
8278 ))
8279 || snapshot.provider_revision().is_some_and(|revision| {
8280 Some(revision) != provider_revision_initial
8281 })
8282 {
8283 drop(commit_guard);
8284 self.publish_compressed_partial_report(&report, &direct_report);
8285 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8286 "ordered fallback planning snapshot became stale",
8287 ));
8288 }
8289 let (plan, formula_count) = match self
8290 .prepare_legacy_batch_fallback(batch, &snapshot)
8291 {
8292 Ok(plan) => plan,
8293 Err(error) => {
8294 drop(commit_guard);
8295 self.publish_compressed_partial_report(&report, &direct_report);
8296 return Err(error);
8297 }
8298 };
8299 let provider_revision_after =
8300 self.resolver.planning_semantic_revision();
8301 if provider_revision_after != provider_revision_initial {
8302 drop(commit_guard);
8303 self.publish_compressed_partial_report(&report, &direct_report);
8304 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8305 "function provider changed while preparing ordered fallback",
8306 ));
8307 }
8308 let graph_vertices = plan.new_vertex_count();
8309 let Some(graph_edges) = plan.planned_edge_count() else {
8310 drop(commit_guard);
8311 self.publish_compressed_partial_report(&report, &direct_report);
8312 return Err(ExcelError::new(ExcelErrorKind::Value)
8313 .with_message("prepared ordered fallback size overflow"));
8314 };
8315 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
8316 {
8317 drop(commit_guard);
8318 self.publish_compressed_partial_report(&report, &direct_report);
8319 return Err(error);
8320 }
8321 if let Err(error) =
8322 self.graph.validate_prepared_legacy_graph_plan(&plan)
8323 {
8324 drop(commit_guard);
8325 self.publish_compressed_partial_report(&report, &direct_report);
8326 return Err(ExcelError::new(ExcelErrorKind::Value)
8327 .with_message(error.to_string()));
8328 }
8329 #[cfg(test)]
8330 if let Some(hook) = self
8331 .before_legacy_fallback_final_provider_sample_hook
8332 .take()
8333 {
8334 hook();
8335 }
8336 let provider_revision_final =
8337 self.resolver.planning_semantic_revision();
8338 if provider_revision_final != provider_revision_initial {
8339 drop(commit_guard);
8340 self.publish_compressed_partial_report(&report, &direct_report);
8341 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8342 "function provider changed after ordered fallback validation",
8343 ));
8344 }
8345 let graph_formulas =
8346 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8347 direct_report.formula_cells_seen = direct_report
8348 .formula_cells_seen
8349 .saturating_add(formula_count);
8350 direct_report.graph_formula_cells_materialized = direct_report
8351 .graph_formula_cells_materialized
8352 .saturating_add(graph_formulas as u64);
8353 direct_report.graph_vertices_created = direct_report
8354 .graph_vertices_created
8355 .saturating_add(graph_vertices as u64);
8356 direct_report.graph_edges_created = direct_report
8357 .graph_edges_created
8358 .saturating_add(graph_edges as u64);
8359 }
8360 SourceProposal::Clean(source_id, _, prepared) => {
8361 let commit_guard =
8362 crate::function_registry::semantic_epoch_read_guard();
8363 let commit_provider_revision =
8364 self.resolver.planning_semantic_revision();
8365 let semantic_changed = preparation.function_semantics_used
8366 && self.prepared_placement_function_semantics_changed(
8367 preparation.function_semantic_epoch,
8368 &prepared,
8369 &commit_guard,
8370 );
8371 let cells = prepared.member_count;
8372 let stats = self.graph.baseline_stats();
8373 self.preflight_graph_admission(
8374 crate::engine::resource_ledger::GraphAdmission {
8375 final_vertices: stats.graph_vertex_count,
8376 final_edges: stats.graph_edge_count,
8377 materialization_cells: 0,
8378 added_vertices: 0,
8379 added_edges: 0,
8380 },
8381 )?;
8382 let append = self
8383 .graph
8384 .formula_authority()
8385 .prepare_formula_plane_append(
8386 vec![prepared],
8387 self.graph.data_store(),
8388 self.graph.sheet_reg(),
8389 )
8390 .map_err(|error| error.to_string())
8391 .and_then(|append| {
8392 if preparation.function_semantics_used
8393 && preparation.function_provider_revision
8394 != commit_provider_revision
8395 {
8396 return Err("FunctionProviderRevisionChanged".to_string());
8397 }
8398 if preparation.function_semantics_used
8399 && self.resolver.planning_semantic_revision()
8400 != commit_provider_revision
8401 {
8402 return Err("FunctionProviderRevisionChanged".to_string());
8403 }
8404 if semantic_changed {
8405 return Err("FunctionSemanticEpochChanged".to_string());
8406 }
8407 self.graph
8408 .formula_authority()
8409 .validate_prepared_formula_plane_append(
8410 &append,
8411 self.graph.data_store(),
8412 self.graph.sheet_reg(),
8413 )
8414 .map_err(|error| error.to_string())?;
8415 Ok(append)
8416 });
8417 match append {
8418 Ok(append) => {
8419 let placement_report = self
8420 .graph
8421 .formula_authority_mut()
8422 .apply_prevalidated_formula_plane_append(append);
8423 self.graph.mark_formula_spans_dirty(
8424 placement_report.spans.iter().copied(),
8425 WholeSpanDirtyReason::NewSpan,
8426 );
8427 direct_report.formula_cells_seen =
8428 direct_report.formula_cells_seen.saturating_add(cells);
8429 direct_report.shadow_candidate_cells =
8430 direct_report.shadow_candidate_cells.saturating_add(cells);
8431 direct_report.shadow_accepted_span_cells = direct_report
8432 .shadow_accepted_span_cells
8433 .saturating_add(cells);
8434 direct_report.shadow_templates_interned =
8435 direct_report.shadow_templates_interned.saturating_add(
8436 placement_report.work.templates_to_append as u64,
8437 );
8438 direct_report.shadow_spans_created = direct_report
8439 .shadow_spans_created
8440 .saturating_add(placement_report.spans.len() as u64);
8441 direct_report.graph_formula_vertices_avoided_shadow =
8442 direct_report
8443 .graph_formula_vertices_avoided_shadow
8444 .saturating_add(cells);
8445 direct_report.ast_roots_avoided_shadow = direct_report
8446 .ast_roots_avoided_shadow
8447 .saturating_add(cells.saturating_sub(1));
8448 direct_report.edge_rows_avoided_shadow = direct_report
8449 .edge_rows_avoided_shadow
8450 .saturating_add(cells);
8451 direct_report.source_family_promoted =
8452 direct_report.source_family_promoted.saturating_add(1);
8453 direct_report.source_family_promoted_cells = direct_report
8454 .source_family_promoted_cells
8455 .saturating_add(cells);
8456 direct_report.source_anchor_parses =
8457 direct_report.source_anchor_parses.saturating_add(1);
8458 direct_report.source_anchor_asts =
8459 direct_report.source_anchor_asts.saturating_add(1);
8460 direct_report.source_anchor_analyses =
8461 direct_report.source_anchor_analyses.saturating_add(1);
8462 let descendants = cells.saturating_sub(1);
8463 direct_report.source_descendant_strings_avoided = direct_report
8464 .source_descendant_strings_avoided
8465 .saturating_add(descendants);
8466 direct_report.source_descendant_events_avoided = direct_report
8467 .source_descendant_events_avoided
8468 .saturating_add(descendants);
8469 direct_report.source_descendant_analyses_avoided =
8470 direct_report
8471 .source_descendant_analyses_avoided
8472 .saturating_add(descendants);
8473 direct_report.source_compressed_families_prepared =
8474 direct_report
8475 .source_compressed_families_prepared
8476 .saturating_add(1);
8477 direct_report.source_compressed_cells_prepared = direct_report
8478 .source_compressed_cells_prepared
8479 .saturating_add(cells);
8480 }
8481 Err(error) => {
8482 drop(commit_guard);
8483 let reason = format!("CleanFormulaPlaneAppend:{error}");
8484 let batch = match self.replay_one_prepared_family_exact(
8485 &preparation,
8486 source_id,
8487 cells,
8488 ) {
8489 Ok(batch) => batch,
8490 Err(error) => {
8491 self.publish_compressed_partial_report(
8492 &report,
8493 &direct_report,
8494 );
8495 return Err(error);
8496 }
8497 };
8498 let snapshot = match self.fallback_planning_snapshot(&batch) {
8499 Ok(snapshot) => snapshot,
8500 Err(error) => {
8501 self.publish_compressed_partial_report(
8502 &report,
8503 &direct_report,
8504 );
8505 return Err(error);
8506 }
8507 };
8508 let commit_guard =
8509 crate::function_registry::semantic_epoch_read_guard();
8510 let provider_revision =
8511 self.resolver.planning_semantic_revision();
8512 if (commit_guard.epoch() != snapshot.epoch()
8513 && snapshot.semantic_changes_affect_requests_since_guarded(
8514 &commit_guard,
8515 snapshot.epoch(),
8516 ))
8517 || snapshot.provider_revision().is_some_and(|revision| {
8518 Some(revision) != provider_revision
8519 })
8520 {
8521 drop(commit_guard);
8522 self.publish_compressed_partial_report(
8523 &report,
8524 &direct_report,
8525 );
8526 return Err(ExcelError::new(ExcelErrorKind::Value)
8527 .with_message(
8528 "clean-family fallback planning snapshot became stale",
8529 ));
8530 }
8531 let (plan, formula_count) = match self
8532 .prepare_legacy_batch_fallback(batch, &snapshot)
8533 {
8534 Ok(plan) => plan,
8535 Err(error) => {
8536 drop(commit_guard);
8537 self.publish_compressed_partial_report(
8538 &report,
8539 &direct_report,
8540 );
8541 return Err(error);
8542 }
8543 };
8544 let provider_revision_after =
8545 self.resolver.planning_semantic_revision();
8546 if provider_revision_after != provider_revision {
8547 drop(commit_guard);
8548 self.publish_compressed_partial_report(
8549 &report,
8550 &direct_report,
8551 );
8552 return Err(ExcelError::new(ExcelErrorKind::Value)
8553 .with_message(
8554 "function provider changed while preparing clean-family fallback",
8555 ));
8556 }
8557 let graph_vertices = plan.new_vertex_count();
8558 let Some(graph_edges) = plan.planned_edge_count() else {
8559 drop(commit_guard);
8560 self.publish_compressed_partial_report(
8561 &report,
8562 &direct_report,
8563 );
8564 return Err(ExcelError::new(ExcelErrorKind::Value)
8565 .with_message(
8566 "prepared clean-family fallback size overflow",
8567 ));
8568 };
8569 if let Err(error) =
8570 self.prepared_legacy_admission(&plan, formula_count)
8571 {
8572 drop(commit_guard);
8573 self.publish_compressed_partial_report(
8574 &report,
8575 &direct_report,
8576 );
8577 return Err(error);
8578 }
8579 if let Err(error) =
8580 self.graph.validate_prepared_legacy_graph_plan(&plan)
8581 {
8582 drop(commit_guard);
8583 self.publish_compressed_partial_report(
8584 &report,
8585 &direct_report,
8586 );
8587 return Err(ExcelError::new(ExcelErrorKind::Value)
8588 .with_message(error.to_string()));
8589 }
8590 #[cfg(test)]
8591 if let Some(hook) = self
8592 .before_legacy_fallback_final_provider_sample_hook
8593 .take()
8594 {
8595 hook();
8596 }
8597 let provider_revision_final =
8598 self.resolver.planning_semantic_revision();
8599 if provider_revision_final != provider_revision {
8600 drop(commit_guard);
8601 self.publish_compressed_partial_report(
8602 &report,
8603 &direct_report,
8604 );
8605 return Err(ExcelError::new(ExcelErrorKind::Value)
8606 .with_message(
8607 "function provider changed after clean-family fallback validation",
8608 ));
8609 }
8610 let graph_formulas =
8611 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8612 direct_report.formula_cells_seen = direct_report
8613 .formula_cells_seen
8614 .saturating_add(formula_count);
8615 direct_report.graph_formula_cells_materialized = direct_report
8616 .graph_formula_cells_materialized
8617 .saturating_add(graph_formulas as u64);
8618 direct_report.graph_vertices_created = direct_report
8619 .graph_vertices_created
8620 .saturating_add(graph_vertices as u64);
8621 direct_report.graph_edges_created = direct_report
8622 .graph_edges_created
8623 .saturating_add(graph_edges as u64);
8624 direct_report.source_spool_replays =
8625 direct_report.source_spool_replays.saturating_add(1);
8626 direct_report.source_family_fallback =
8627 direct_report.source_family_fallback.saturating_add(1);
8628 direct_report.source_family_fallback_cells = direct_report
8629 .source_family_fallback_cells
8630 .saturating_add(cells);
8631 direct_report.source_anchor_parses =
8632 direct_report.source_anchor_parses.saturating_add(1);
8633 direct_report.source_anchor_asts =
8634 direct_report.source_anchor_asts.saturating_add(1);
8635 direct_report.source_anchor_analyses =
8636 direct_report.source_anchor_analyses.saturating_add(1);
8637 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8638 }
8639 }
8640 }
8641 SourceProposal::Fragment(fragment) => {
8642 let commit_guard =
8643 crate::function_registry::semantic_epoch_read_guard();
8644 let commit_epoch = commit_guard.epoch();
8645 let commit_provider_revision =
8646 self.resolver.planning_semantic_revision();
8647 let source = fragment.source;
8648 let source_id = source.source_id;
8649 #[cfg(test)]
8650 let commit_fault = self
8651 .fragmented_commit_fault_for_test
8652 .take()
8653 .unwrap_or(
8654 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
8655 );
8656 #[cfg(not(test))]
8657 let commit_fault =
8658 crate::engine::fragmented_transaction::FragmentedCommitFault::None;
8659 let outcome = self
8660 .prepare_fragmented_source_transaction(
8661 &source,
8662 &preparation.replay_disposition,
8663 fragment.prepared,
8664 fragment.legacy,
8665 )
8666 .map(|transaction| {
8667 let resolver = &self.resolver;
8668 self.graph.commit_fragmented_source_transaction(
8669 transaction,
8670 &self.source_formula_token,
8671 &preparation.replay_disposition,
8672 commit_epoch,
8673 commit_provider_revision,
8674 || resolver.planning_semantic_revision(),
8675 commit_fault,
8676 )
8677 });
8678 let fallback_reason = match outcome {
8679 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
8680 let mut delta = success.report_delta;
8681 delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
8682 delta.source_family_promoted = 1;
8683 delta.source_family_promoted_cells =
8684 source.surviving_member_count;
8685 delta.source_partitioned_families_seen = 0;
8686 delta.source_partition_holes = 0;
8687 delta.source_partition_ordinary_exceptions = 0;
8688 delta.source_partition_surviving_cells = 0;
8689 delta.shadow_templates_interned =
8690 success.plane.work.templates_to_append as u64;
8691 delta.shadow_spans_created = success.plane.spans.len() as u64;
8692 let descendants =
8693 source.surviving_member_count.saturating_sub(1);
8694 delta.source_descendant_strings_avoided = descendants;
8695 delta.source_descendant_events_avoided = descendants;
8696 delta.source_descendant_analyses_avoided = descendants;
8697 direct_report.accumulate(&delta);
8698 None
8699 }
8700 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
8701 Some(format!("{reason:?}"))
8702 }
8703 Err(error) => Some(format!("{error:?}")),
8704 };
8705 if let Some(reason) = fallback_reason {
8706 drop(commit_guard);
8707 let mut replay_disposition = preparation.replay_disposition.clone();
8708 replay_disposition.force_family_legacy(source_id);
8709 let replayed = (|| {
8710 let records = fragment
8711 .replay
8712 .lock()
8713 .map_err(|_| {
8714 ExcelError::new(ExcelErrorKind::Value).with_message(
8715 "compressed formula exact replay lock poisoned",
8716 )
8717 })?
8718 .replay_partitioned(
8719 &replay_disposition,
8720 std::slice::from_ref(&source),
8721 )
8722 .map_err(|message| {
8723 ExcelError::new(ExcelErrorKind::Value)
8724 .with_message(message)
8725 })?
8726 .into_iter()
8727 .filter(|record| {
8728 record.family == Some(source_id)
8729 || (record.family.is_none()
8730 && record.partition_owner == Some(source_id))
8731 })
8732 .collect::<Vec<_>>();
8733 Self::validate_whole_partition_replay(&source, &records)?;
8734 Ok::<_, ExcelError>(records)
8735 })();
8736 let replayed = match replayed {
8737 Ok(replayed) => replayed,
8738 Err(error) => {
8739 self.publish_compressed_partial_report(
8740 &report,
8741 &direct_report,
8742 );
8743 return Err(error);
8744 }
8745 };
8746 let batch = match self.formula_batch_from_exact_replay(
8747 preparation.sheet_name.as_ref(),
8748 replayed,
8749 ) {
8750 Ok(batch) => batch,
8751 Err(error) => {
8752 self.publish_compressed_partial_report(
8753 &report,
8754 &direct_report,
8755 );
8756 return Err(error);
8757 }
8758 };
8759 let snapshot = match self.fallback_planning_snapshot(&batch) {
8760 Ok(snapshot) => snapshot,
8761 Err(error) => {
8762 self.publish_compressed_partial_report(
8763 &report,
8764 &direct_report,
8765 );
8766 return Err(error);
8767 }
8768 };
8769 let commit_guard =
8770 crate::function_registry::semantic_epoch_read_guard();
8771 let provider_revision = self.resolver.planning_semantic_revision();
8772 if (commit_guard.epoch() != snapshot.epoch()
8773 && snapshot.semantic_changes_affect_requests_since_guarded(
8774 &commit_guard,
8775 snapshot.epoch(),
8776 ))
8777 || snapshot
8778 .provider_revision()
8779 .is_some_and(|revision| Some(revision) != provider_revision)
8780 {
8781 self.publish_compressed_partial_report(&report, &direct_report);
8782 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8783 format!(
8784 "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
8785 snapshot.epoch(),
8786 commit_guard.epoch(),
8787 snapshot.provider_revision(),
8788 provider_revision,
8789 ),
8790 ));
8791 }
8792 let (plan, formula_count) =
8793 match self.prepare_legacy_batch_fallback(batch, &snapshot) {
8794 Ok(plan) => plan,
8795 Err(error) => {
8796 drop(commit_guard);
8797 self.publish_compressed_partial_report(
8798 &report,
8799 &direct_report,
8800 );
8801 return Err(error);
8802 }
8803 };
8804 let provider_revision_after =
8805 self.resolver.planning_semantic_revision();
8806 if provider_revision_after != provider_revision {
8807 drop(commit_guard);
8808 self.publish_compressed_partial_report(&report, &direct_report);
8809 return Err(ExcelError::new(ExcelErrorKind::Value)
8810 .with_message(
8811 "function provider changed while preparing whole-family fallback",
8812 ));
8813 }
8814 let graph_vertices = plan.new_vertex_count();
8815 let Some(graph_edges) = plan.planned_edge_count() else {
8816 drop(commit_guard);
8817 self.publish_compressed_partial_report(&report, &direct_report);
8818 return Err(ExcelError::new(ExcelErrorKind::Value)
8819 .with_message(
8820 "prepared whole-family fallback size overflow",
8821 ));
8822 };
8823 if let Err(error) =
8824 self.prepared_legacy_admission(&plan, formula_count)
8825 {
8826 drop(commit_guard);
8827 self.publish_compressed_partial_report(&report, &direct_report);
8828 return Err(error);
8829 }
8830 if let Err(error) =
8831 self.graph.validate_prepared_legacy_graph_plan(&plan)
8832 {
8833 drop(commit_guard);
8834 self.publish_compressed_partial_report(&report, &direct_report);
8835 return Err(ExcelError::new(ExcelErrorKind::Value)
8836 .with_message(error.to_string()));
8837 }
8838 #[cfg(test)]
8839 if let Some(hook) = self
8840 .before_legacy_fallback_final_provider_sample_hook
8841 .take()
8842 {
8843 hook();
8844 }
8845 let provider_revision_final =
8846 self.resolver.planning_semantic_revision();
8847 if provider_revision_final != provider_revision {
8848 drop(commit_guard);
8849 self.publish_compressed_partial_report(&report, &direct_report);
8850 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8851 "function provider changed after whole-family fallback validation",
8852 ));
8853 }
8854 let graph_formulas =
8855 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8856 direct_report.formula_cells_seen = direct_report
8857 .formula_cells_seen
8858 .saturating_add(formula_count);
8859 direct_report.graph_formula_cells_materialized = direct_report
8860 .graph_formula_cells_materialized
8861 .saturating_add(graph_formulas as u64);
8862 direct_report.graph_vertices_created = direct_report
8863 .graph_vertices_created
8864 .saturating_add(graph_vertices as u64);
8865 direct_report.graph_edges_created = direct_report
8866 .graph_edges_created
8867 .saturating_add(graph_edges as u64);
8868 direct_report.source_spool_replays =
8869 direct_report.source_spool_replays.saturating_add(1);
8870 direct_report.source_family_fallback =
8871 direct_report.source_family_fallback.saturating_add(1);
8872 direct_report.source_family_fallback_cells = direct_report
8873 .source_family_fallback_cells
8874 .saturating_add(source.surviving_member_count);
8875 direct_report.source_partitioned_families_rejected = direct_report
8876 .source_partitioned_families_rejected
8877 .saturating_add(1);
8878 direct_report.source_partition_failures =
8879 direct_report.source_partition_failures.saturating_add(1);
8880 direct_report.source_partition_fallback_cells = direct_report
8881 .source_partition_fallback_cells
8882 .saturating_add(source.surviving_member_count);
8883 direct_report.source_anchor_parses =
8884 direct_report.source_anchor_parses.saturating_add(1);
8885 direct_report.source_anchor_asts =
8886 direct_report.source_anchor_asts.saturating_add(1);
8887 direct_report.source_anchor_analyses =
8888 direct_report.source_anchor_analyses.saturating_add(1);
8889 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8890 }
8891 }
8892 }
8893 #[cfg(test)]
8894 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
8895 hook();
8896 }
8897 }
8898 }
8899 break;
8900 }
8901 report.accumulate(&direct_report);
8902 self.record_formula_ingest_report(report.clone());
8903 Ok(report)
8904 }
8905 pub(crate) fn ingest_compressed_formula_source_batches(
8907 &mut self,
8908 batches: Vec<(
8909 FormulaIngestBatch,
8910 crate::engine::FormulaCompressedSourceBatch,
8911 )>,
8912 ) -> Result<FormulaIngestReport, ExcelError> {
8913 self.ingest_compressed_formula_source_batches_inner(batches, true)
8914 }
8915
8916 fn ingest_compressed_formula_source_batches_inner(
8917 &mut self,
8918 batches: Vec<(
8919 FormulaIngestBatch,
8920 crate::engine::FormulaCompressedSourceBatch,
8921 )>,
8922 publish_report: bool,
8923 ) -> Result<FormulaIngestReport, ExcelError> {
8924 let mut source_counts = [0_u64; 11];
8925 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
8926 let mut formula_batches = Vec::with_capacity(batches.len());
8927 let mut compressed_families = Vec::new();
8928 let mut partitioned_families = Vec::new();
8929 for (batch, compressed_batch) in batches {
8930 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
8931 if sheet_name.as_ref() != batch.sheet_name {
8932 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8933 "compressed formula source sheet does not match its replay batch",
8934 ));
8935 }
8936 compressed_families.push((batch.sheet_name.clone(), families));
8937 partitioned_families.push((batch.sheet_name.clone(), partitions));
8938 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
8939 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
8940 source_counts[2] =
8941 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
8942 source_counts[3] =
8943 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
8944 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
8945 source_counts[5] =
8946 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
8947 source_counts[6] =
8948 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
8949 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
8950 source_counts[8] =
8951 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
8952 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
8953 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
8954 source_report.families_seen = source_report
8955 .families_seen
8956 .saturating_add(compressed.families_seen);
8957 source_report.family_cells_seen = source_report
8958 .family_cells_seen
8959 .saturating_add(compressed.family_cells_seen);
8960 source_report.source_clean_families = source_report
8961 .source_clean_families
8962 .saturating_add(compressed.source_clean_families);
8963 source_report.source_clean_cells = source_report
8964 .source_clean_cells
8965 .saturating_add(compressed.source_clean_cells);
8966 source_report.source_fragmentable_families = source_report
8967 .source_fragmentable_families
8968 .saturating_add(compressed.source_fragmentable_families);
8969 source_report.source_fragmentable_cells = source_report
8970 .source_fragmentable_cells
8971 .saturating_add(compressed.source_fragmentable_cells);
8972 source_report.source_fragment_count = source_report
8973 .source_fragment_count
8974 .saturating_add(compressed.source_fragment_count);
8975 source_report.source_isolated_fallback_cells = source_report
8976 .source_isolated_fallback_cells
8977 .saturating_add(compressed.source_isolated_fallback_cells);
8978 source_report.source_hole_exclusions = source_report
8979 .source_hole_exclusions
8980 .saturating_add(compressed.source_hole_exclusions);
8981 source_report.source_ordinary_exclusions = source_report
8982 .source_ordinary_exclusions
8983 .saturating_add(compressed.source_ordinary_exclusions);
8984 source_report.source_partition_failures = source_report
8985 .source_partition_failures
8986 .saturating_add(compressed.source_partition_failures);
8987 source_report.replay_families = source_report
8988 .replay_families
8989 .saturating_add(compressed.replay_families);
8990 source_report.replay_cells = source_report
8991 .replay_cells
8992 .saturating_add(compressed.replay_cells);
8993 source_report.forward_descendants = source_report
8994 .forward_descendants
8995 .saturating_add(compressed.forward_descendants);
8996 source_report.evidence_limit_fallbacks = source_report
8997 .evidence_limit_fallbacks
8998 .saturating_add(compressed.evidence_limit_fallbacks);
8999 source_report.evidence_peak_bytes = source_report
9000 .evidence_peak_bytes
9001 .max(compressed.evidence_peak_bytes);
9002 for (reason, count) in compressed.fallback_reasons {
9003 *source_report.fallback_reasons.entry(reason).or_default() += count;
9004 }
9005 formula_batches.push(batch);
9006 }
9007 self.ingest_formula_batches_inner(
9008 formula_batches,
9009 source_counts,
9010 Some(source_report),
9011 compressed_families,
9012 partitioned_families,
9013 publish_report,
9014 )
9015 }
9016
9017 pub fn ingest_formula_batches(
9018 &mut self,
9019 batches: Vec<FormulaIngestBatch>,
9020 ) -> Result<FormulaIngestReport, ExcelError> {
9021 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
9022 let report = self.ingest_formula_batches_inner(
9023 batches,
9024 [0; 11],
9025 None,
9026 Vec::new(),
9027 Vec::new(),
9028 true,
9029 )?;
9030 if has_formulas {
9031 self.mark_topology_edited();
9032 }
9033 Ok(report)
9034 }
9035
9036 fn ingest_formula_batches_unpublished(
9037 &mut self,
9038 batches: Vec<FormulaIngestBatch>,
9039 ) -> Result<FormulaIngestReport, ExcelError> {
9040 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
9041 }
9042
9043 fn ingest_formula_batches_inner(
9044 &mut self,
9045 batches: Vec<FormulaIngestBatch>,
9046 source_counts: [u64; 11],
9047 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
9048 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
9049 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
9050 publish_report: bool,
9051 ) -> Result<FormulaIngestReport, ExcelError> {
9052 self.observe_function_semantic_epoch()?;
9053 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
9054 #[cfg(feature = "benchmark_internal")]
9055 let benchmark_forced_replay =
9056 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9057 #[cfg(not(feature = "benchmark_internal"))]
9058 let benchmark_forced_replay = false;
9059 let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9060 && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9061 {
9062 let mut report =
9063 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9064 report
9065 .fallback_reasons
9066 .insert("ForcedReplay".to_string(), formula_cells_seen);
9067 (report, batches, BTreeMap::new())
9068 } else {
9069 match self.config.formula_plane_mode {
9070 FormulaPlaneMode::Off => (
9071 FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9072 batches,
9073 BTreeMap::new(),
9074 ),
9075 FormulaPlaneMode::Shadow => (
9076 if partitioned_families
9077 .iter()
9078 .all(|(_, families)| families.is_empty())
9079 {
9080 if compressed_families.is_empty() {
9081 self.analyze_formula_plane_shadow_candidates(&batches)
9082 } else {
9083 self.analyze_compressed_formula_plane_shadow(&compressed_families)
9084 }
9085 } else {
9086 let mut partition_report =
9087 self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9088 if compressed_families
9089 .iter()
9090 .any(|(_, families)| !families.is_empty())
9091 {
9092 partition_report.accumulate(
9093 &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9094 );
9095 }
9096 partition_report
9097 },
9098 batches,
9099 BTreeMap::new(),
9100 ),
9101 FormulaPlaneMode::AuthoritativeExperimental => {
9102 self.analyze_formula_plane_authoritative_ingest(&batches)
9103 }
9104 }
9105 };
9106 report.formula_cells_seen = formula_cells_seen;
9107 report.source_formula_events = source_counts[0];
9108 report.source_ordinary_events = source_counts[1];
9109 report.source_shared_anchor_events = source_counts[2];
9110 report.source_shared_descendant_events = source_counts[3];
9111 report.source_unknown_events = source_counts[4];
9112 report.source_formula_records_spooled = source_counts[5];
9113 report.source_spool_encoded_bytes = source_counts[6];
9114 report.source_spool_peak_memory_bytes = source_counts[7];
9115 report.source_spool_spilled_bytes = source_counts[8];
9116 report.source_spool_spill_files = source_counts[9];
9117 report.source_spool_replays = source_counts[10];
9118 if let Some(source) = source_report {
9119 report.source_families_seen = source.families_seen;
9120 report.source_family_cells_seen = source.family_cells_seen;
9121 report.source_family_shadow_eligible = source.source_clean_families;
9122 report.source_family_shadow_eligible_cells = source.source_clean_cells;
9123 report.source_partitioned_families_seen = source.source_fragmentable_families;
9124 report.source_partition_holes = source.source_hole_exclusions;
9125 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9126 report.source_partition_failures = source.source_partition_failures;
9127 report.source_partition_surviving_cells = source.source_fragmentable_cells;
9128 report.source_family_fallback = report
9129 .source_family_fallback
9130 .saturating_add(source.replay_families);
9131 report.source_family_fallback_cells = report
9132 .source_family_fallback_cells
9133 .saturating_add(source.replay_cells);
9134 report.source_forward_descendants = source.forward_descendants;
9135 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9136 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9137 for (reason, count) in source.fallback_reasons {
9138 let total = report.fallback_reasons.entry(reason).or_default();
9139 *total = total.saturating_add(count);
9140 }
9141 }
9142
9143 if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9144 || !planned_materialize.is_empty()
9145 {
9146 let mut prepared_by_sheet = planned_materialize;
9147 for batch in materialize_batches {
9148 if batch.is_empty() {
9149 continue;
9150 }
9151 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9152 ExcelError::new(ExcelErrorKind::Ref)
9153 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9154 })?;
9155 let mut pipeline = self.ingest_pipeline();
9156 let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9157 let placement = CellRef::new(
9158 sheet_id,
9159 Coord::from_excel(record.row, record.col, true, true),
9160 );
9161 (
9162 FormulaAstInput::RawArena(record.ast_id),
9163 placement,
9164 record.formula_text,
9165 )
9166 }))?;
9167 prepared_by_sheet
9168 .entry(batch.sheet_name)
9169 .or_default()
9170 .extend(ingested.into_iter().map(|formula| {
9171 (
9172 formula.placement.coord.row() + 1,
9173 formula.placement.coord.col() + 1,
9174 formula.ast_id,
9175 formula.dep_plan,
9176 )
9177 }));
9178 }
9179 let admission_preflighted = self.graph_admission_enabled();
9180 if admission_preflighted {
9181 let mut preview = Vec::new();
9182 for (sheet_name, formulas) in &prepared_by_sheet {
9183 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9184 ExcelError::new(ExcelErrorKind::Ref)
9185 .with_message(format!("unknown ingest sheet: {sheet_name}"))
9186 })?;
9187 preview.extend(
9188 formulas
9189 .iter()
9190 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9191 );
9192 }
9193 let admission = self.graph.preview_formula_mutations(&preview)?;
9194 self.preflight_graph_admission(admission)?;
9195 }
9196
9197 let mut builder = self.begin_bulk_ingest();
9198 if admission_preflighted {
9199 builder.mark_admission_preflighted();
9200 }
9201 for (sheet_name, formulas) in prepared_by_sheet {
9202 if formulas.is_empty() {
9203 continue;
9204 }
9205 let sheet_id = builder.add_sheet(&sheet_name);
9206 builder.add_formula_plans(sheet_id, formulas);
9207 }
9208 let summary = builder.finish()?;
9209 report.graph_formula_cells_materialized = summary.formulas as u64;
9210 report.graph_vertices_created = summary.vertices as u64;
9211 report.graph_edges_created = summary.edges as u64;
9212 }
9213
9214 if publish_report {
9215 self.record_formula_ingest_report(report.clone());
9216 }
9217 Ok(report)
9218 }
9219
9220 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9221 let mut unique = Vec::new();
9222 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9223 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9224 prior.sheet == diagnostic.sheet
9225 && prior.row == diagnostic.row
9226 && prior.col == diagnostic.col
9227 && prior.formula == diagnostic.formula
9228 && prior.policy == diagnostic.policy
9229 });
9230 if !duplicate {
9231 unique.push(diagnostic);
9232 }
9233 }
9234 self.formula_parse_diagnostics.extend(unique);
9235 }
9236
9237 pub fn handle_formula_parse_error(
9238 &mut self,
9239 sheet: &str,
9240 row: u32,
9241 col: u32,
9242 formula: &str,
9243 message: String,
9244 ) -> Result<Option<ASTNode>, ExcelError> {
9245 let policy = self.config.formula_parse_policy;
9246
9247 if policy == FormulaParsePolicy::Strict {
9248 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9249 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9250 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9251 )));
9252 }
9253
9254 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9255 sheet: sheet.to_string(),
9256 row,
9257 col,
9258 formula: formula.to_string(),
9259 message: message.clone(),
9260 policy,
9261 });
9262
9263 match policy {
9264 FormulaParsePolicy::Strict => unreachable!(),
9265 FormulaParsePolicy::KeepCachedValue => Ok(None),
9266 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9267 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9268 None,
9269 ))),
9270 FormulaParsePolicy::CoerceToError => {
9271 let err = ExcelError::new(ExcelErrorKind::Error)
9272 .with_message(format!("Malformed formula: {message}"));
9273 Ok(Some(ASTNode::new(
9274 ASTNodeType::Literal(LiteralValue::Error(err)),
9275 None,
9276 )))
9277 }
9278 }
9279 }
9280
9281 #[cfg(test)]
9282 fn target_preparation_fault(
9283 &mut self,
9284 seam: crate::engine::target_preparation::TargetPreparationFault,
9285 ) -> Result<(), ExcelError> {
9286 if self.target_preparation_fault_for_test == Some(seam) {
9287 self.target_preparation_fault_for_test = None;
9288 Err(ExcelError::new(ExcelErrorKind::Value)
9289 .with_message(format!("injected target preparation fault: {seam:?}")))
9290 } else {
9291 Ok(())
9292 }
9293 }
9294
9295 fn preparation_stale(
9296 reason: formualizer_common::PreparationStaleReason,
9297 message: impl Into<String>,
9298 ) -> ExcelError {
9299 ExcelError::new(ExcelErrorKind::Value)
9300 .with_message(message)
9301 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
9302 }
9303
9304 fn preparation_revision_stale_reason(
9305 assumptions: &crate::engine::PreparationRevision,
9306 current: &crate::engine::PreparationRevision,
9307 planning_requests: &BTreeSet<(String, String, usize)>,
9308 staged_leases_match: bool,
9309 ) -> Option<formualizer_common::PreparationStaleReason> {
9310 if assumptions.graph != current.graph {
9311 Some(formualizer_common::PreparationStaleReason::Graph)
9312 } else if assumptions.authority != current.authority
9313 || assumptions.authority_indexes != current.authority_indexes
9314 || assumptions.authority_indexed_plane != current.authority_indexed_plane
9315 {
9316 Some(formualizer_common::PreparationStaleReason::Authority)
9317 } else if assumptions.staged != current.staged || !staged_leases_match {
9318 Some(formualizer_common::PreparationStaleReason::Staged)
9319 } else if assumptions.symbols != current.symbols {
9320 Some(formualizer_common::PreparationStaleReason::Symbols)
9321 } else if assumptions.provider != current.provider {
9322 Some(formualizer_common::PreparationStaleReason::Provider)
9323 } else if assumptions.semantic != current.semantic
9324 && crate::function_registry::semantic_changes_affect_requests_since(
9325 assumptions.semantic,
9326 planning_requests.iter().cloned(),
9327 )
9328 {
9329 Some(formualizer_common::PreparationStaleReason::Semantic)
9330 } else {
9331 None
9332 }
9333 }
9334
9335 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
9336 let (authority, authority_indexes, authority_indexed_plane) =
9337 self.graph.authority_revisions();
9338 crate::engine::PreparationRevision {
9339 graph: self.graph.topology_revision(),
9340 authority,
9341 authority_indexes,
9342 authority_indexed_plane,
9343 staged: self.staged_formula_index.revision(),
9344 symbols: self.graph.symbol_revision(),
9345 semantic: crate::function_registry::semantic_epoch(),
9346 provider: self.resolver.planning_semantic_revision(),
9347 }
9348 }
9349
9350 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
9351 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
9352 let provider = self.resolver.planning_semantic_revision();
9353 let semantic = registry_guard.epoch();
9354 let (authority, authority_indexes, authority_indexed_plane) =
9355 self.graph.authority_revisions();
9356 let mut span_refs = self.graph.formula_authority().active_span_refs();
9357 span_refs.sort_unstable_by_key(|span_ref| {
9358 (span_ref.id.0, span_ref.generation, span_ref.version)
9359 });
9360 PlanningRevisionSnapshot {
9361 engine_topology_epoch: self.topology_epoch,
9362 graph_topology_revision: self.graph.topology_revision(),
9363 authority,
9364 authority_indexes,
9365 authority_indexed_plane,
9366 staged: self.staged_formula_index.revision(),
9367 symbols: self.graph.symbol_revision(),
9368 semantic,
9369 provider,
9370 formula_plane_mode: self.config.formula_plane_mode,
9371 deterministic_mode: self.config.deterministic_mode.clone(),
9372 budgets: self.evaluation_resource_budgets.clone(),
9373 span_refs,
9374 }
9375 }
9376
9377 fn recalc_plan_key(&self) -> RecalcPlanKey {
9378 RecalcPlanKey {
9379 engine_token: Arc::clone(&self.recalc_plan_token),
9380 revisions: self.planning_revision_snapshot(),
9381 }
9382 }
9383
9384 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
9385 ExcelError::new(ExcelErrorKind::Value)
9386 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
9387 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
9388 }
9389
9390 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
9391 use formualizer_common::PlanStaleReason;
9392
9393 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
9394 return Err(Self::plan_stale(PlanStaleReason::Engine));
9395 }
9396
9397 let current = self.planning_revision_snapshot();
9398 let expected = &key.revisions;
9399 let stale = if expected.provider != current.provider {
9400 Some(PlanStaleReason::Provider)
9401 } else if expected.semantic != current.semantic {
9402 Some(PlanStaleReason::Semantic)
9403 } else if expected.budgets != current.budgets
9404 || expected.deterministic_mode != current.deterministic_mode
9405 {
9406 Some(PlanStaleReason::Budget)
9407 } else if expected.staged != current.staged {
9408 Some(PlanStaleReason::Staged)
9409 } else if expected.symbols != current.symbols {
9410 Some(PlanStaleReason::Symbols)
9411 } else if expected.formula_plane_mode != current.formula_plane_mode
9412 || expected.authority != current.authority
9413 || expected.authority_indexes != current.authority_indexes
9414 || expected.authority_indexed_plane != current.authority_indexed_plane
9415 {
9416 Some(PlanStaleReason::Authority)
9417 } else if expected.span_refs != current.span_refs {
9418 Some(PlanStaleReason::SpanGeneration)
9419 } else if expected.graph_topology_revision != current.graph_topology_revision
9420 || expected.engine_topology_epoch != current.engine_topology_epoch
9421 {
9422 Some(PlanStaleReason::Graph)
9423 } else {
9424 None
9425 };
9426 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
9427 }
9428
9429 fn target_preparation_checkpoint(
9430 &mut self,
9431 deadline: Option<std::time::Instant>,
9432 work: u64,
9433 ) -> Result<(), ExcelError> {
9434 if self
9435 .active_cancel_flag
9436 .as_ref()
9437 .is_some_and(|cancel| cancel.is_cancelled())
9438 {
9439 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
9440 .with_message("target graph preparation cancelled"));
9441 }
9442 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
9443 return Err(crate::engine::ResourceLedgerError::Exhausted(
9444 formualizer_common::ResourceExhaustionDetail {
9445 reason: formualizer_common::ResourceExhaustionReason::Deadline,
9446 limit: 0,
9447 observed: 1,
9448 request_id: self
9449 .active_evaluation_resource_request
9450 .as_ref()
9451 .map(|stats| stats.request_id),
9452 },
9453 )
9454 .into_excel_error());
9455 }
9456 self.charge_bounded_work(work)
9457 }
9458
9459 fn opaque_reason_in_ast(
9460 &self,
9461 ast: &ASTNode,
9462 provider: &dyn crate::traits::FunctionProvider,
9463 ) -> Option<crate::engine::OpaqueReason> {
9464 match &ast.node_type {
9465 ASTNodeType::Function { name, args } => {
9466 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
9467 if canonical == "INDIRECT" {
9468 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
9469 }
9470 let Some(function) = provider.get_function_for_planning("", name) else {
9471 return Some(crate::engine::OpaqueReason::UnknownFunction);
9472 };
9473 let caps = function.caps();
9474 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9475 || caps.contains(FnCaps::RETURNS_REFERENCE)
9476 {
9477 return Some(crate::engine::OpaqueReason::DynamicReference);
9478 }
9479 args.iter()
9480 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
9481 }
9482 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
9483 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
9484 ASTNodeType::BinaryOp { left, right, .. } => self
9485 .opaque_reason_in_ast(left, provider)
9486 .or_else(|| self.opaque_reason_in_ast(right, provider)),
9487 ASTNodeType::Array(rows) => rows
9488 .iter()
9489 .flat_map(|row| row.iter())
9490 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
9491 ASTNodeType::Reference {
9492 reference:
9493 ReferenceType::Cell {
9494 sheet: Some(sheet), ..
9495 }
9496 | ReferenceType::Range {
9497 sheet: Some(sheet), ..
9498 },
9499 ..
9500 } if self.graph.sheet_id(sheet).is_none() => {
9501 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
9502 }
9503 ASTNodeType::Reference {
9504 reference:
9505 ReferenceType::External(_)
9506 | ReferenceType::Cell3D { .. }
9507 | ReferenceType::Range3D { .. },
9508 ..
9509 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
9510 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
9511 }
9512 }
9513
9514 fn target_planning_snapshot(
9515 &mut self,
9516 ast: &ASTNode,
9517 planning_requests: &mut BTreeSet<(String, String, usize)>,
9518 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
9519 #[cfg(test)]
9520 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
9521 return Err(Self::preparation_stale(
9522 formualizer_common::PreparationStaleReason::Semantic,
9523 "injected target semantic preparation movement",
9524 ));
9525 }
9526 #[cfg(test)]
9527 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
9528 hook();
9529 }
9530 let mut requests = Vec::new();
9531 Self::collect_planning_function_requests(ast, &mut requests);
9532 requests.sort();
9533 requests.dedup();
9534 planning_requests.extend(requests.iter().cloned());
9535 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
9536 &self.resolver,
9537 requests,
9538 )
9539 .map_err(|error| {
9540 ExcelError::new(ExcelErrorKind::Value)
9541 .with_message(format!("target planning snapshot unavailable: {error:?}"))
9542 })
9543 }
9544
9545 fn target_planning_snapshot_stale_reason(
9546 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
9547 assumptions: &crate::engine::PreparationRevision,
9548 ) -> Option<formualizer_common::PreparationStaleReason> {
9549 if snapshot
9550 .provider_revision()
9551 .is_some_and(|revision| Some(revision) != assumptions.provider)
9552 {
9553 Some(formualizer_common::PreparationStaleReason::Provider)
9554 } else if snapshot.epoch() != assumptions.semantic
9555 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
9556 {
9557 Some(formualizer_common::PreparationStaleReason::Semantic)
9558 } else {
9559 None
9560 }
9561 }
9562
9563 fn widen_target_preparation(
9564 policy: crate::engine::OpaquePreparePolicy,
9565 scope: &mut crate::engine::PrepareScope,
9566 reasons: &mut Vec<crate::engine::OpaqueReason>,
9567 reason: crate::engine::OpaqueReason,
9568 ) -> Result<bool, ExcelError> {
9569 if policy == crate::engine::OpaquePreparePolicy::Error {
9570 return Err(ExcelError::new(ExcelErrorKind::NImpl)
9571 .with_message(format!("opaque target preparation semantics: {reason:?}")));
9572 }
9573 if !reasons.contains(&reason) {
9574 reasons.push(reason);
9575 }
9576 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
9577 *scope = crate::engine::PrepareScope::Workbook;
9578 Ok(true)
9579 } else {
9580 Ok(false)
9581 }
9582 }
9583
9584 fn widen_target_preparation_to_sheet(
9585 policy: crate::engine::OpaquePreparePolicy,
9586 scope: &mut crate::engine::PrepareScope,
9587 reasons: &mut Vec<crate::engine::OpaqueReason>,
9588 reason: crate::engine::OpaqueReason,
9589 sheet: &str,
9590 ) -> Result<bool, ExcelError> {
9591 if policy == crate::engine::OpaquePreparePolicy::Error {
9592 return Err(ExcelError::new(ExcelErrorKind::NImpl)
9593 .with_message(format!("opaque target preparation semantics: {reason:?}")));
9594 }
9595 if !reasons.contains(&reason) {
9596 reasons.push(reason);
9597 }
9598 match scope {
9599 crate::engine::PrepareScope::Exact => {
9600 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
9601 Ok(true)
9602 }
9603 crate::engine::PrepareScope::Sheets(sheets) => {
9604 if sheets.iter().any(|candidate| candidate == sheet) {
9605 Ok(false)
9606 } else {
9607 sheets.push(sheet.to_string());
9608 sheets.sort();
9609 Ok(true)
9610 }
9611 }
9612 crate::engine::PrepareScope::Workbook => Ok(false),
9613 }
9614 }
9615
9616 fn ast_has_proven_sheet_local_dynamic(
9617 ast: &ASTNode,
9618 provider: &dyn crate::traits::FunctionProvider,
9619 ) -> bool {
9620 fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
9621 match &ast.node_type {
9622 ASTNodeType::Function { name, args } => {
9623 let Some(function) = provider.get_function_for_planning("", name) else {
9624 return (false, false);
9625 };
9626 let caps = function.caps();
9627 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9628 || caps.contains(FnCaps::RETURNS_REFERENCE);
9629 let canonical = name.rsplit('.').next().unwrap_or(name);
9630 if dynamic
9631 && !canonical.eq_ignore_ascii_case("OFFSET")
9632 && !canonical.eq_ignore_ascii_case("INDEX")
9633 {
9634 return (false, true);
9635 }
9636 let mut has_dynamic = dynamic;
9637 for arg in args {
9638 let (safe, child_dynamic) = classify(arg, provider);
9639 if !safe {
9640 return (false, has_dynamic || child_dynamic);
9641 }
9642 has_dynamic |= child_dynamic;
9643 }
9644 (true, has_dynamic)
9645 }
9646 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
9647 ASTNodeType::BinaryOp { left, right, .. } => {
9648 let (left_safe, left_dynamic) = classify(left, provider);
9649 let (right_safe, right_dynamic) = classify(right, provider);
9650 (left_safe && right_safe, left_dynamic || right_dynamic)
9651 }
9652 ASTNodeType::Array(rows) => {
9653 let mut has_dynamic = false;
9654 for item in rows.iter().flatten() {
9655 let (safe, child_dynamic) = classify(item, provider);
9656 if !safe {
9657 return (false, has_dynamic || child_dynamic);
9658 }
9659 has_dynamic |= child_dynamic;
9660 }
9661 (true, has_dynamic)
9662 }
9663 ASTNodeType::Reference {
9664 reference:
9665 ReferenceType::Cell { sheet: None, .. }
9666 | ReferenceType::Range { sheet: None, .. },
9667 ..
9668 }
9669 | ASTNodeType::Literal(_)
9670 | ASTNodeType::Omitted => (true, false),
9671 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
9672 }
9673 }
9674
9675 let (safe, dynamic) = classify(ast, provider);
9676 safe && dynamic
9677 }
9678
9679 fn table_selection_region(
9680 &self,
9681 entry: &crate::engine::graph::TableEntry,
9682 selection: &crate::engine::TableSelection,
9683 ) -> Result<PreparationRegion, ExcelError> {
9684 let mut start_row = entry.range.start.coord.row() + 1;
9685 let mut end_row = entry.range.end.coord.row() + 1;
9686 let mut start_col = entry.range.start.coord.col() + 1;
9687 let mut end_col = entry.range.end.coord.col() + 1;
9688 match selection {
9689 crate::engine::TableSelection::Whole => {}
9690 crate::engine::TableSelection::Headers => {
9691 if !entry.header_row {
9692 return Err(ExcelError::new(ExcelErrorKind::Value)
9693 .with_message(format!("table {} has no header row", entry.name)));
9694 }
9695 end_row = start_row;
9696 }
9697 crate::engine::TableSelection::Data => {
9698 if entry.header_row {
9699 start_row = start_row.saturating_add(1);
9700 }
9701 if entry.totals_row {
9702 end_row = end_row.saturating_sub(1);
9703 }
9704 }
9705 crate::engine::TableSelection::Totals => {
9706 if !entry.totals_row {
9707 return Err(ExcelError::new(ExcelErrorKind::Value)
9708 .with_message(format!("table {} has no totals row", entry.name)));
9709 }
9710 start_row = end_row;
9711 }
9712 crate::engine::TableSelection::Column(column) => {
9713 let index = entry.col_index(column).ok_or_else(|| {
9714 ExcelError::new(ExcelErrorKind::Name)
9715 .with_message(format!("unknown table column: {column}"))
9716 })?;
9717 start_col = start_col.saturating_add(index as u32);
9718 end_col = start_col;
9719 }
9720 crate::engine::TableSelection::Columns { start, end } => {
9721 let first = entry.col_index(start).ok_or_else(|| {
9722 ExcelError::new(ExcelErrorKind::Name)
9723 .with_message(format!("unknown table column: {start}"))
9724 })?;
9725 let last = entry.col_index(end).ok_or_else(|| {
9726 ExcelError::new(ExcelErrorKind::Name)
9727 .with_message(format!("unknown table column: {end}"))
9728 })?;
9729 if first > last {
9730 return Err(ExcelError::new(ExcelErrorKind::Value)
9731 .with_message("table column selection is reversed"));
9732 }
9733 start_col = start_col.saturating_add(first as u32);
9734 end_col = entry
9735 .range
9736 .start
9737 .coord
9738 .col()
9739 .saturating_add(last as u32)
9740 .saturating_add(1);
9741 }
9742 }
9743 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
9744 start_row = entry.range.start.coord.row() + 1;
9745 end_row = start_row;
9746 }
9747 if start_row > end_row || start_col > end_col {
9748 return Err(
9749 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
9750 );
9751 }
9752 Ok(PreparationRegion {
9753 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
9754 sheet_id: entry.sheet_id(),
9755 start_row,
9756 start_col,
9757 end_row,
9758 end_col,
9759 })
9760 }
9761
9762 #[cfg(test)]
9763 pub(crate) fn reset_target_root_dedup_probes_for_test() {
9764 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
9765 }
9766
9767 #[cfg(test)]
9768 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
9769 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
9770 }
9771
9772 pub(crate) fn resolve_target_producers(
9773 &mut self,
9774 targets: &[crate::engine::EvaluationTarget],
9775 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9776 use crate::engine::target_preparation::TargetProducer;
9777
9778 let request_id = self
9779 .active_evaluation_resource_request
9780 .as_ref()
9781 .map(|request| request.request_id);
9782 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
9783 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
9784 let active_span_refs = self
9785 .graph
9786 .formula_authority()
9787 .active_span_refs()
9788 .into_iter()
9789 .map(|span_ref| (span_ref.id, span_ref))
9790 .collect::<BTreeMap<_, _>>();
9791 let resolve_region = |engine: &mut Self,
9792 region: Region,
9793 value_only: Option<CellRef>,
9794 roots: &mut OrderedTargetProducers|
9795 -> Result<(), ExcelError> {
9796 let before = roots.len();
9797 let authority = engine.graph.formula_authority();
9798 for matched in authority.producer_results.query(region).matches {
9799 if let FormulaProducerId::Span(span_id) = matched.value.producer
9800 && let Some(span_ref) = active_span_refs.get(&span_id).copied()
9801 && let Some(demanded) = matched.value.result_region.intersection(region)
9802 {
9803 roots
9804 .push(TargetProducer::Span { span_ref, demanded })
9805 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9806 }
9807 }
9808 for anchor in engine.graph.spill_anchors_in_region(
9809 region.sheet_id(),
9810 region.axis_ranges().0.query_bounds().0,
9811 region.axis_ranges().1.query_bounds().0,
9812 region.axis_ranges().0.query_bounds().1,
9813 region.axis_ranges().1.query_bounds().1,
9814 ) {
9815 roots
9816 .push(TargetProducer::Legacy(anchor))
9817 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9818 }
9819 for vertex in engine.graph.vertices_in_region(
9820 region.sheet_id(),
9821 region.axis_ranges().0.query_bounds().0,
9822 region.axis_ranges().0.query_bounds().1,
9823 region.axis_ranges().1.query_bounds().0,
9824 region.axis_ranges().1.query_bounds().1,
9825 ) {
9826 let vertex = engine
9827 .graph
9828 .get_cell_ref(vertex)
9829 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
9830 .unwrap_or(vertex);
9831 match engine.graph.get_vertex_kind(vertex) {
9832 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
9833 roots.push(TargetProducer::Legacy(vertex)).map_err(|_| {
9834 target_root_allocation_error(roots.len() + 1, request_id)
9835 })?;
9836 }
9837 VertexKind::NamedScalar
9838 | VertexKind::NamedArray
9839 | VertexKind::Range
9840 | VertexKind::InfiniteRange
9841 | VertexKind::Table => {
9842 roots.push(TargetProducer::Symbol(vertex)).map_err(|_| {
9843 target_root_allocation_error(roots.len() + 1, request_id)
9844 })?;
9845 }
9846 VertexKind::Empty | VertexKind::Cell | VertexKind::External => {}
9847 }
9848 }
9849 if roots.len() == before
9850 && let Some(cell) = value_only
9851 {
9852 roots
9853 .push(TargetProducer::ValueOnly(cell))
9854 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9855 }
9856 Ok(())
9857 };
9858
9859 for target in targets {
9860 match target {
9861 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
9862 if *row == 0 || *col == 0 {
9863 return Err(ExcelError::new(ExcelErrorKind::Ref)
9864 .with_message("target cell coordinates are one-based"));
9865 }
9866 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
9867 ExcelError::new(ExcelErrorKind::Ref)
9868 .with_message(format!("target sheet not found: {sheet}"))
9869 })?;
9870 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
9871 resolve_region(
9872 self,
9873 Region::point(sheet_id, *row - 1, *col - 1),
9874 Some(cell),
9875 &mut roots,
9876 )?;
9877 }
9878 crate::engine::EvaluationTarget::Range(range) => {
9879 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
9880 ExcelError::new(ExcelErrorKind::Ref)
9881 .with_message(format!("target sheet not found: {}", range.sheet))
9882 })?;
9883 resolve_region(
9884 self,
9885 Region::rect(
9886 sheet_id,
9887 range.start_row - 1,
9888 range.end_row - 1,
9889 range.start_col - 1,
9890 range.end_col - 1,
9891 ),
9892 None,
9893 &mut roots,
9894 )?;
9895 }
9896 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
9897 let scope = self.name_query_scope(scope_sheet.as_deref())?;
9898 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
9899 roots
9900 .push(TargetProducer::Symbol(entry.vertex))
9901 .map_err(|_| {
9902 target_root_allocation_error(roots.len() + 1, request_id)
9903 })?;
9904 }
9905 }
9906 crate::engine::EvaluationTarget::Table { name, .. } => {
9907 if let Some(entry) = self.graph.resolve_table_entry(name) {
9908 roots
9909 .push(TargetProducer::Symbol(entry.vertex))
9910 .map_err(|_| {
9911 target_root_allocation_error(roots.len() + 1, request_id)
9912 })?;
9913 }
9914 }
9915 }
9916 }
9917 Ok(roots.into_vec())
9918 }
9919
9920 fn resolve_target_producers_from_existing_roots(
9921 &mut self,
9922 roots: &[crate::engine::target_preparation::TargetProducer],
9923 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9924 use crate::engine::target_preparation::TargetProducer;
9925 let request_id = self
9926 .active_evaluation_resource_request
9927 .as_ref()
9928 .map(|request| request.request_id);
9929 let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
9930 .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
9931 for root in roots {
9932 match *root {
9933 TargetProducer::Span { demanded, .. } => {
9934 let (rows, cols) = demanded.axis_ranges();
9935 for vertex in self.graph.vertices_in_region(
9936 demanded.sheet_id(),
9937 rows.query_bounds().0,
9938 rows.query_bounds().1,
9939 cols.query_bounds().0,
9940 cols.query_bounds().1,
9941 ) {
9942 if matches!(
9943 self.graph.get_vertex_kind(vertex),
9944 VertexKind::FormulaScalar | VertexKind::FormulaArray
9945 ) {
9946 refreshed
9947 .push(TargetProducer::Legacy(vertex))
9948 .map_err(|_| {
9949 target_root_allocation_error(refreshed.len() + 1, request_id)
9950 })?;
9951 }
9952 }
9953 }
9954 other => {
9955 refreshed.push(other).map_err(|_| {
9956 target_root_allocation_error(refreshed.len() + 1, request_id)
9957 })?;
9958 }
9959 }
9960 }
9961 Ok(refreshed.into_vec())
9962 }
9963
9964 pub fn prepare_graph_for_targets(
9968 &mut self,
9969 targets: &[crate::engine::EvaluationTarget],
9970 options: crate::engine::TargetEvalOptions<'_>,
9971 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9972 let previous_budgets = self.evaluation_resource_budgets.clone();
9973 let diagnostics_len = self.formula_parse_diagnostics.len();
9974 let previous_report = self.last_formula_ingest_report.clone();
9975 if let Some(budgets) = options.budgets {
9976 self.evaluation_resource_budgets = budgets.clone();
9977 }
9978 let previous_graph_budget_override = self
9979 .graph
9980 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
9981 let previous_cancel = self.active_cancel_flag.take();
9985 self.active_cancel_flag = options.cancel.clone();
9986 let result = self.observe_evaluation_resource_request(
9987 EvaluationRequestKind::TargetPreparation,
9988 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
9989 );
9990 self.active_cancel_flag = previous_cancel;
9991 self.graph
9992 .set_admission_budget_override(previous_graph_budget_override);
9993 self.evaluation_resource_budgets = previous_budgets;
9994 if result.is_err() {
9995 self.formula_parse_diagnostics.truncate(diagnostics_len);
9996 self.last_formula_ingest_report = previous_report;
9997 }
9998 result
9999 }
10000
10001 fn prepare_graph_for_targets_unobserved(
10002 &mut self,
10003 targets: &[crate::engine::EvaluationTarget],
10004 options: &crate::engine::TargetEvalOptions<'_>,
10005 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10006 let scratch_checkpoint = self
10007 .active_resource_ledger
10008 .as_ref()
10009 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
10010 let result = self.prepare_graph_for_targets_transaction(targets, options);
10011 let release = self
10012 .active_resource_ledger
10013 .as_mut()
10014 .map_or(Ok(()), |ledger| {
10015 ledger.release_scratch_to(scratch_checkpoint)
10016 });
10017 match (result, release) {
10018 (result, Ok(())) => result,
10019 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
10020 }
10021 }
10022
10023 fn prepare_graph_for_targets_transaction(
10024 &mut self,
10025 targets: &[crate::engine::EvaluationTarget],
10026 options: &crate::engine::TargetEvalOptions<'_>,
10027 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10028 use crate::engine::{
10029 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
10030 TableSelection,
10031 };
10032
10033 self.target_preparation_checkpoint(options.deadline, 0)?;
10034 self.observe_function_semantic_epoch()?;
10035 let assumptions = self.preparation_revisions();
10036 let ledger_at_start = self
10037 .active_resource_ledger
10038 .as_ref()
10039 .map(|ledger| ledger.snapshot());
10040 let diagnostics_len = self.formula_parse_diagnostics.len();
10041 let report_len = self.last_formula_ingest_report.clone();
10042 let request_id = options.request_id.or_else(|| {
10043 self.active_evaluation_resource_request
10044 .as_ref()
10045 .map(|stats| stats.request_id)
10046 });
10047
10048 let mut scope = PrepareScope::Exact;
10049 let mut reasons = Vec::new();
10050 let mut regions = VecDeque::new();
10051 let mut normalized = Vec::with_capacity(targets.len());
10052 let mut symbol_vertices = VecDeque::new();
10053 for target in targets {
10054 self.target_preparation_checkpoint(options.deadline, 1)?;
10055 match target {
10056 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10057 if *row == 0 || *col == 0 {
10058 return Err(ExcelError::new(ExcelErrorKind::Ref)
10059 .with_message("target cell coordinates are one-based"));
10060 }
10061 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10062 ExcelError::new(ExcelErrorKind::Ref)
10063 .with_message(format!("target sheet not found: {sheet}"))
10064 })?;
10065 regions.push_back(PreparationRegion {
10066 sheet: sheet.clone(),
10067 sheet_id,
10068 start_row: *row,
10069 start_col: *col,
10070 end_row: *row,
10071 end_col: *col,
10072 });
10073 normalized.push(target.clone());
10074 }
10075 crate::engine::EvaluationTarget::Range(range) => {
10076 if range.start_row == 0
10077 || range.start_col == 0
10078 || range.end_row < range.start_row
10079 || range.end_col < range.start_col
10080 {
10081 return Err(ExcelError::new(ExcelErrorKind::Ref)
10082 .with_message("invalid target range"));
10083 }
10084 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10085 ExcelError::new(ExcelErrorKind::Ref)
10086 .with_message(format!("target sheet not found: {}", range.sheet))
10087 })?;
10088 regions.push_back(PreparationRegion {
10089 sheet: range.sheet.clone(),
10090 sheet_id,
10091 start_row: range.start_row,
10092 start_col: range.start_col,
10093 end_row: range.end_row,
10094 end_col: range.end_col,
10095 });
10096 normalized.push(target.clone());
10097 }
10098 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10099 let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
10100 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
10101 symbol_vertices.push_back(entry.vertex);
10102 } else {
10103 Self::widen_target_preparation(
10104 options.opaque_policy,
10105 &mut scope,
10106 &mut reasons,
10107 OpaqueReason::UnresolvedName,
10108 )?;
10109 }
10110 normalized.push(target.clone());
10111 }
10112 crate::engine::EvaluationTarget::Table { name, selection } => {
10113 if let Some(entry) = self.graph.resolve_table_entry(name) {
10114 let region = self.table_selection_region(entry, selection)?;
10115 symbol_vertices.push_back(entry.vertex);
10116 regions.push_back(region);
10117 } else {
10118 Self::widen_target_preparation(
10119 options.opaque_policy,
10120 &mut scope,
10121 &mut reasons,
10122 OpaqueReason::UnresolvedTable,
10123 )?;
10124 }
10125 normalized.push(target.clone());
10126 }
10127 }
10128 }
10129
10130 let mut visited_regions = FxHashSet::default();
10131 let mut visited_vertices = FxHashSet::default();
10132 let mut selected = FxHashSet::default();
10133 let mut prepared = Vec::new();
10134 let mut pending_diagnostics = Vec::new();
10135 let mut planning_requests = BTreeSet::new();
10136 let mut selected_cells = Vec::new();
10137 let mut workbook_seeded = false;
10138 let mut sheet_scope_seeded = BTreeSet::new();
10139 let mut indexed_query_sheets = FxHashSet::default();
10140 let mut discovery_scratch_reserved = 0u64;
10141 let mut package_encountered = false;
10142 let mut selected_package_sheets = FxHashSet::default();
10143 let mut prepared_packages = Vec::new();
10144 let authoritative_with_ordinary = self.config.formula_plane_mode
10145 == FormulaPlaneMode::AuthoritativeExperimental
10146 && self.staged_formula_index.ordinary_count() != 0;
10147 let has_unknown_package_sheet = self
10148 .staged_formula_index
10149 .package_sheets()
10150 .any(|sheet| self.graph.sheet_id(sheet).is_none());
10151
10152 loop {
10153 if let PrepareScope::Sheets(sheets) = &scope {
10154 for sheet in sheets.clone() {
10155 if !sheet_scope_seeded.insert(sheet.clone()) {
10156 continue;
10157 }
10158 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10159 package_encountered = true;
10160 Self::widen_target_preparation(
10161 options.opaque_policy,
10162 &mut scope,
10163 &mut reasons,
10164 OpaqueReason::UnsupportedSourceSemantics,
10165 )?;
10166 continue;
10167 };
10168 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10169 self.target_preparation_checkpoint(options.deadline, 1)?;
10170 regions.push_back(PreparationRegion {
10171 sheet: sheet.clone(),
10172 sheet_id,
10173 start_row: lease.row,
10174 start_col: lease.col,
10175 end_row: lease.row,
10176 end_col: lease.col,
10177 });
10178 }
10179 if self
10180 .staged_formula_index
10181 .package_lease_for_sheet(&sheet)
10182 .is_some()
10183 {
10184 regions.push_back(PreparationRegion {
10185 sheet: sheet.clone(),
10186 sheet_id,
10187 start_row: 1,
10188 start_col: 1,
10189 end_row: self.workbook_load_limits.max_sheet_rows,
10190 end_col: self.workbook_load_limits.max_sheet_cols,
10191 });
10192 }
10193 }
10194 }
10195 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10196 workbook_seeded = true;
10197 for (sheet, lease) in self.staged_formula_index.all_leases() {
10198 self.target_preparation_checkpoint(options.deadline, 1)?;
10199 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10200 package_encountered = true;
10201 Self::widen_target_preparation(
10202 options.opaque_policy,
10203 &mut scope,
10204 &mut reasons,
10205 OpaqueReason::UnsupportedSourceSemantics,
10206 )?;
10207 continue;
10208 };
10209 regions.push_back(PreparationRegion {
10210 sheet,
10211 sheet_id,
10212 start_row: lease.row,
10213 start_col: lease.col,
10214 end_row: lease.row,
10215 end_col: lease.col,
10216 });
10217 }
10218 let package_sheets = self
10219 .staged_formula_index
10220 .package_sheets()
10221 .map(str::to_string)
10222 .collect::<Vec<_>>();
10223 for sheet in package_sheets {
10224 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10225 package_encountered = true;
10226 Self::widen_target_preparation(
10227 options.opaque_policy,
10228 &mut scope,
10229 &mut reasons,
10230 OpaqueReason::UnsupportedSourceSemantics,
10231 )?;
10232 continue;
10233 };
10234 regions.push_back(PreparationRegion {
10235 sheet,
10236 sheet_id,
10237 start_row: 1,
10238 start_col: 1,
10239 end_row: self.workbook_load_limits.max_sheet_rows,
10240 end_col: self.workbook_load_limits.max_sheet_cols,
10241 });
10242 }
10243 }
10244
10245 let Some(region) = regions.pop_front() else {
10246 if let Some(vertex) = symbol_vertices.pop_front() {
10247 self.target_preparation_checkpoint(options.deadline, 1)?;
10248 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10249 continue;
10250 }
10251 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10252 if let Some(ast) = self.graph.get_formula(vertex) {
10253 let snapshot =
10254 self.target_planning_snapshot(&ast, &mut planning_requests)?;
10255 if let Some(reason) =
10256 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10257 {
10258 return Err(Self::preparation_stale(
10259 reason,
10260 "target planning snapshot became stale during discovery",
10261 ));
10262 }
10263 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10264 if let Some(reason) =
10265 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10266 {
10267 if reason == OpaqueReason::DynamicReference
10268 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10269 {
10270 let sheet = self.graph.get_vertex_sheet_id(vertex);
10271 let sheet = self.graph.sheet_name(sheet).to_string();
10272 Self::widen_target_preparation_to_sheet(
10273 options.opaque_policy,
10274 &mut scope,
10275 &mut reasons,
10276 reason,
10277 &sheet,
10278 )?;
10279 } else {
10280 Self::widen_target_preparation(
10281 options.opaque_policy,
10282 &mut scope,
10283 &mut reasons,
10284 reason,
10285 )?;
10286 }
10287 }
10288 } else if vertex_is_dynamic {
10289 Self::widen_target_preparation(
10290 options.opaque_policy,
10291 &mut scope,
10292 &mut reasons,
10293 OpaqueReason::DynamicReference,
10294 )?;
10295 }
10296 if let Some(anchor) = self
10297 .graph
10298 .get_cell_ref(vertex)
10299 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
10300 {
10301 symbol_vertices.push_back(anchor);
10302 }
10303 if let Some(cell) = self.graph.get_cell_ref(vertex) {
10304 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
10305 regions.push_back(PreparationRegion {
10306 sheet,
10307 sheet_id: cell.sheet_id,
10308 start_row: cell.coord.row() + 1,
10309 start_col: cell.coord.col() + 1,
10310 end_row: cell.coord.row() + 1,
10311 end_col: cell.coord.col() + 1,
10312 });
10313 }
10314 for dependency in self.graph.get_dependencies(vertex) {
10315 self.target_preparation_checkpoint(options.deadline, 1)?;
10316 symbol_vertices.push_back(dependency);
10317 }
10318 if let Some(range_dependencies) = self
10319 .graph
10320 .formula_range_dependencies(vertex)
10321 .map(<[_]>::to_vec)
10322 {
10323 for range in range_dependencies {
10324 self.target_preparation_checkpoint(options.deadline, 1)?;
10325 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
10327 let Ok(sheet_id) =
10328 self.resolve_sheet_locator(&range.sheet, context_sheet)
10329 else {
10330 if Self::widen_target_preparation(
10331 options.opaque_policy,
10332 &mut scope,
10333 &mut reasons,
10334 OpaqueReason::UnresolvedCrossSheetBinding,
10335 )? {
10336 break;
10337 }
10338 continue;
10339 };
10340 let sheet = self.graph.sheet_name(sheet_id).to_string();
10341 regions.push_back(PreparationRegion {
10342 sheet,
10343 sheet_id,
10344 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10345 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10346 end_row: range
10347 .end_row
10348 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10349 bound.index + 1
10350 }),
10351 end_col: range
10352 .end_col
10353 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10354 bound.index + 1
10355 }),
10356 });
10357 }
10358 }
10359 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
10360 match &name.definition {
10361 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
10362 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
10363 sheet_id: cell.sheet_id,
10364 start_row: cell.coord.row() + 1,
10365 start_col: cell.coord.col() + 1,
10366 end_row: cell.coord.row() + 1,
10367 end_col: cell.coord.col() + 1,
10368 }),
10369 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
10370 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
10371 sheet_id: range.start.sheet_id,
10372 start_row: range.start.coord.row() + 1,
10373 start_col: range.start.coord.col() + 1,
10374 end_row: range.end.coord.row() + 1,
10375 end_col: range.end.coord.col() + 1,
10376 }),
10377 NamedDefinition::Formula {
10378 ast,
10379 dependencies,
10380 range_deps,
10381 } => {
10382 let snapshot =
10383 self.target_planning_snapshot(ast, &mut planning_requests)?;
10384 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
10385 &snapshot,
10386 &assumptions,
10387 ) {
10388 return Err(Self::preparation_stale(
10389 reason,
10390 "target planning snapshot became stale during discovery",
10391 ));
10392 }
10393 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
10394 Self::widen_target_preparation(
10395 options.opaque_policy,
10396 &mut scope,
10397 &mut reasons,
10398 reason,
10399 )?;
10400 }
10401 for dependency in dependencies {
10402 self.target_preparation_checkpoint(options.deadline, 1)?;
10403 symbol_vertices.push_back(*dependency);
10404 }
10405 for range in range_deps {
10406 self.target_preparation_checkpoint(options.deadline, 1)?;
10407 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
10415 let Ok(sheet_id) =
10416 self.resolve_sheet_locator(&range.sheet, context_sheet)
10417 else {
10418 if Self::widen_target_preparation(
10419 options.opaque_policy,
10420 &mut scope,
10421 &mut reasons,
10422 OpaqueReason::UnresolvedCrossSheetBinding,
10423 )? {
10424 break;
10425 }
10426 continue;
10427 };
10428 regions.push_back(PreparationRegion {
10429 sheet: self.graph.sheet_name(sheet_id).to_string(),
10430 sheet_id,
10431 start_row: range
10432 .start_row
10433 .map_or(1, |bound| bound.index + 1),
10434 start_col: range
10435 .start_col
10436 .map_or(1, |bound| bound.index + 1),
10437 end_row: range.end_row.map_or(
10438 self.workbook_load_limits.max_sheet_rows,
10439 |bound| bound.index + 1,
10440 ),
10441 end_col: range.end_col.map_or(
10442 self.workbook_load_limits.max_sheet_cols,
10443 |bound| bound.index + 1,
10444 ),
10445 });
10446 }
10447 }
10448 NamedDefinition::Literal(_) => {}
10449 }
10450 }
10451 if let Some(table) = self.graph.table_by_vertex(vertex) {
10452 regions
10453 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
10454 }
10455 continue;
10456 }
10457 break;
10458 };
10459
10460 self.target_preparation_checkpoint(options.deadline, 1)?;
10461 if !visited_regions.insert(region.clone()) {
10462 continue;
10463 }
10464 if let PrepareScope::Sheets(sheets) = &mut scope
10465 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
10466 {
10467 sheets.push(region.sheet.clone());
10468 sheets.sort();
10469 }
10470 let package_match = self.staged_formula_index.package_for_region(
10471 ®ion.sheet,
10472 region.start_row,
10473 region.start_col,
10474 region.end_row,
10475 region.end_col,
10476 );
10477 let package_lease = match package_match {
10478 Some(Ok(lease)) => Some(lease),
10479 Some(Err(()))
10480 if region.start_row == 1
10481 && region.start_col == 1
10482 && region.end_row == self.workbook_load_limits.max_sheet_rows
10483 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
10484 {
10485 self.staged_formula_index
10486 .package_lease_for_sheet(®ion.sheet)
10487 }
10488 Some(Err(())) => {
10489 Self::widen_target_preparation(
10490 options.opaque_policy,
10491 &mut scope,
10492 &mut reasons,
10493 OpaqueReason::DeferredSourcePackage,
10494 )?;
10495 None
10496 }
10497 None => None,
10498 };
10499 let compatibility_before_package_replay = package_lease.is_some()
10500 && (authoritative_with_ordinary || has_unknown_package_sheet);
10501 let package_lease = if compatibility_before_package_replay {
10502 package_encountered = true;
10503 Self::widen_target_preparation(
10504 options.opaque_policy,
10505 &mut scope,
10506 &mut reasons,
10507 OpaqueReason::UnsupportedSourceSemantics,
10508 )?;
10509 None
10510 } else {
10511 package_lease
10512 };
10513 if let Some(package_lease) = package_lease
10514 && selected_package_sheets.insert(region.sheet.clone())
10515 {
10516 self.target_preparation_checkpoint(options.deadline, 1)?;
10517 let mut package = self.prepare_target_source_package(
10518 ®ion.sheet,
10519 package_lease,
10520 options.deadline,
10521 )?;
10522 for placement in &package.placements {
10523 self.target_preparation_checkpoint(options.deadline, 1)?;
10524 for dependency in &placement.fragment_dependency_proof().1.dependencies {
10525 self.target_preparation_checkpoint(options.deadline, 1)?;
10526 let (rows, cols) = dependency.read_region.axis_ranges();
10527 let (start_row, end_row) = rows.query_bounds();
10528 let (start_col, end_col) = cols.query_bounds();
10529 let dependency_sheet = dependency.read_region.sheet_id();
10530 regions.push_back(PreparationRegion {
10531 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10532 sheet_id: dependency_sheet,
10533 start_row: start_row.saturating_add(1),
10534 start_col: start_col.saturating_add(1),
10535 end_row: end_row
10536 .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
10537 .saturating_add(1),
10538 end_col: end_col
10539 .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
10540 .saturating_add(1),
10541 });
10542 }
10543 }
10544
10545 let mut final_fallback = BTreeMap::new();
10546 for record in package.fallback_records() {
10547 final_fallback.insert((record.row, record.col), record.clone());
10548 }
10549 let batch = self
10550 .formula_batch_from_exact_replay(®ion.sheet, final_fallback.into_values())?;
10551 for record in batch.formulas {
10552 self.target_preparation_checkpoint(options.deadline, 1)?;
10553 let ast = self
10554 .graph
10555 .data_store()
10556 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
10557 .ok_or_else(|| {
10558 ExcelError::new(ExcelErrorKind::Value)
10559 .with_message("target fallback AST is unavailable")
10560 })?;
10561 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10562 if let Some(reason) =
10563 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10564 {
10565 return Err(Self::preparation_stale(
10566 reason,
10567 "target fallback planning snapshot became stale during discovery",
10568 ));
10569 }
10570 let proven_sheet_local_dynamic =
10571 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10572 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10573 if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
10574 Self::widen_target_preparation_to_sheet(
10575 options.opaque_policy,
10576 &mut scope,
10577 &mut reasons,
10578 reason,
10579 ®ion.sheet,
10580 )?;
10581 } else {
10582 Self::widen_target_preparation(
10583 options.opaque_policy,
10584 &mut scope,
10585 &mut reasons,
10586 reason,
10587 )?;
10588 }
10589 }
10590 let placement = CellRef::new(
10591 package.sheet_id,
10592 Coord::from_excel(record.row, record.col, true, true),
10593 );
10594 let ingested = self
10595 .graph
10596 .ingest_pipeline(&snapshot)
10597 .enable_function_semantics()
10598 .ingest_formula(
10599 FormulaAstInput::RawArena(record.ast_id),
10600 placement,
10601 record.formula_text,
10602 )?;
10603 if ingested.dep_plan.dynamic {
10604 if proven_sheet_local_dynamic {
10605 Self::widen_target_preparation_to_sheet(
10606 options.opaque_policy,
10607 &mut scope,
10608 &mut reasons,
10609 OpaqueReason::DynamicReference,
10610 ®ion.sheet,
10611 )?;
10612 } else {
10613 Self::widen_target_preparation(
10614 options.opaque_policy,
10615 &mut scope,
10616 &mut reasons,
10617 OpaqueReason::DynamicReference,
10618 )?;
10619 }
10620 }
10621 for dep in &ingested.dep_plan.direct_cell_deps {
10622 self.target_preparation_checkpoint(options.deadline, 1)?;
10623 regions.push_back(PreparationRegion {
10624 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10625 sheet_id: dep.sheet_id,
10626 start_row: dep.coord.row().saturating_add(1),
10627 start_col: dep.coord.col().saturating_add(1),
10628 end_row: dep.coord.row().saturating_add(1),
10629 end_col: dep.coord.col().saturating_add(1),
10630 });
10631 }
10632 for range in &ingested.dep_plan.range_deps {
10633 self.target_preparation_checkpoint(options.deadline, 1)?;
10634 let Ok(dependency_sheet) =
10637 self.resolve_sheet_locator(&range.sheet, package.sheet_id)
10638 else {
10639 Self::widen_target_preparation(
10640 options.opaque_policy,
10641 &mut scope,
10642 &mut reasons,
10643 OpaqueReason::UnresolvedCrossSheetBinding,
10644 )?;
10645 continue;
10646 };
10647 regions.push_back(PreparationRegion {
10648 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10649 sheet_id: dependency_sheet,
10650 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10651 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10652 end_row: range
10653 .end_row
10654 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10655 bound.index + 1
10656 }),
10657 end_col: range
10658 .end_col
10659 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10660 bound.index + 1
10661 }),
10662 });
10663 }
10664 for name in ingested
10665 .dep_plan
10666 .resolved_named_refs
10667 .iter()
10668 .chain(&ingested.dep_plan.named_refs)
10669 {
10670 self.target_preparation_checkpoint(options.deadline, 1)?;
10671 if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
10672 symbol_vertices.push_back(entry.vertex);
10673 } else if self.graph.resolve_source_scalar_entry(name).is_none()
10674 && self.graph.resolve_source_table_entry(name).is_none()
10675 {
10676 Self::widen_target_preparation(
10677 options.opaque_policy,
10678 &mut scope,
10679 &mut reasons,
10680 OpaqueReason::UnresolvedName,
10681 )?;
10682 }
10683 }
10684 for table in &ingested.dep_plan.table_refs {
10685 self.target_preparation_checkpoint(options.deadline, 1)?;
10686 if let Some(entry) = self.graph.resolve_table_entry(table) {
10687 symbol_vertices.push_back(entry.vertex);
10688 } else if self.graph.resolve_source_table_entry(table).is_none() {
10689 Self::widen_target_preparation(
10690 options.opaque_policy,
10691 &mut scope,
10692 &mut reasons,
10693 OpaqueReason::UnresolvedTable,
10694 )?;
10695 }
10696 }
10697 package.legacy.push((
10698 record.row,
10699 record.col,
10700 ingested.ast_id,
10701 ingested.dep_plan,
10702 ));
10703 }
10704 prepared_packages.push(package);
10705 }
10706 let leases = self.staged_formula_index.leases_in_region(
10707 ®ion.sheet,
10708 region.start_row,
10709 region.start_col,
10710 region.end_row,
10711 region.end_col,
10712 );
10713 for lease in leases {
10714 self.target_preparation_checkpoint(options.deadline, 1)?;
10715 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
10716 ExcelError::new(ExcelErrorKind::Ref)
10717 .with_message(format!("staged formula sheet not found: {}", region.sheet))
10718 })?;
10719 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
10720 if !selected.insert(key) {
10721 continue;
10722 }
10723 let text = self
10724 .staged_formulas
10725 .get(®ion.sheet)
10726 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
10727 .ok_or_else(|| {
10728 ExcelError::new(ExcelErrorKind::Value)
10729 .with_message("staged formula index is stale")
10730 })?
10731 .to_string();
10732 let formula = if text.starts_with('=') {
10733 text.clone()
10734 } else {
10735 format!("={text}")
10736 };
10737 self.target_preparation_checkpoint(options.deadline, 1)?;
10738 let ast = match formualizer_parse::parser::parse(&formula) {
10739 Ok(ast) => ast,
10740 Err(error) => {
10741 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
10742 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
10743 format!(
10744 "Formula parse error at {}!{}{}: {error}",
10745 region.sheet,
10746 col_letters_from_1based(lease.col)
10747 .unwrap_or_else(|_| "?".to_string()),
10748 lease.row
10749 ),
10750 ));
10751 }
10752 pending_diagnostics.push(FormulaParseDiagnostic {
10753 sheet: region.sheet.clone(),
10754 row: lease.row,
10755 col: lease.col,
10756 formula: formula.clone(),
10757 message: error.to_string(),
10758 policy: self.config.formula_parse_policy,
10759 });
10760 match self.config.formula_parse_policy {
10761 FormulaParsePolicy::KeepCachedValue => {
10762 selected_cells.push(
10763 formualizer_common::RangeAddress::new(
10764 region.sheet.clone(),
10765 lease.row,
10766 lease.col,
10767 lease.row,
10768 lease.col,
10769 )
10770 .expect("selected staged coordinates are valid"),
10771 );
10772 prepared.push(PreparedOrdinaryStagedFormula {
10773 sheet: region.sheet.clone(),
10774 sheet_id,
10775 lease,
10776 ast_id: None,
10777 plan: None,
10778 });
10779 continue;
10780 }
10781 FormulaParsePolicy::AsText => ASTNode::new(
10782 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
10783 None,
10784 ),
10785 FormulaParsePolicy::CoerceToError => ASTNode::new(
10786 ASTNodeType::Literal(LiteralValue::Error(
10787 ExcelError::new(ExcelErrorKind::Error)
10788 .with_message(format!("Malformed formula: {error}")),
10789 )),
10790 None,
10791 ),
10792 FormulaParsePolicy::Strict => unreachable!(),
10793 }
10794 }
10795 };
10796 self.target_preparation_checkpoint(options.deadline, 1)?;
10797 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10798 self.target_preparation_checkpoint(options.deadline, 1)?;
10799 if let Some(reason) =
10800 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10801 {
10802 return Err(Self::preparation_stale(
10803 reason,
10804 "target planning snapshot became stale during discovery",
10805 ));
10806 }
10807 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10808 if reason == OpaqueReason::DynamicReference
10809 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10810 {
10811 Self::widen_target_preparation_to_sheet(
10812 options.opaque_policy,
10813 &mut scope,
10814 &mut reasons,
10815 reason,
10816 ®ion.sheet,
10817 )?;
10818 } else {
10819 Self::widen_target_preparation(
10820 options.opaque_policy,
10821 &mut scope,
10822 &mut reasons,
10823 reason,
10824 )?;
10825 }
10826 }
10827 let proven_sheet_local_dynamic =
10828 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10829 let placement = CellRef::new(
10830 sheet_id,
10831 Coord::from_excel(lease.row, lease.col, true, true),
10832 );
10833 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
10834 FormulaAstInput::Tree(ast),
10835 placement,
10836 Some(Arc::from(formula)),
10837 )?;
10838 self.target_preparation_checkpoint(options.deadline, 1)?;
10839 if ingested.dep_plan.dynamic {
10840 if proven_sheet_local_dynamic {
10841 Self::widen_target_preparation_to_sheet(
10842 options.opaque_policy,
10843 &mut scope,
10844 &mut reasons,
10845 OpaqueReason::DynamicReference,
10846 ®ion.sheet,
10847 )?;
10848 } else {
10849 Self::widen_target_preparation(
10850 options.opaque_policy,
10851 &mut scope,
10852 &mut reasons,
10853 OpaqueReason::DynamicReference,
10854 )?;
10855 }
10856 }
10857 for dep in &ingested.dep_plan.direct_cell_deps {
10858 self.target_preparation_checkpoint(options.deadline, 1)?;
10859 regions.push_back(PreparationRegion {
10860 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10861 sheet_id: dep.sheet_id,
10862 start_row: dep.coord.row() + 1,
10863 start_col: dep.coord.col() + 1,
10864 end_row: dep.coord.row() + 1,
10865 end_col: dep.coord.col() + 1,
10866 });
10867 }
10868 for range in &ingested.dep_plan.range_deps {
10869 self.target_preparation_checkpoint(options.deadline, 1)?;
10870 let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
10872 else {
10873 Self::widen_target_preparation(
10874 options.opaque_policy,
10875 &mut scope,
10876 &mut reasons,
10877 OpaqueReason::UnresolvedCrossSheetBinding,
10878 )?;
10879 continue;
10880 };
10881 regions.push_back(PreparationRegion {
10882 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10883 sheet_id: dependency_sheet,
10884 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10885 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10886 end_row: range
10887 .end_row
10888 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10889 bound.index + 1
10890 }),
10891 end_col: range
10892 .end_col
10893 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10894 bound.index + 1
10895 }),
10896 });
10897 }
10898 for name in ingested
10899 .dep_plan
10900 .resolved_named_refs
10901 .iter()
10902 .chain(&ingested.dep_plan.named_refs)
10903 {
10904 self.target_preparation_checkpoint(options.deadline, 1)?;
10905 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
10906 symbol_vertices.push_back(entry.vertex);
10907 } else if self.graph.resolve_source_scalar_entry(name).is_none()
10908 && self.graph.resolve_source_table_entry(name).is_none()
10909 {
10910 Self::widen_target_preparation(
10911 options.opaque_policy,
10912 &mut scope,
10913 &mut reasons,
10914 OpaqueReason::UnresolvedName,
10915 )?;
10916 }
10917 }
10918 for table in &ingested.dep_plan.table_refs {
10919 self.target_preparation_checkpoint(options.deadline, 1)?;
10920 if let Some(entry) = self.graph.resolve_table_entry(table) {
10921 symbol_vertices.push_back(entry.vertex);
10922 } else if self.graph.resolve_source_table_entry(table).is_none() {
10923 Self::widen_target_preparation(
10924 options.opaque_policy,
10925 &mut scope,
10926 &mut reasons,
10927 OpaqueReason::UnresolvedTable,
10928 )?;
10929 }
10930 }
10931 selected_cells.push(
10932 formualizer_common::RangeAddress::new(
10933 region.sheet.clone(),
10934 lease.row,
10935 lease.col,
10936 lease.row,
10937 lease.col,
10938 )
10939 .expect("selected staged coordinates are valid"),
10940 );
10941 prepared.push(PreparedOrdinaryStagedFormula {
10942 sheet: region.sheet.clone(),
10943 sheet_id,
10944 lease,
10945 ast_id: Some(ingested.ast_id),
10946 plan: Some(ingested.dep_plan),
10947 });
10948 }
10949
10950 if indexed_query_sheets.insert(region.sheet_id) {
10951 self.graph.prepare_sheet_index_for_query(region.sheet_id);
10952 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
10953 .saturating_mul(32);
10954 self.reserve_graph_source_scratch(bytes)?;
10955 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
10956 }
10957 let spill_anchors = self.graph.spill_anchors_in_region(
10958 region.sheet_id,
10959 region.start_row - 1,
10960 region.start_col - 1,
10961 region.end_row - 1,
10962 region.end_col - 1,
10963 );
10964 for anchor in spill_anchors {
10965 self.target_preparation_checkpoint(options.deadline, 1)?;
10966 symbol_vertices.push_back(anchor);
10967 }
10968 let vertices = self.graph.vertices_in_region(
10969 region.sheet_id,
10970 region.start_row - 1,
10971 region.end_row - 1,
10972 region.start_col - 1,
10973 region.end_col - 1,
10974 );
10975 for vertex in vertices {
10976 self.target_preparation_checkpoint(options.deadline, 1)?;
10977 symbol_vertices.push_back(vertex);
10978 }
10979 }
10980
10981 #[cfg(test)]
10982 self.target_preparation_fault(
10983 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
10984 )?;
10985
10986 if package_encountered
10987 || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
10988 && !prepared.is_empty())
10989 {
10990 Self::widen_target_preparation(
10991 options.opaque_policy,
10992 &mut scope,
10993 &mut reasons,
10994 OpaqueReason::UnsupportedSourceSemantics,
10995 )?;
10996 self.target_preparation_checkpoint(options.deadline, 0)?;
10997 let selected_count = self.staged_formula_count();
10998 let selected_packages = self
10999 .staged_formulas
11000 .values()
11001 .filter_map(|staged| staged.deferred_package.as_ref())
11002 .map(|package| package.families.len() + package.partitioned_families.len())
11003 .sum();
11004 self.formula_parse_diagnostics.truncate(diagnostics_len);
11005 self.last_formula_ingest_report = report_len;
11006 #[cfg(test)]
11007 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11008 hook();
11009 }
11010 self.target_preparation_checkpoint(options.deadline, 0)?;
11011 #[cfg(test)]
11012 self.target_preparation_fault(
11013 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11014 )?;
11015 let current_revisions = self.preparation_revisions();
11016 if let Some(reason) = Self::preparation_revision_stale_reason(
11017 &assumptions,
11018 ¤t_revisions,
11019 &planning_requests,
11020 true,
11021 ) {
11022 return Err(Self::preparation_stale(
11023 reason,
11024 "target compatibility preparation plan is stale",
11025 ));
11026 }
11027 #[cfg(test)]
11028 self.target_preparation_fault(
11029 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11030 )?;
11031 let commit_work_before = self
11032 .active_resource_ledger
11033 .as_ref()
11034 .map_or(0, |ledger| ledger.snapshot().work_charged);
11035 let commit_started = crate::instant::FzInstant::now();
11036 self.build_graph_all_unobserved()?;
11037 let commit_window = commit_started.elapsed();
11038 let ledger_after = self
11039 .active_resource_ledger
11040 .as_ref()
11041 .map(|ledger| ledger.snapshot());
11042 let actual_commit_work = ledger_after
11043 .map_or(0, |snapshot| snapshot.work_charged)
11044 .saturating_sub(commit_work_before);
11045 let observed_scratch_bytes = ledger_after
11046 .map_or(0, |snapshot| snapshot.scratch_peak)
11047 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11048 let revisions = assumptions.clone();
11049 let report = PreparedTargetGraphReport {
11050 request_id: request_id.unwrap_or_default(),
11051 requested_targets: targets.len(),
11052 normalized_regions: visited_regions.len(),
11053 normalized_target_list: normalized,
11054 selected_staged_cells: selected_count,
11055 selected_source_families: selected_packages,
11056 retained_staged_cells: self.staged_formula_count(),
11057 selected_cells,
11058 retained_cells: Vec::new(),
11059 widened_scope: PrepareScope::Workbook,
11060 widening_reasons: reasons,
11061 revisions,
11062 commit_window,
11063 estimated_scratch_bytes: discovery_scratch_reserved
11064 .saturating_add((selected_count as u64).saturating_mul(256)),
11065 observed_scratch_bytes,
11066 estimated_commit_work: selected_count as u64,
11067 actual_commit_work,
11068 outcome: PreparationOutcome::CompatibilityPrepared,
11069 };
11070 self.observe_target_preparation_report(&report);
11071 return Ok(report);
11072 }
11073
11074 prepared.sort_by_key(|formula| formula.lease.insertion_order);
11075 for package in &prepared_packages {
11076 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11077 formualizer_common::RangeAddress::new(
11078 package.sheet.clone(),
11079 record.row,
11080 record.col,
11081 record.row,
11082 record.col,
11083 )
11084 .ok()
11085 }));
11086 }
11087 let (mut legacy_graph, mut planned_formula_count) =
11088 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11089 let placements = prepared_packages
11090 .iter_mut()
11091 .flat_map(|package| std::mem::take(&mut package.placements))
11092 .collect::<Vec<_>>();
11093 let formula_plane = if placements.is_empty() {
11094 None
11095 } else {
11096 match self.graph.formula_authority().prepare_formula_plane_append(
11097 placements,
11098 self.graph.data_store(),
11099 self.graph.sheet_reg(),
11100 ) {
11101 Ok(append) => Some(append),
11102 Err(error) => {
11103 let reason = format!("TargetFormulaPlaneAppend:{error}");
11104 for package in &mut prepared_packages {
11105 *package
11106 .source_report
11107 .fallback_reasons
11108 .entry(reason.clone())
11109 .or_default() += package.direct_families as u64;
11110 if self.config.formula_plane_mode
11111 == FormulaPlaneMode::AuthoritativeExperimental
11112 {
11113 self.materialize_target_package_direct_records(
11114 package,
11115 &assumptions,
11116 &mut planning_requests,
11117 options.deadline,
11118 )?;
11119 } else {
11120 package.direct_families = 0;
11121 package.direct_cells = 0;
11122 package.direct_fragments = 0;
11123 package.direct_complete_families = 0;
11124 package.direct_complete_cells = 0;
11125 package.direct_partition_families = 0;
11126 package.direct_partition_cells = 0;
11127 }
11128 }
11129 (legacy_graph, planned_formula_count) =
11130 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11131 None
11132 }
11133 }
11134 };
11135 let new_vertices = legacy_graph.new_vertex_count();
11136 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11137 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11138 })?;
11139 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11140 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11141 })?;
11142 let current = self.graph.baseline_stats();
11143 let final_vertices = current
11144 .graph_vertex_count
11145 .checked_add(new_vertices)
11146 .ok_or_else(|| {
11147 crate::engine::ResourceLedgerError::Exhausted(
11148 formualizer_common::ResourceExhaustionDetail {
11149 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11150 limit: u64::MAX,
11151 observed: u64::MAX,
11152 request_id,
11153 },
11154 )
11155 .into_excel_error()
11156 })?;
11157 let final_edges = current
11158 .graph_edge_count
11159 .checked_sub(removed_edges)
11160 .and_then(|count| count.checked_add(new_edges))
11161 .ok_or_else(|| {
11162 crate::engine::ResourceLedgerError::Exhausted(
11163 formualizer_common::ResourceExhaustionDetail {
11164 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11165 limit: u64::MAX,
11166 observed: u64::MAX,
11167 request_id,
11168 },
11169 )
11170 .into_excel_error()
11171 })?;
11172 #[cfg(test)]
11173 self.target_preparation_fault(
11174 crate::engine::target_preparation::TargetPreparationFault::Admission,
11175 )?;
11176 let resource = |reason, limit: u64, observed: u64| {
11177 crate::engine::ResourceLedgerError::Exhausted(
11178 formualizer_common::ResourceExhaustionDetail {
11179 reason,
11180 limit,
11181 observed,
11182 request_id,
11183 },
11184 )
11185 .into_excel_error()
11186 };
11187 let admission = crate::engine::resource_ledger::GraphAdmission {
11188 final_vertices,
11189 final_edges,
11190 materialization_cells: planned_formula_count as u64,
11191 added_vertices: new_vertices,
11192 added_edges: new_edges,
11193 };
11194 let materialized_bytes = admission
11195 .materialized_graph_bytes()
11196 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11197 if let Err(error) = self.preflight_graph_admission(admission) {
11198 if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11199 self.observe_target_admission_failure(detail.reason);
11200 }
11201 return Err(error);
11202 }
11203 let selected_package_records = prepared_packages
11204 .iter()
11205 .map(|package| package.replay_records.len() as u64)
11206 .sum::<u64>();
11207 let planned_working_bytes = (prepared.len() as u64)
11208 .saturating_add(selected_package_records)
11209 .saturating_mul(256)
11210 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11211 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11212 .saturating_add((selected.len() as u64).saturating_mul(96))
11213 .saturating_add(materialized_bytes);
11214 let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11215 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11216 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11217 self.observe_target_admission_failure(
11218 formualizer_common::ResourceExhaustionReason::ScratchMemory,
11219 );
11220 return Err(error);
11221 }
11222
11223 let estimated_commit_duration = std::time::Duration::from_nanos(
11224 (new_vertices as u64)
11225 .saturating_add(new_edges as u64)
11226 .saturating_add(prepared.len() as u64)
11227 .saturating_add(selected_package_records)
11228 .max(1)
11229 .saturating_mul(100),
11230 );
11231 if options.deadline.is_some_and(|deadline| {
11232 std::time::Instant::now()
11233 .checked_add(estimated_commit_duration)
11234 .is_none_or(|finish| finish > deadline)
11235 }) {
11236 self.observe_target_admission_failure(
11237 formualizer_common::ResourceExhaustionReason::Deadline,
11238 );
11239 return Err(resource(
11240 formualizer_common::ResourceExhaustionReason::Deadline,
11241 0,
11242 1,
11243 ));
11244 }
11245 #[cfg(test)]
11246 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11247 hook();
11248 }
11249 self.target_preparation_checkpoint(options.deadline, 0)?;
11250 #[cfg(test)]
11251 self.target_preparation_fault(
11252 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11253 )?;
11254 let current_revisions = self.preparation_revisions();
11255 let staged_leases_match = prepared.iter().all(|formula| {
11256 self.staged_formula_index
11257 .lease_matches(&formula.sheet, formula.lease)
11258 }) && prepared_packages.iter().all(|package| {
11259 self.staged_formula_index
11260 .package_lease_matches(&package.sheet, package.lease)
11261 });
11262 let stale_reason = Self::preparation_revision_stale_reason(
11263 &assumptions,
11264 ¤t_revisions,
11265 &planning_requests,
11266 staged_leases_match,
11267 );
11268 if let Some(reason) = stale_reason {
11269 return Err(Self::preparation_stale(
11270 reason,
11271 "target graph preparation plan is stale",
11272 ));
11273 }
11274 #[cfg(test)]
11275 self.target_preparation_fault(
11276 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11277 )?;
11278 self.graph
11279 .validate_prepared_legacy_graph_plan(&legacy_graph)
11280 .map_err(|error| {
11281 Self::preparation_stale(
11282 formualizer_common::PreparationStaleReason::Graph,
11283 format!("target graph preparation plan is stale: {error}"),
11284 )
11285 })?;
11286 if let Some(append) = formula_plane.as_ref() {
11287 self.graph
11288 .formula_authority()
11289 .validate_prepared_formula_plane_append(
11290 append,
11291 self.graph.data_store(),
11292 self.graph.sheet_reg(),
11293 )
11294 .map_err(|error| {
11295 Self::preparation_stale(
11296 formualizer_common::PreparationStaleReason::Authority,
11297 format!("target FormulaPlane preparation is stale: {error}"),
11298 )
11299 })?;
11300 }
11301 #[cfg(test)]
11302 self.target_preparation_fault(
11303 crate::engine::target_preparation::TargetPreparationFault::Reservation,
11304 )?;
11305 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
11306 self.formula_parse_diagnostics
11307 .try_reserve(pending_diagnostics.len())
11308 .map_err(|_| {
11309 resource(
11310 formualizer_common::ResourceExhaustionReason::Admission,
11311 pending_diagnostics.len() as u64,
11312 pending_diagnostics.len() as u64,
11313 )
11314 })?;
11315 self.target_preparation_checkpoint(options.deadline, 0)?;
11316 #[cfg(test)]
11317 self.target_preparation_fault(
11318 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
11319 )?;
11320
11321 let commit_started = crate::instant::FzInstant::now();
11322 let committed = self
11323 .graph
11324 .apply_prevalidated_legacy_graph_plan(legacy_graph);
11325 let plane_report =
11326 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11327 formula_plane.map(|append| {
11328 self.graph
11329 .formula_authority_mut()
11330 .apply_prevalidated_formula_plane_append(append)
11331 })
11332 } else {
11333 None
11334 };
11335 if let Some(report) = plane_report.as_ref() {
11336 self.graph.mark_formula_spans_dirty(
11337 report.spans.iter().copied(),
11338 WholeSpanDirtyReason::NewSpan,
11339 );
11340 }
11341 for formula in &prepared {
11342 let removed = self
11343 .staged_formulas
11344 .get_mut(&formula.sheet)
11345 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
11346 debug_assert!(removed.is_some());
11347 let index_removed = self.staged_formula_index.remove(
11348 &formula.sheet,
11349 formula.lease.row,
11350 formula.lease.col,
11351 );
11352 debug_assert!(index_removed);
11353 }
11354 for package in &prepared_packages {
11355 let removed = self
11356 .staged_formulas
11357 .get_mut(&package.sheet)
11358 .and_then(|staged| staged.deferred_package.take());
11359 debug_assert!(removed.is_some());
11360 self.staged_formula_index.set_package(&package.sheet, None);
11361 }
11362 let empty_sheets = self
11363 .staged_formulas
11364 .iter()
11365 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
11366 .collect::<Vec<_>>();
11367 for sheet in empty_sheets {
11368 self.staged_formulas.remove(&sheet);
11369 }
11370 if committed > 0 || plane_report.is_some() {
11371 self.mark_topology_edited();
11372 }
11373 self.formula_parse_diagnostics.extend(pending_diagnostics);
11374 if !prepared.is_empty() || !prepared_packages.is_empty() {
11375 let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
11376 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
11377 prepared_packages
11378 .iter()
11379 .map(|package| package.replay_records.len() as u64)
11380 .sum::<u64>(),
11381 );
11382 ingest_delta.graph_formula_cells_materialized = committed as u64;
11383 ingest_delta.graph_vertices_created = new_vertices as u64;
11384 ingest_delta.graph_edges_created = new_edges as u64;
11385 for package in &prepared_packages {
11386 let source = &package.source_report;
11387 ingest_delta.source_formula_events = ingest_delta
11388 .source_formula_events
11389 .saturating_add(source.source_formula_events);
11390 ingest_delta.source_formula_records_spooled = ingest_delta
11391 .source_formula_records_spooled
11392 .saturating_add(source.source_formula_records_spooled);
11393 ingest_delta.source_spool_encoded_bytes = ingest_delta
11394 .source_spool_encoded_bytes
11395 .saturating_add(source.source_spool_encoded_bytes);
11396 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
11397 .source_spool_peak_memory_bytes
11398 .max(source.source_spool_peak_memory_bytes);
11399 ingest_delta.source_spool_spilled_bytes = ingest_delta
11400 .source_spool_spilled_bytes
11401 .saturating_add(source.source_spool_spilled_bytes);
11402 ingest_delta.source_spool_spill_files = ingest_delta
11403 .source_spool_spill_files
11404 .saturating_add(source.source_spool_spill_files);
11405 ingest_delta.source_spool_replays = ingest_delta
11406 .source_spool_replays
11407 .saturating_add(source.source_spool_replays)
11408 .saturating_add(1);
11409 ingest_delta.source_families_seen = ingest_delta
11410 .source_families_seen
11411 .saturating_add(source.families_seen);
11412 ingest_delta.source_family_cells_seen = ingest_delta
11413 .source_family_cells_seen
11414 .saturating_add(source.family_cells_seen);
11415 ingest_delta.source_family_shadow_eligible = ingest_delta
11416 .source_family_shadow_eligible
11417 .saturating_add(source.source_clean_families);
11418 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
11419 .source_family_shadow_eligible_cells
11420 .saturating_add(source.source_clean_cells);
11421 ingest_delta.source_partitioned_families_seen = ingest_delta
11422 .source_partitioned_families_seen
11423 .saturating_add(source.source_fragmentable_families);
11424 ingest_delta.source_partition_holes = ingest_delta
11425 .source_partition_holes
11426 .saturating_add(source.source_hole_exclusions);
11427 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
11428 .source_partition_ordinary_exceptions
11429 .saturating_add(source.source_ordinary_exclusions);
11430 ingest_delta.source_partition_surviving_cells = ingest_delta
11431 .source_partition_surviving_cells
11432 .saturating_add(source.source_fragmentable_cells);
11433 for (reason, count) in &source.fallback_reasons {
11434 let total = ingest_delta
11435 .fallback_reasons
11436 .entry(reason.clone())
11437 .or_default();
11438 *total = total.saturating_add(*count);
11439 }
11440 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
11441 ingest_delta.source_family_fallback = ingest_delta
11442 .source_family_fallback
11443 .saturating_add(source.families_seen);
11444 ingest_delta.source_family_fallback_cells = ingest_delta
11445 .source_family_fallback_cells
11446 .saturating_add(source.family_cells_seen);
11447 } else {
11448 ingest_delta.shadow_candidate_cells = ingest_delta
11449 .shadow_candidate_cells
11450 .saturating_add(source.family_cells_seen);
11451 ingest_delta.shadow_accepted_span_cells = ingest_delta
11452 .shadow_accepted_span_cells
11453 .saturating_add(package.direct_cells);
11454 ingest_delta.shadow_fallback_cells =
11455 ingest_delta.shadow_fallback_cells.saturating_add(
11456 source
11457 .family_cells_seen
11458 .saturating_sub(package.direct_cells),
11459 );
11460 ingest_delta.source_anchor_parses = ingest_delta
11461 .source_anchor_parses
11462 .saturating_add(package.anchor_parses);
11463 ingest_delta.source_anchor_asts = ingest_delta
11464 .source_anchor_asts
11465 .saturating_add(package.anchor_asts);
11466 ingest_delta.source_anchor_analyses = ingest_delta
11467 .source_anchor_analyses
11468 .saturating_add(package.anchor_analyses);
11469 ingest_delta.source_compressed_families_prepared = ingest_delta
11470 .source_compressed_families_prepared
11471 .saturating_add(package.direct_complete_families);
11472 ingest_delta.source_compressed_cells_prepared = ingest_delta
11473 .source_compressed_cells_prepared
11474 .saturating_add(package.direct_complete_cells);
11475 ingest_delta.source_partitioned_families_prepared = ingest_delta
11476 .source_partitioned_families_prepared
11477 .saturating_add(package.direct_partition_families);
11478 ingest_delta.source_partition_fragments_prepared = ingest_delta
11479 .source_partition_fragments_prepared
11480 .saturating_add(package.direct_fragments);
11481 ingest_delta.source_partition_span_cells_prepared = ingest_delta
11482 .source_partition_span_cells_prepared
11483 .saturating_add(package.direct_partition_cells);
11484 ingest_delta.source_partition_analyses_reused = ingest_delta
11485 .source_partition_analyses_reused
11486 .saturating_add(
11487 package
11488 .direct_fragments
11489 .saturating_sub(package.direct_partition_families),
11490 );
11491 ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
11492 .graph_formula_vertices_avoided_shadow
11493 .saturating_add(package.direct_cells);
11494 ingest_delta.ast_roots_avoided_shadow =
11495 ingest_delta.ast_roots_avoided_shadow.saturating_add(
11496 package
11497 .direct_complete_cells
11498 .saturating_sub(package.direct_complete_families)
11499 .saturating_add(
11500 package
11501 .direct_partition_cells
11502 .saturating_sub(package.direct_fragments),
11503 ),
11504 );
11505 ingest_delta.edge_rows_avoided_shadow = ingest_delta
11506 .edge_rows_avoided_shadow
11507 .saturating_add(package.direct_cells);
11508 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11509 {
11510 ingest_delta.source_family_promoted = ingest_delta
11511 .source_family_promoted
11512 .saturating_add(package.direct_families as u64);
11513 ingest_delta.source_family_promoted_cells = ingest_delta
11514 .source_family_promoted_cells
11515 .saturating_add(package.direct_cells);
11516 let descendants = package
11517 .direct_complete_cells
11518 .saturating_sub(package.direct_complete_families)
11519 .saturating_add(
11520 package
11521 .direct_partition_cells
11522 .saturating_sub(package.direct_partition_families),
11523 );
11524 ingest_delta.source_descendant_strings_avoided = ingest_delta
11525 .source_descendant_strings_avoided
11526 .saturating_add(descendants);
11527 ingest_delta.source_descendant_events_avoided = ingest_delta
11528 .source_descendant_events_avoided
11529 .saturating_add(descendants);
11530 ingest_delta.source_descendant_analyses_avoided = ingest_delta
11531 .source_descendant_analyses_avoided
11532 .saturating_add(descendants);
11533 }
11534 ingest_delta.source_family_fallback =
11535 ingest_delta.source_family_fallback.saturating_add(
11536 source
11537 .families_seen
11538 .saturating_sub(package.direct_families as u64),
11539 );
11540 ingest_delta.source_family_fallback_cells =
11541 ingest_delta.source_family_fallback_cells.saturating_add(
11542 source
11543 .family_cells_seen
11544 .saturating_sub(package.direct_cells),
11545 );
11546 }
11547 }
11548 if let Some(report) = plane_report.as_ref() {
11549 ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
11550 ingest_delta.shadow_spans_created = report.spans.len() as u64;
11551 }
11552 self.record_formula_ingest_report(ingest_delta);
11553 }
11554 let commit_window = commit_started.elapsed();
11555 let retained_cells = self
11556 .staged_formula_index
11557 .all_leases()
11558 .into_iter()
11559 .filter_map(|(sheet, lease)| {
11560 formualizer_common::RangeAddress::new(
11561 sheet, lease.row, lease.col, lease.row, lease.col,
11562 )
11563 .ok()
11564 })
11565 .collect::<Vec<_>>();
11566 let observed_scratch_bytes = self
11567 .active_resource_ledger
11568 .as_ref()
11569 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
11570 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11571 let committed_spans = plane_report
11572 .as_ref()
11573 .map_or(0, |report| report.spans.len() as u64);
11574 let selected_source_families = prepared_packages
11575 .iter()
11576 .map(|package| package.lease.family_count)
11577 .sum::<usize>();
11578 let selected_staged_cells = prepared
11579 .len()
11580 .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
11581 let actual_commit_work = (new_vertices as u64)
11582 .saturating_add(new_edges as u64)
11583 .saturating_add(committed as u64)
11584 .saturating_add(committed_spans)
11585 .saturating_add(prepared.len() as u64)
11586 .saturating_add(prepared_packages.len() as u64);
11587 let report = PreparedTargetGraphReport {
11588 request_id: request_id.unwrap_or_default(),
11589 requested_targets: targets.len(),
11590 normalized_regions: visited_regions.len(),
11591 normalized_target_list: normalized,
11592 selected_staged_cells,
11593 selected_source_families,
11594 retained_staged_cells: self.staged_formula_count(),
11595 selected_cells,
11596 retained_cells,
11597 widened_scope: scope,
11598 widening_reasons: reasons,
11599 revisions: assumptions,
11600 commit_window,
11601 estimated_scratch_bytes: scratch_bytes,
11602 observed_scratch_bytes,
11603 estimated_commit_work: (new_vertices as u64)
11604 .saturating_add(new_edges as u64)
11605 .saturating_add(prepared.len() as u64)
11606 .saturating_add(selected_package_records),
11607 actual_commit_work,
11608 outcome: PreparationOutcome::Prepared,
11609 };
11610 self.observe_target_preparation_report(&report);
11611 Ok(report)
11612 }
11613
11614 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11616 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
11617 engine.build_graph_all_unobserved()
11618 })
11619 }
11620
11621 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11622 let selected = self.staged_formula_count();
11623 let started = crate::instant::FzInstant::now();
11624 self.resource_checkpoint(selected as u64)?;
11625 let scratch_bytes = (selected as u64).saturating_mul(256);
11626 let result = self.with_request_scratch(scratch_bytes, |engine| {
11627 let index_snapshot = engine.staged_formula_index.clone();
11628 let collected = std::mem::take(&mut engine.staged_formulas)
11629 .into_iter()
11630 .collect();
11631 engine.staged_formula_index.clear_all();
11632 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
11633 });
11634 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11635 result
11636 }
11637
11638 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
11640 &mut self,
11641 sheets: I,
11642 ) -> Result<(), formualizer_parse::ExcelError> {
11643 let mut sheets = sheets.into_iter();
11644 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
11645 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
11646 engine.with_request_scratch(name_scratch, |engine| {
11647 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
11650 engine.charge_bounded_work(names.len() as u64)?;
11651 engine.build_graph_for_sheet_names_unobserved(names)
11652 })
11653 })
11654 }
11655
11656 fn build_graph_for_sheet_names_unobserved(
11657 &mut self,
11658 sheets: Vec<String>,
11659 ) -> Result<(), formualizer_parse::ExcelError> {
11660 let started = crate::instant::FzInstant::now();
11661 let selected = sheets
11662 .iter()
11663 .filter_map(|sheet| self.staged_formulas.get(sheet))
11664 .map(StagedSheet::len)
11665 .sum::<usize>();
11666 self.resource_checkpoint(selected as u64)?;
11667 let scratch_bytes = (selected as u64).saturating_mul(256);
11668 self.reserve_request_scratch(scratch_bytes)?;
11669 let index_snapshot = self.staged_formula_index.clone();
11670 let mut collected = Vec::new();
11671 for sheet in sheets {
11672 if let Some(staged) = self.staged_formulas.remove(&sheet) {
11673 self.index_removed_staged_sheet(&sheet, &staged);
11674 collected.push((sheet, staged));
11675 }
11676 }
11677 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
11678 self.release_request_scratch(scratch_bytes);
11679 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11680 result
11681 }
11682
11683 fn build_graph_from_staged_batches(
11684 &mut self,
11685 collected: StagedFormulaBatches,
11686 share_parse_cache_across_sheets: bool,
11687 staged_index_snapshot: StagedFormulaIndex,
11688 ) -> Result<(), formualizer_parse::ExcelError> {
11689 if collected.is_empty() {
11690 return Ok(());
11691 }
11692 for (sheet, _) in &collected {
11693 let _ = self.add_sheet(sheet);
11694 }
11695
11696 let diagnostics_len = self.formula_parse_diagnostics.len();
11697 let mut collected = collected;
11698 let prepared = match self
11699 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
11700 {
11701 Ok(prepared) => prepared,
11702 Err(error) => {
11703 self.formula_parse_diagnostics.truncate(diagnostics_len);
11704 for (sheet, staged) in collected {
11705 self.restore_staged_sheet(sheet, staged);
11706 }
11707 self.staged_formula_index = staged_index_snapshot;
11708 return Err(error);
11709 }
11710 };
11711 let (ordinary, compressed, direct) = prepared;
11712
11713 if !ordinary.is_empty()
11716 && let Err(error) = self.ingest_formula_batches(ordinary)
11717 {
11718 self.formula_parse_diagnostics.truncate(diagnostics_len);
11719 for (sheet, staged) in collected {
11720 self.restore_staged_sheet(sheet, staged);
11721 }
11722 self.staged_formula_index = staged_index_snapshot;
11723 return Err(error);
11724 }
11725 if !compressed.is_empty() {
11726 let _ = self.ingest_compressed_formula_source_batches(compressed)?;
11727 }
11728 if !direct.is_empty() {
11729 let _ = self.finish_compressed_formula_sources(direct)?;
11730 }
11731 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
11732 Ok(())
11733 }
11734
11735 fn prepare_staged_formula_batches(
11736 &mut self,
11737 collected: &mut StagedFormulaBatches,
11738 share_parse_cache_across_sheets: bool,
11739 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
11740 let mut ordinary = Vec::new();
11741 let mut compressed = Vec::new();
11742 let mut direct = Vec::new();
11743 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
11744 rustc_hash::FxHashMap::default();
11745 cache.reserve(4096);
11746
11747 for (sheet, staged) in collected {
11748 if !share_parse_cache_across_sheets {
11749 cache.clear();
11750 }
11751 let mut entries: Vec<_> = staged
11752 .entries
11753 .iter()
11754 .cloned()
11755 .map(|(row, col, text)| (row, col, text, None))
11756 .collect();
11757 let mut deferred_source = None;
11758 let mut deferred_fallback = None;
11759 if let Some(package) = staged.deferred_package.as_mut() {
11760 if package.sheet_name != *sheet {
11761 return Err(ExcelError::new(ExcelErrorKind::Value)
11762 .with_message("deferred formula package sheet mismatch"));
11763 }
11764 let eligible: Vec<_> = package
11765 .families
11766 .iter()
11767 .filter(|family| !package.invalidated.contains(&family.source_id))
11768 .cloned()
11769 .collect();
11770 let eligible_partitions: Vec<_> = package
11771 .partitioned_families
11772 .iter()
11773 .filter(|family| !package.invalidated.contains(&family.source_id))
11774 .cloned()
11775 .collect();
11776 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11777 let mut preparation = self.prepare_source_formula_proposals(
11778 sheet,
11779 &eligible,
11780 &eligible_partitions,
11781 &package.partitioned_families,
11782 package.report.source_formula_records_spooled,
11783 Arc::clone(&package.replay),
11784 &package.suppressed,
11785 )?;
11786 let replay_records = std::mem::take(&mut preparation.eager_replay);
11787 let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
11788 let accepted: BTreeMap<_, _> = preparation
11789 .fragmented
11790 .iter()
11791 .map(|fragment| (fragment.source.source_id, &fragment.source))
11792 .collect();
11793 replay_records.into_iter().partition(|record| {
11794 record.partition_owner.is_some_and(|owner| {
11795 accepted.get(&owner).is_some_and(|source| {
11796 Self::exact_replay_record_is_prepared_partition_legacy(
11797 source, record,
11798 )
11799 })
11800 })
11801 })
11802 };
11803 preparation.eager_replay = fragment_legacy;
11804 entries.extend(ordered_fallback.into_iter().map(|record| {
11805 (
11806 record.row,
11807 record.col,
11808 record.text,
11809 Some((record.source_order, record.family, record.partition_owner)),
11810 )
11811 }));
11812 deferred_source = Some((package.report.clone(), preparation));
11813 } else {
11814 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
11815 for partition in &eligible_partitions {
11816 replay_disposition
11817 .register_partition(partition, false)
11818 .map_err(|reason| {
11819 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
11820 })?;
11821 }
11822 replay_disposition
11823 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
11824 let replayed = package
11825 .replay
11826 .lock()
11827 .map_err(|_| {
11828 ExcelError::new(ExcelErrorKind::Value)
11829 .with_message("deferred formula spool lock poisoned")
11830 })?
11831 .replay(&replay_disposition)
11832 .map_err(|message| {
11833 ExcelError::new(ExcelErrorKind::Value).with_message(message)
11834 })?;
11835 entries.extend(replayed.into_iter().map(|record| {
11836 (
11837 record.row,
11838 record.col,
11839 record.text,
11840 Some((record.source_order, record.family, record.partition_owner)),
11841 )
11842 }));
11843 let mut report = package.report.clone();
11844 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
11845 deferred_fallback =
11846 Some((report, package.families.clone(), eligible_partitions));
11847 }
11848 }
11849
11850 let mut formulas = Vec::new();
11851 let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
11852 for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
11853 let key = if txt.starts_with('=') {
11854 txt
11855 } else {
11856 format!("={txt}")
11857 };
11858 let ast_id = if let Some(cached) = cache.get(&key) {
11859 *cached
11860 } else {
11861 let parsed = match formualizer_parse::parser::parse(&key) {
11862 Ok(parsed) => Some(parsed),
11863 Err(error) => self.handle_formula_parse_error(
11864 sheet,
11865 row,
11866 col,
11867 &key,
11868 error.to_string(),
11869 )?,
11870 };
11871 let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
11872 cache.insert(key.clone(), ast_id);
11873 ast_id
11874 };
11875
11876 if let Some(ast_id) = ast_id {
11877 let mut formula = FormulaIngestRecord::new(
11878 row,
11879 col,
11880 ast_id,
11881 Some(Arc::<str>::from(key.clone())),
11882 );
11883 if let Some((order, family, owner)) = source_proof {
11884 formula = formula.with_source_proof(order, family, owner);
11885 } else if deferred_source.is_some() {
11886 formula = formula.with_source_proof(
11887 crate::engine::SourceFormulaOrder::new(
11888 staged_order_base.saturating_add(entry_index as u64),
11889 ),
11890 None,
11891 None,
11892 );
11893 }
11894 formulas.push(formula);
11895 }
11896 }
11897
11898 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
11899 if let Some((report, preparation)) = deferred_source {
11900 direct.push((batch, report, preparation));
11901 } else if let Some((report, families, partitions)) = deferred_fallback {
11902 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
11903 batch.sheet_name.clone(),
11904 report,
11905 families,
11906 partitions,
11907 );
11908 compressed.push((batch, source_batch));
11909 } else if !batch.is_empty() {
11910 ordinary.push(batch);
11911 }
11912 }
11913 Ok((ordinary, compressed, direct))
11914 }
11915
11916 pub fn begin_bulk_ingest_arrow(
11918 &mut self,
11919 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
11920 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
11921 }
11922
11923 pub fn begin_bulk_update_arrow(
11925 &mut self,
11926 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
11927 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
11928 }
11929
11930 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
11931 self.graph.sheet_id(sheet).ok_or(
11932 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11933 name: sheet.to_string(),
11934 reason: "Unknown sheet".to_string(),
11935 },
11936 )
11937 }
11938
11939 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
11940 if row_1based == 0 {
11941 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11942 }
11943 Ok(row_1based - 1)
11944 }
11945
11946 fn normalize_row_range_1based(
11947 start_row_1based: u32,
11948 end_row_1based: u32,
11949 ) -> Result<(u32, u32), crate::engine::EditorError> {
11950 if start_row_1based == 0 || end_row_1based == 0 {
11951 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11952 }
11953 if start_row_1based > end_row_1based {
11954 return Err(crate::engine::EditorError::TransactionFailed {
11955 reason: "Row range start is greater than end".to_string(),
11956 });
11957 }
11958 Ok((start_row_1based - 1, end_row_1based - 1))
11959 }
11960
11961 fn invalidate_row_visibility_mask_cache(&self) {
11962 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11963 cache.clear();
11964 }
11965 }
11966
11967 fn set_row_hidden_by_sheet_id(
11968 &mut self,
11969 sheet_id: SheetId,
11970 row0: u32,
11971 hidden: bool,
11972 source: RowVisibilitySource,
11973 ) -> bool {
11974 let changed = {
11975 let state = self.row_visibility.entry(sheet_id).or_default();
11976 state.set_row_hidden(row0, hidden, source)
11977 };
11978
11979 let remove_entry = self
11980 .row_visibility
11981 .get(&sheet_id)
11982 .map(|state| state.is_empty())
11983 .unwrap_or(false);
11984 if remove_entry {
11985 self.row_visibility.remove(&sheet_id);
11986 }
11987
11988 if changed {
11989 self.invalidate_row_visibility_mask_cache();
11990 }
11991
11992 changed
11993 }
11994
11995 fn set_rows_hidden_by_sheet_id(
11996 &mut self,
11997 sheet_id: SheetId,
11998 start_row0: u32,
11999 end_row0: u32,
12000 hidden: bool,
12001 source: RowVisibilitySource,
12002 ) -> bool {
12003 let changed = {
12004 let state = self.row_visibility.entry(sheet_id).or_default();
12005 state.set_rows_hidden(start_row0, end_row0, hidden, source)
12006 };
12007
12008 let remove_entry = self
12009 .row_visibility
12010 .get(&sheet_id)
12011 .map(|state| state.is_empty())
12012 .unwrap_or(false);
12013 if remove_entry {
12014 self.row_visibility.remove(&sheet_id);
12015 }
12016
12017 if changed {
12018 self.invalidate_row_visibility_mask_cache();
12019 }
12020
12021 changed
12022 }
12023
12024 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
12025 if count == 0 {
12026 return;
12027 }
12028 let mut changed = false;
12029 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12030 changed = state.insert_rows(before0, count);
12031 state.is_empty()
12032 } else {
12033 false
12034 };
12035 if remove_entry {
12036 self.row_visibility.remove(&sheet_id);
12037 }
12038 if changed {
12039 self.invalidate_row_visibility_mask_cache();
12040 }
12041 }
12042
12043 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
12044 if count == 0 {
12045 return;
12046 }
12047 let mut changed = false;
12048 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12049 changed = state.delete_rows(start0, count);
12050 state.is_empty()
12051 } else {
12052 false
12053 };
12054 if remove_entry {
12055 self.row_visibility.remove(&sheet_id);
12056 }
12057 if changed {
12058 self.invalidate_row_visibility_mask_cache();
12059 }
12060 }
12061
12062 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12063 if let crate::engine::ChangeEvent::SetRowVisibility {
12064 sheet_id,
12065 row0,
12066 source,
12067 old_hidden,
12068 ..
12069 } = event
12070 {
12071 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12072 }
12073 }
12074
12075 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12076 if let crate::engine::ChangeEvent::SetRowVisibility {
12077 sheet_id,
12078 row0,
12079 source,
12080 new_hidden,
12081 ..
12082 } = event
12083 {
12084 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12085 }
12086 }
12087
12088 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12089 for event in events.iter().rev() {
12090 self.apply_inverse_row_visibility_event(event);
12091 }
12092 }
12093
12094 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12095 for event in events {
12096 self.apply_forward_row_visibility_event(event);
12097 }
12098 }
12099
12100 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12101 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12102 sheet,
12103 row,
12104 col,
12105 old,
12106 ..
12107 } = event
12108 {
12109 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12110 }
12111 }
12112
12113 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12114 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12115 sheet,
12116 row,
12117 col,
12118 new,
12119 ..
12120 } = event
12121 {
12122 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12123 }
12124 }
12125
12126 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12129 match target {
12130 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12131 None => {
12132 self.clear_staged_formula_text(sheet, row, col);
12133 }
12134 }
12135 }
12136
12137 pub fn set_row_hidden(
12138 &mut self,
12139 sheet: &str,
12140 row_1based: u32,
12141 hidden: bool,
12142 source: RowVisibilitySource,
12143 ) -> Result<(), crate::engine::EditorError> {
12144 self.observe_function_semantic_epoch()
12145 .map_err(crate::engine::EditorError::Excel)?;
12146 self.observe_function_semantic_epoch()
12147 .map_err(crate::engine::EditorError::Excel)?;
12148 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12149 let row0 = Self::normalize_row_1based(row_1based)?;
12150 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12151 self.record_formula_plane_structural_change(StructuralScope::Region(
12152 Region::whole_row(sheet_id, row0),
12153 ));
12154 self.mark_data_edited();
12155 }
12156 Ok(())
12157 }
12158
12159 pub fn set_rows_hidden(
12160 &mut self,
12161 sheet: &str,
12162 start_row_1based: u32,
12163 end_row_1based: u32,
12164 hidden: bool,
12165 source: RowVisibilitySource,
12166 ) -> Result<(), crate::engine::EditorError> {
12167 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12168 let (start_row0, end_row0) =
12169 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12170 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12171 if start_row0 == end_row0 {
12172 self.record_formula_plane_structural_change(StructuralScope::Region(
12173 Region::whole_row(sheet_id, start_row0),
12174 ));
12175 } else {
12176 self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12177 }
12178 self.mark_data_edited();
12179 }
12180 Ok(())
12181 }
12182
12183 pub fn is_row_hidden(
12184 &self,
12185 sheet: &str,
12186 row_1based: u32,
12187 source: Option<RowVisibilitySource>,
12188 ) -> Option<bool> {
12189 let sheet_id = self.graph.sheet_id(sheet)?;
12190 let row0 = row_1based.checked_sub(1)?;
12191 Some(
12192 self.row_visibility
12193 .get(&sheet_id)
12194 .map(|state| state.is_row_hidden(row0, source))
12195 .unwrap_or(false),
12196 )
12197 }
12198
12199 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12200 let sheet_id = self.graph.sheet_id(sheet)?;
12201 Some(
12202 self.row_visibility
12203 .get(&sheet_id)
12204 .map(|state| state.version())
12205 .unwrap_or(0),
12206 )
12207 }
12208
12209 fn build_row_visibility_mask_for_view(
12210 &self,
12211 view: &RangeView<'_>,
12212 mode: VisibilityMaskMode,
12213 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12214 let sheet_rows = view.sheet().nrows as usize;
12215 if sheet_rows == 0 || view.start_row() >= sheet_rows {
12216 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12217 }
12218
12219 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12220 let start_row0 = view.start_row() as u32;
12221 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12222 let version = self
12223 .row_visibility
12224 .get(&sheet_id)
12225 .map(|state| state.version())
12226 .unwrap_or(0);
12227 let key = VisibilityMaskCacheKey {
12228 sheet_id,
12229 start_row0,
12230 end_row0,
12231 mode,
12232 version,
12233 };
12234
12235 if let Ok(cache) = self.row_visibility_mask_cache.read()
12236 && let Some(mask) = cache.get(&key)
12237 {
12238 #[cfg(test)]
12239 visibility_mask_test_hooks::inc_hit();
12240 return Some(mask.clone());
12241 }
12242
12243 #[cfg(test)]
12244 visibility_mask_test_hooks::inc_miss();
12245
12246 let state = self.row_visibility.get(&sheet_id);
12247 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12248 for row0 in start_row0..=end_row0 {
12249 let manual_hidden = state
12250 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12251 .unwrap_or(false);
12252 let filter_hidden = state
12253 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12254 .unwrap_or(false);
12255
12256 let include = match mode {
12257 VisibilityMaskMode::IncludeAll => true,
12258 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12259 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12260 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12261 !(manual_hidden || filter_hidden)
12262 }
12263 };
12264 out.push(include);
12265 }
12266
12267 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12268 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12269 const MAX_CACHE_ENTRIES: usize = 4096;
12270 if cache.len() >= MAX_CACHE_ENTRIES {
12271 cache.clear();
12272 #[cfg(test)]
12273 visibility_mask_test_hooks::inc_eviction();
12274 }
12275 cache.insert(key, mask.clone());
12276 }
12277
12278 Some(mask)
12279 }
12280
12281 fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
12282 match error {
12283 crate::engine::EditorError::Excel(error) => error,
12284 other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
12285 }
12286 }
12287
12288 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
12289 let changes =
12290 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
12291 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
12292 let provider_revision = self.resolver.planning_semantic_revision();
12293 let provider_changed = provider_revision != self.function_provider_revision_seen;
12294 if !global_changed && !provider_changed {
12295 return Ok(false);
12296 }
12297
12298 let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
12299 let sheets: BTreeSet<_> = self
12300 .graph
12301 .formula_authority()
12302 .plane
12303 .spans
12304 .active_spans()
12305 .filter_map(|span| {
12306 let Some(template) = self
12307 .graph
12308 .formula_authority()
12309 .plane
12310 .templates
12311 .get(span.template_id)
12312 else {
12313 return (provider_changed || global_changed && !changes.complete)
12314 .then_some(span.domain.sheet_id());
12315 };
12316 let Some(ast) = self
12317 .graph
12318 .data_store()
12319 .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
12320 else {
12321 return (provider_changed || global_changed && !changes.complete)
12322 .then_some(span.domain.sheet_id());
12323 };
12324 let global_affected = global_changed
12325 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
12326 let provider_affected = provider_changed && Self::ast_contains_function(&ast);
12327 (global_affected || provider_affected).then_some(span.domain.sheet_id())
12328 })
12329 .collect();
12330 let invalidated = !sheets.is_empty();
12331 for sheet_id in sheets {
12332 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
12333 .map_err(Self::editor_error_to_excel)?;
12334 }
12335 if global_changed && !changed.is_empty() || provider_changed {
12336 self.cached_static_schedule = None;
12337 }
12338 self.function_semantic_epoch_seen = changes.epoch;
12339 self.function_provider_revision_seen = provider_revision;
12340 Ok(invalidated)
12341 }
12342
12343 pub(crate) fn ast_uses_changed_function(
12344 ast: &ASTNode,
12345 changed: &BTreeSet<(String, String)>,
12346 ) -> bool {
12347 match &ast.node_type {
12348 ASTNodeType::Function { name, args } => {
12349 let normalized = name.to_uppercase();
12350 let mut spellings = vec![(String::new(), normalized.clone())];
12351 let mut stripped = normalized.as_str();
12352 loop {
12353 let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
12354 .iter()
12355 .find_map(|prefix| stripped.strip_prefix(prefix))
12356 else {
12357 break;
12358 };
12359 stripped = rest;
12360 spellings.push((String::new(), stripped.to_string()));
12361 }
12362 let resolved = crate::function_registry::resolve("", name);
12363 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
12364 || resolved.as_ref().is_some_and(|resolved| {
12365 changed.contains(&(
12366 resolved.namespace.clone(),
12367 resolved.canonical_name.clone(),
12368 ))
12369 });
12370 directly_changed
12371 || resolved.is_none()
12372 || args
12373 .iter()
12374 .any(|arg| Self::ast_uses_changed_function(arg, changed))
12375 }
12376 ASTNodeType::Call { callee, args } => {
12377 Self::ast_uses_changed_function(callee, changed)
12378 || args
12379 .iter()
12380 .any(|arg| Self::ast_uses_changed_function(arg, changed))
12381 }
12382 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
12383 ASTNodeType::BinaryOp { left, right, .. } => {
12384 Self::ast_uses_changed_function(left, changed)
12385 || Self::ast_uses_changed_function(right, changed)
12386 }
12387 ASTNodeType::Array(rows) => rows
12388 .iter()
12389 .flatten()
12390 .any(|node| Self::ast_uses_changed_function(node, changed)),
12391 _ => false,
12392 }
12393 }
12394
12395 fn ast_contains_function(ast: &ASTNode) -> bool {
12396 match &ast.node_type {
12397 ASTNodeType::Function { .. } => true,
12398 ASTNodeType::Call { callee, args } => {
12399 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
12400 }
12401 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
12402 ASTNodeType::BinaryOp { left, right, .. } => {
12403 Self::ast_contains_function(left) || Self::ast_contains_function(right)
12404 }
12405 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
12406 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
12407 }
12408 }
12409
12410 fn demote_span_containing_cell_for_write(
12411 &mut self,
12412 sheet_id: SheetId,
12413 row0: u32,
12414 col0: u32,
12415 ) -> Result<(), crate::engine::EditorError> {
12416 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12417 return Ok(());
12418 }
12419 let placement = PlacementCoord::new(sheet_id, row0, col0);
12420 let inside_active_span = self
12421 .graph
12422 .formula_authority()
12423 .plane
12424 .spans
12425 .find_at(placement)
12426 .is_some();
12427 if inside_active_span {
12428 self.demote_spans_preserving_computed_overlays(
12429 sheet_id,
12430 Region::point(sheet_id, row0, col0),
12431 )?;
12432 }
12433 Ok(())
12434 }
12435
12436 fn demote_spans_preserving_computed_overlays(
12437 &mut self,
12438 _sheet_id: SheetId,
12439 affected_region: Region,
12440 ) -> Result<(), crate::engine::EditorError> {
12441 self.demote_spans_for_structural_op_impl(None, affected_region, false)
12445 }
12446
12447 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
12448 Region::rows_from(sheet_id, start_row0)
12449 }
12450
12451 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
12452 Region::cols_from(sheet_id, start_col0)
12453 }
12454
12455 fn structural_dirty_region_for_domain(
12456 domain: &PlacementDomain,
12457 op: StructuralOp,
12458 ) -> Option<Region> {
12459 let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
12460 PlacementDomain::RowRun {
12461 sheet_id,
12462 row_start,
12463 row_end,
12464 col,
12465 } => (*sheet_id, *row_start, *row_end, *col, *col),
12466 PlacementDomain::ColRun {
12467 sheet_id,
12468 row,
12469 col_start,
12470 col_end,
12471 } => (*sheet_id, *row, *row, *col_start, *col_end),
12472 PlacementDomain::Rect {
12473 sheet_id,
12474 row_start,
12475 row_end,
12476 col_start,
12477 col_end,
12478 } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
12479 };
12480 match op {
12481 StructuralOp::InsertRows {
12482 sheet_id: edited,
12483 before,
12484 ..
12485 } if sheet_id == edited && row_end >= before => Some(Region::rect(
12486 sheet_id,
12487 row_start.max(before),
12488 row_end,
12489 col_start,
12490 col_end,
12491 )),
12492 StructuralOp::DeleteRows {
12493 sheet_id: edited,
12494 start,
12495 ..
12496 } if sheet_id == edited && row_end >= start => Some(Region::rect(
12497 sheet_id,
12498 row_start.max(start),
12499 row_end,
12500 col_start,
12501 col_end,
12502 )),
12503 StructuralOp::InsertColumns {
12504 sheet_id: edited,
12505 before,
12506 ..
12507 } if sheet_id == edited && col_end >= before => Some(Region::rect(
12508 sheet_id,
12509 row_start,
12510 row_end,
12511 col_start.max(before),
12512 col_end,
12513 )),
12514 StructuralOp::DeleteColumns {
12515 sheet_id: edited,
12516 start,
12517 ..
12518 } if sheet_id == edited && col_end >= start => Some(Region::rect(
12519 sheet_id,
12520 row_start,
12521 row_end,
12522 col_start.max(start),
12523 col_end,
12524 )),
12525 _ => None,
12526 }
12527 }
12528
12529 fn span_result_region_intersects_affected(
12530 span: &crate::formula_plane::runtime::FormulaSpan,
12531 affected_region: &Region,
12532 ) -> bool {
12533 Region::from_domain(span.result_region.domain()).intersects(affected_region)
12534 }
12535
12536 fn span_any_read_region_intersects_affected(
12537 plane: &FormulaPlane,
12538 span: &crate::formula_plane::runtime::FormulaSpan,
12539 affected_region: &Region,
12540 ) -> bool {
12541 span.read_summary_id
12542 .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
12543 .is_some_and(|summary| {
12544 summary
12545 .dependencies
12546 .iter()
12547 .any(|dependency| dependency.read_region.intersects(affected_region))
12548 })
12549 }
12550
12551 fn insert_formula_plane_dirty_coords_for_span(
12552 &self,
12553 span_ref: FormulaSpanRef,
12554 dirty: ProducerDirtyDomain,
12555 out: &mut FxHashSet<(SheetId, u32, u32)>,
12556 ) -> Result<(), crate::engine::EditorError> {
12557 let authority = self.graph.formula_authority();
12558 let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
12559 ExcelError::new(ExcelErrorKind::NImpl)
12560 .with_message("FormulaPlane dirty transfer referenced a stale span")
12561 })?;
12562 match dirty {
12563 ProducerDirtyDomain::Whole => {
12564 out.extend(
12565 span.domain
12566 .iter()
12567 .map(|coord| (coord.sheet_id, coord.row, coord.col)),
12568 );
12569 }
12570 ProducerDirtyDomain::Cells(cells) => {
12571 out.extend(cells.into_iter().filter_map(|key| {
12572 let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
12573 span.domain
12574 .contains(coord)
12575 .then_some((coord.sheet_id, coord.row, coord.col))
12576 }));
12577 }
12578 ProducerDirtyDomain::Regions(regions) => {
12579 out.extend(span.domain.iter().filter_map(|coord| {
12580 let key = crate::formula_plane::region_index::RegionKey::from(coord);
12581 regions
12582 .iter()
12583 .any(|region| region.contains_key(key))
12584 .then_some((coord.sheet_id, coord.row, coord.col))
12585 }));
12586 }
12587 }
12588 Ok(())
12589 }
12590
12591 fn compute_current_formula_plane_dirty_result_coords(
12592 &self,
12593 ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
12594 use crate::formula_plane::producer::compute_dirty_closure;
12595
12596 let authority = self.graph.formula_authority();
12597 let span_refs = authority.active_span_refs();
12598 let span_refs_by_id = span_refs
12599 .iter()
12600 .copied()
12601 .map(|span_ref| (span_ref.id, span_ref))
12602 .collect::<BTreeMap<_, _>>();
12603 let mut dirty_coords = FxHashSet::default();
12604
12605 for span_ref in self.graph.pending_formula_dirty_whole_spans() {
12606 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12607 self.insert_formula_plane_dirty_coords_for_span(
12608 span_ref,
12609 ProducerDirtyDomain::Whole,
12610 &mut dirty_coords,
12611 )?;
12612 }
12613 }
12614 for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
12615 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12616 self.insert_formula_plane_dirty_coords_for_span(
12617 span_ref,
12618 ProducerDirtyDomain::Regions(vec![region]),
12619 &mut dirty_coords,
12620 )?;
12621 }
12622 }
12623
12624 let pending_changed_regions = self
12625 .graph
12626 .pending_formula_dirty_regions()
12627 .collect::<Vec<_>>();
12628 if pending_changed_regions.is_empty() {
12629 return Ok(dirty_coords);
12630 }
12631
12632 let closure = compute_dirty_closure(
12633 &authority.consumer_reads,
12634 pending_changed_regions,
12635 |producer| authority.producer_results.producer_result_region(producer),
12636 );
12637 if closure.incomplete {
12638 for span_ref in span_refs {
12639 self.insert_formula_plane_dirty_coords_for_span(
12640 span_ref,
12641 ProducerDirtyDomain::Whole,
12642 &mut dirty_coords,
12643 )?;
12644 }
12645 return Ok(dirty_coords);
12646 }
12647 for work in closure.work {
12648 let FormulaProducerId::Span(span_id) = work.producer else {
12649 continue;
12650 };
12651 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12652 continue;
12653 };
12654 self.insert_formula_plane_dirty_coords_for_span(
12655 span_ref,
12656 work.dirty,
12657 &mut dirty_coords,
12658 )?;
12659 }
12660 for fallback in closure.fallbacks {
12661 let FormulaProducerId::Span(span_id) = fallback.consumer else {
12662 continue;
12663 };
12664 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12665 continue;
12666 };
12667 self.insert_formula_plane_dirty_coords_for_span(
12668 span_ref,
12669 ProducerDirtyDomain::Whole,
12670 &mut dirty_coords,
12671 )?;
12672 }
12673
12674 Ok(dirty_coords)
12675 }
12676
12677 fn demote_spans_for_structural_op(
12687 &mut self,
12688 op: StructuralOp,
12689 affected_region: Region,
12690 ) -> Result<(), crate::engine::EditorError> {
12691 if op.count() == 0 {
12692 return Ok(());
12693 }
12694 self.legacy_island_structural_summaries_trusted = false;
12697 self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
12698 }
12699
12700 fn indexed_structural_candidate_span_refs(
12701 &self,
12702 affected_region: Region,
12703 ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
12704 use crate::formula_plane::region_index::BoundedRegionQueryResult;
12705
12706 let authority = self.graph.formula_authority();
12707 if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
12708 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12709 .with_message("FormulaPlane structural candidate indexes are stale")
12710 .into());
12711 }
12712 let indexed_region = match affected_region.axis_ranges() {
12713 (
12714 crate::formula_plane::region_index::AxisRange::From(start),
12715 crate::formula_plane::region_index::AxisRange::All,
12716 ) => Region {
12717 sheet_id: affected_region.sheet_id(),
12718 rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12719 cols: crate::formula_plane::region_index::AxisRange::All,
12720 },
12721 (
12722 crate::formula_plane::region_index::AxisRange::All,
12723 crate::formula_plane::region_index::AxisRange::From(start),
12724 ) => Region {
12725 sheet_id: affected_region.sheet_id(),
12726 rows: crate::formula_plane::region_index::AxisRange::All,
12727 cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12728 },
12729 _ => affected_region,
12730 };
12731 let max_candidates = self.config.max_formula_plane_cache_candidates;
12732 let result_query = match authority
12733 .producer_results
12734 .query_bounded(indexed_region, max_candidates)
12735 {
12736 BoundedRegionQueryResult::Complete(result) => result,
12737 BoundedRegionQueryResult::Incomplete {
12738 observed_candidates,
12739 } => {
12740 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12741 .with_message(format!(
12742 "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
12743 ))
12744 .into());
12745 }
12746 };
12747 let remaining = max_candidates.saturating_sub(result_query.matches.len());
12748 let read_query = match authority
12749 .consumer_reads
12750 .query_changed_region_bounded(indexed_region, remaining)
12751 {
12752 BoundedRegionQueryResult::Complete(result) => result,
12753 BoundedRegionQueryResult::Incomplete { .. } => {
12754 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12755 .with_message("FormulaPlane structural read candidate limit exceeded")
12756 .into());
12757 }
12758 };
12759
12760 let mut ids = BTreeSet::new();
12761 for matched in result_query.matches {
12762 if let FormulaProducerId::Span(id) = matched.value.producer {
12763 ids.insert(id);
12764 }
12765 }
12766 for matched in read_query.matches {
12767 if let FormulaProducerId::Span(id) = matched.value.consumer {
12768 ids.insert(id);
12769 }
12770 }
12771
12772 ids.into_iter()
12773 .map(|id| {
12774 authority.plane.spans.current_ref(id).ok_or_else(|| {
12775 crate::engine::EditorError::Excel(
12776 ExcelError::new(ExcelErrorKind::NImpl)
12777 .with_message("FormulaPlane structural index referenced a stale span"),
12778 )
12779 })
12780 })
12781 .collect()
12782 }
12783
12784 fn demote_spans_for_structural_op_impl(
12785 &mut self,
12786 op: Option<StructuralOp>,
12787 affected_region: Region,
12788 clear_computed_overlays: bool,
12789 ) -> Result<(), crate::engine::EditorError> {
12790 struct SpanPlan {
12791 span_ref: FormulaSpanRef,
12792 sheet_id: SheetId,
12793 ast: ASTNode,
12794 origin_row: u32,
12795 origin_col: u32,
12796 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12797 placements: Vec<(u32, u32)>,
12798 }
12799
12800 fn substitute_literal_slots_for_template_placement(
12801 ast: &ASTNode,
12802 binding: &[LiteralValue],
12803 ) -> ASTNode {
12804 fn clone_with_slots(
12805 ast: &ASTNode,
12806 binding: &[LiteralValue],
12807 next: &mut usize,
12808 in_array: bool,
12809 ) -> ASTNode {
12810 let node_type = match &ast.node_type {
12811 ASTNodeType::Literal(_) if !in_array => {
12812 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
12813 *next = next.saturating_add(1);
12814 ASTNodeType::Literal(value)
12815 }
12816 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
12817 ASTNodeType::Omitted => ASTNodeType::Omitted,
12818 ASTNodeType::Reference {
12819 original,
12820 reference,
12821 } => ASTNodeType::Reference {
12822 original: original.clone(),
12823 reference: reference.clone(),
12824 },
12825 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
12826 op: op.clone(),
12827 expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
12828 },
12829 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
12830 op: op.clone(),
12831 left: Box::new(clone_with_slots(left, binding, next, in_array)),
12832 right: Box::new(clone_with_slots(right, binding, next, in_array)),
12833 },
12834 ASTNodeType::Function { name, args } => ASTNodeType::Function {
12835 name: name.clone(),
12836 args: args
12837 .iter()
12838 .map(|arg| clone_with_slots(arg, binding, next, in_array))
12839 .collect(),
12840 },
12841 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
12842 callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
12843 args: args
12844 .iter()
12845 .map(|arg| clone_with_slots(arg, binding, next, in_array))
12846 .collect(),
12847 },
12848 ASTNodeType::Array(rows) => ASTNodeType::Array(
12849 rows.iter()
12850 .map(|row| {
12851 row.iter()
12852 .map(|cell| clone_with_slots(cell, binding, next, true))
12853 .collect()
12854 })
12855 .collect(),
12856 ),
12857 };
12858 ASTNode::new(node_type, ast.source_token.clone())
12859 }
12860 let mut next = 0usize;
12861 clone_with_slots(ast, binding, &mut next, false)
12862 }
12863
12864 let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
12865 self.formula_plane_structural_span_candidates = self
12866 .formula_plane_structural_span_candidates
12867 .saturating_add(span_refs.len() as u64);
12868 if span_refs.is_empty() {
12869 return Ok(());
12870 }
12871 let dirty_span_coords = if clear_computed_overlays {
12872 FxHashSet::default()
12873 } else {
12874 self.compute_current_formula_plane_dirty_result_coords()?
12875 };
12876
12877 struct SpanTemplateRewrite {
12884 adjusted_ast: ASTNode,
12885 exact_canonical_key: Arc<str>,
12886 parameterized_canonical_key: Arc<str>,
12887 formula_text: Option<Arc<str>>,
12888 canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
12894 value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
12895 }
12896
12897 struct ShiftPlan {
12898 span_ref: FormulaSpanRef,
12899 template_id: crate::formula_plane::ids::FormulaTemplateId,
12900 new_origin_row: u32,
12901 new_origin_col: u32,
12902 new_domain: crate::formula_plane::runtime::PlacementDomain,
12903 new_read_summary: Option<SpanReadSummary>,
12904 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12905 force_binding_residual_axes: bool,
12906 rewrite: Option<SpanTemplateRewrite>,
12907 dirty_region: Option<Region>,
12908 }
12909
12910 fn shift_operation_for(
12913 op: StructuralOp,
12914 ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
12915 use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
12916 match op {
12917 StructuralOp::InsertRows {
12918 sheet_id,
12919 before,
12920 count,
12921 } => ShiftOperation::InsertRows {
12922 sheet_id,
12923 before,
12924 count,
12925 },
12926 StructuralOp::DeleteRows {
12927 sheet_id,
12928 start,
12929 count,
12930 } => ShiftOperation::DeleteRows {
12931 sheet_id,
12932 start,
12933 count,
12934 },
12935 StructuralOp::InsertColumns {
12936 sheet_id,
12937 before,
12938 count,
12939 } => ShiftOperation::InsertColumns {
12940 sheet_id,
12941 before,
12942 count,
12943 },
12944 StructuralOp::DeleteColumns {
12945 sheet_id,
12946 start,
12947 count,
12948 } => ShiftOperation::DeleteColumns {
12949 sheet_id,
12950 start,
12951 count,
12952 },
12953 }
12954 }
12955
12956 struct SplitPlan {
12961 span_ref: FormulaSpanRef,
12962 sheet_id: SheetId,
12963 template_id: crate::formula_plane::ids::FormulaTemplateId,
12964 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12965 is_constant_result: bool,
12966 upper_domain: crate::formula_plane::runtime::PlacementDomain,
12967 upper_read_summary: Option<SpanReadSummary>,
12968 lower_new_origin_row: u32,
12969 lower_new_origin_col: u32,
12970 lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
12971 lower_new_read_summary: Option<SpanReadSummary>,
12972 lower_force_binding_residual_axes: bool,
12973 lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
12974 upper_dirty_region: Option<Region>,
12975 lower_dirty_region: Option<Region>,
12976 }
12977
12978 fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
12979 u32::try_from(i64::from(value).checked_add(delta)?).ok()
12980 }
12981
12982 fn shifted_read_summary(
12993 read_summary: &SpanReadSummary,
12994 new_result_region: Region,
12995 op: StructuralOp,
12996 row_delta: i64,
12997 col_delta: i64,
12998 origin_row_delta: i64,
12999 origin_col_delta: i64,
13000 ) -> Option<SpanReadSummary> {
13001 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13002 for dependency in &read_summary.dependencies {
13003 let read_region = match op.classify_region(dependency.read_region) {
13004 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13005 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13006 dependency.read_region
13007 }
13008 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13009 ..
13010 } => dependency
13011 .read_region
13012 .project_through_axis_shift(row_delta, col_delta)?,
13013 crate::formula_plane::structural_shift::AxisShiftCase::Straddles
13014 | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13015 return None;
13016 }
13017 };
13018 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13019 read_region,
13020 projection: dependency
13021 .projection
13022 .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
13023 });
13024 }
13025 Some(SpanReadSummary {
13026 result_region: new_result_region,
13027 dependencies,
13028 })
13029 }
13030
13031 fn compact_axis_through_delete(
13032 min: u32,
13033 max: u32,
13034 start: u32,
13035 count: u32,
13036 ) -> Option<(u32, u32)> {
13037 let end = start.saturating_add(count);
13038 if max < start || min >= end {
13039 return Some((min.saturating_sub(count), max.saturating_sub(count)));
13040 }
13041 let keeps_left = min < start;
13042 let keeps_right = max >= end;
13043 match (keeps_left, keeps_right) {
13044 (false, false) => None,
13045 (true, false) => Some((min, start.checked_sub(1)?)),
13046 (false, true) => Some((start, max.checked_sub(count)?)),
13047 (true, true) => Some((min, max.checked_sub(count)?)),
13048 }
13049 }
13050
13051 fn compact_domain_through_delete(
13052 domain: &PlacementDomain,
13053 op: StructuralOp,
13054 ) -> Option<PlacementDomain> {
13055 match (domain, op) {
13056 (
13057 PlacementDomain::RowRun {
13058 sheet_id,
13059 row_start,
13060 row_end,
13061 col,
13062 },
13063 StructuralOp::DeleteRows { start, count, .. },
13064 ) => {
13065 let (row_start, row_end) =
13066 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13067 Some(PlacementDomain::row_run(
13068 *sheet_id, row_start, row_end, *col,
13069 ))
13070 }
13071 (
13072 PlacementDomain::Rect {
13073 sheet_id,
13074 row_start,
13075 row_end,
13076 col_start,
13077 col_end,
13078 },
13079 StructuralOp::DeleteRows { start, count, .. },
13080 ) => {
13081 let (row_start, row_end) =
13082 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13083 Some(PlacementDomain::rect(
13084 *sheet_id, row_start, row_end, *col_start, *col_end,
13085 ))
13086 }
13087 (
13088 PlacementDomain::ColRun {
13089 sheet_id,
13090 row,
13091 col_start,
13092 col_end,
13093 },
13094 StructuralOp::DeleteColumns { start, count, .. },
13095 ) => {
13096 let (col_start, col_end) =
13097 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13098 Some(PlacementDomain::col_run(
13099 *sheet_id, *row, col_start, col_end,
13100 ))
13101 }
13102 (
13103 PlacementDomain::Rect {
13104 sheet_id,
13105 row_start,
13106 row_end,
13107 col_start,
13108 col_end,
13109 },
13110 StructuralOp::DeleteColumns { start, count, .. },
13111 ) => {
13112 let (col_start, col_end) =
13113 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13114 Some(PlacementDomain::rect(
13115 *sheet_id, *row_start, *row_end, col_start, col_end,
13116 ))
13117 }
13118 _ => None,
13119 }
13120 }
13121
13122 fn compact_axis_range_through_delete(
13123 axis: crate::formula_plane::region_index::AxisRange,
13124 start: u32,
13125 count: u32,
13126 ) -> Option<crate::formula_plane::region_index::AxisRange> {
13127 use crate::formula_plane::region_index::AxisRange;
13128 match axis {
13129 AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13130 .map(|(point, _)| AxisRange::Point(point)),
13131 AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13132 .map(|(min, max)| AxisRange::Span(min, max)),
13133 AxisRange::All => Some(AxisRange::All),
13134 AxisRange::From(_) | AxisRange::To(_) => None,
13135 }
13136 }
13137
13138 fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13139 let (rows, cols) = region.axis_ranges();
13140 match op {
13141 StructuralOp::DeleteRows {
13142 sheet_id,
13143 start,
13144 count,
13145 } if region.sheet_id() == sheet_id => Some(Region {
13146 sheet_id,
13147 rows: compact_axis_range_through_delete(rows, start, count)?,
13148 cols,
13149 }),
13150 StructuralOp::DeleteColumns {
13151 sheet_id,
13152 start,
13153 count,
13154 } if region.sheet_id() == sheet_id => Some(Region {
13155 sheet_id,
13156 rows,
13157 cols: compact_axis_range_through_delete(cols, start, count)?,
13158 }),
13159 _ => Some(region),
13160 }
13161 }
13162
13163 fn compact_read_summary_through_delete(
13164 read_summary: &SpanReadSummary,
13165 new_result_region: Region,
13166 op: StructuralOp,
13167 ) -> Option<SpanReadSummary> {
13168 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13169 for dependency in &read_summary.dependencies {
13170 let read_region = match op.classify_region(dependency.read_region) {
13171 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13172 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13173 dependency.read_region
13174 }
13175 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13176 ..
13177 } => {
13178 let (row_delta, col_delta) = op.axis_shift_delta();
13179 dependency
13180 .read_region
13181 .project_through_axis_shift(row_delta, col_delta)?
13182 }
13183 crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13184 compact_region_through_delete(dependency.read_region, op)?
13185 }
13186 crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13187 return None;
13188 }
13189 };
13190 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13191 read_region,
13192 projection: dependency.projection,
13193 });
13194 }
13195 Some(SpanReadSummary {
13196 result_region: new_result_region,
13197 dependencies,
13198 })
13199 }
13200
13201 fn rederive_read_summary_for_result(
13208 read_summary: &SpanReadSummary,
13209 result_region: Region,
13210 ) -> Option<SpanReadSummary> {
13211 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13212 for dependency in &read_summary.dependencies {
13213 let sheet_id = dependency.read_region.sheet_id();
13214 let read_regions = dependency
13215 .projection
13216 .read_regions_for_result(sheet_id, result_region)
13217 .ok()?;
13218 for read_region in read_regions {
13219 let dependency = crate::formula_plane::producer::SpanReadDependency {
13220 read_region,
13221 projection: dependency.projection,
13222 };
13223 if !dependencies.contains(&dependency) {
13224 dependencies.push(dependency);
13225 }
13226 }
13227 }
13228 Some(SpanReadSummary {
13229 result_region,
13230 dependencies,
13231 })
13232 }
13233
13234 fn plan_span_split(
13241 authority: &crate::formula_plane::authority::FormulaAuthority,
13242 span: &crate::formula_plane::runtime::FormulaSpan,
13243 span_ref: FormulaSpanRef,
13244 read_summary: Option<&SpanReadSummary>,
13245 op: StructuralOp,
13246 ) -> Option<SplitPlan> {
13247 use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13248
13249 if span.intrinsic_mask_id.is_some() {
13250 return None;
13252 }
13253 let binding_set = span
13254 .binding_set_id
13255 .and_then(|id| authority.plane.binding_sets.get(id));
13256 if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13257 return None;
13261 }
13262
13263 let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13264 let upper_result_region = Region::from_domain(&upper_domain);
13265 let lower_result_region = Region::from_domain(&lower_domain);
13266 let (upper_read_summary, lower_read_summary) = match read_summary {
13267 Some(summary) => (
13268 Some(rederive_read_summary_for_result(
13269 summary,
13270 upper_result_region,
13271 )?),
13272 Some(rederive_read_summary_for_result(
13273 summary,
13274 lower_result_region,
13275 )?),
13276 ),
13277 None => (None, None),
13278 };
13279
13280 let upper_span = crate::formula_plane::runtime::FormulaSpan {
13284 domain: upper_domain.clone(),
13285 result_region: ResultRegion::scalar_cells(upper_domain.clone()),
13286 ..span.clone()
13287 };
13288 if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
13289 != SpanShiftPlan::NoOp
13290 {
13291 return None;
13292 }
13293 let lower_span = crate::formula_plane::runtime::FormulaSpan {
13294 domain: lower_domain.clone(),
13295 result_region: ResultRegion::scalar_cells(lower_domain.clone()),
13296 ..span.clone()
13297 };
13298 let SpanShiftPlan::Shift {
13299 row_delta,
13300 col_delta,
13301 origin_row_delta,
13302 origin_col_delta,
13303 rewrite_absolute_reads,
13304 } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
13305 else {
13306 return None;
13307 };
13308 if rewrite_absolute_reads {
13309 return None;
13315 }
13316
13317 let lower_new_origin_row =
13321 checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
13322 let lower_new_origin_col =
13323 checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
13324 let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
13325 let lower_new_result_region = Region::from_domain(&lower_new_domain);
13326 let lower_new_read_summary = match lower_read_summary.as_ref() {
13327 Some(summary) => Some(shifted_read_summary(
13328 summary,
13329 lower_new_result_region,
13330 op,
13331 row_delta,
13332 col_delta,
13333 origin_row_delta,
13334 origin_col_delta,
13335 )?),
13336 None => None,
13337 };
13338
13339 let mut lower_overlay_refs = Vec::new();
13344 for overlay_ref in authority
13345 .plane
13346 .formula_overlay
13347 .refs_for_source_span(span_ref)
13348 {
13349 let entry = authority.plane.formula_overlay.get(overlay_ref)?;
13350 match op.classify_region(Region::from_domain(&entry.domain)) {
13351 AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
13352 AxisShiftCase::EntirelyAboveShift { .. } => {
13353 lower_overlay_refs.push(overlay_ref);
13354 }
13355 AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
13356 return None;
13357 }
13358 }
13359 }
13360
13361 let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
13362 !binding_set.value_ref_slots.is_empty()
13363 && (origin_row_delta != 0 || origin_col_delta != 0)
13364 });
13365
13366 Some(SplitPlan {
13367 span_ref,
13368 sheet_id: span.sheet_id,
13369 template_id: span.template_id,
13370 binding_set_id: span.binding_set_id,
13371 is_constant_result: span.is_constant_result,
13372 upper_domain,
13373 upper_read_summary,
13374 lower_new_origin_row,
13375 lower_new_origin_col,
13376 lower_new_domain,
13377 lower_new_read_summary,
13378 lower_force_binding_residual_axes,
13379 lower_overlay_refs,
13380 upper_dirty_region: None,
13381 lower_dirty_region: None,
13382 })
13383 }
13384
13385 fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
13391 let origin0 = origin.checked_sub(1)?;
13392 let end = start.saturating_add(count);
13393 if origin0 < start {
13394 Some(origin)
13395 } else if origin0 >= end {
13396 Some(origin - count)
13397 } else {
13398 None
13399 }
13400 }
13401
13402 fn origin_through_delete(
13409 origin_row: u32,
13410 origin_col: u32,
13411 op: StructuralOp,
13412 ) -> Option<(u32, u32)> {
13413 match op {
13414 StructuralOp::DeleteRows { start, count, .. } => Some((
13415 origin_coord_through_delete(origin_row, start, count)?,
13416 origin_col,
13417 )),
13418 StructuralOp::DeleteColumns { start, count, .. } => Some((
13419 origin_row,
13420 origin_coord_through_delete(origin_col, start, count)?,
13421 )),
13422 StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
13423 }
13424 }
13425
13426 let mut shift_plans = Vec::new();
13427 let mut split_plans = Vec::new();
13428 let mut remove_refs = Vec::new();
13429 let mut demote_refs = Vec::new();
13430 for span_ref in span_refs {
13431 let authority = self.graph.formula_authority();
13432 let Some(span) = authority.plane.spans.get(span_ref) else {
13433 continue;
13434 };
13435 let read_summary = span
13436 .read_summary_id
13437 .and_then(|id| authority.plane.span_read_summaries.get(id));
13438 let Some(op) = op else {
13439 let result_region_affected =
13444 Self::span_result_region_intersects_affected(span, &affected_region);
13445 let read_region_affected = Self::span_any_read_region_intersects_affected(
13446 &authority.plane,
13447 span,
13448 &affected_region,
13449 );
13450 if result_region_affected || read_region_affected {
13451 demote_refs.push(span_ref);
13452 }
13453 continue;
13454 };
13455 match classify_span_for_op(span, read_summary, op) {
13456 SpanShiftPlan::NoOp => {}
13457 SpanShiftPlan::Remove => {
13458 remove_refs.push(span_ref);
13459 }
13460 SpanShiftPlan::Demote {
13461 reason:
13462 crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
13463 } => {
13464 let binding_compaction_safe = span
13465 .binding_set_id
13466 .and_then(|id| authority.plane.binding_sets.get(id))
13467 .is_none_or(|binding_set| binding_set.is_single_literal_binding());
13468 let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
13473 !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
13474 summary, op,
13475 )
13476 });
13477 if binding_compaction_safe
13478 && absolute_reads_compaction_safe
13479 && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
13480 {
13481 let new_result_region = Region::from_domain(&new_domain);
13482 let new_read_summary = if let Some(summary) = read_summary {
13483 compact_read_summary_through_delete(summary, new_result_region, op)
13484 } else {
13485 None
13486 };
13487 let Some(template) = authority.plane.templates.get(span.template_id) else {
13488 return Err(ExcelError::new(ExcelErrorKind::Ref)
13489 .with_message(
13490 "FormulaPlane delete compaction found a span with a missing template",
13491 )
13492 .into());
13493 };
13494 let carried_origin =
13495 origin_through_delete(template.origin_row, template.origin_col, op);
13496 if (read_summary.is_none() || new_read_summary.is_some())
13497 && let Some((new_origin_row, new_origin_col)) = carried_origin
13498 {
13499 let force_binding_residual_axes = span
13500 .binding_set_id
13501 .and_then(|id| authority.plane.binding_sets.get(id))
13502 .is_some_and(|binding_set| {
13503 !binding_set.value_ref_slots.is_empty()
13504 && (new_origin_row != template.origin_row
13505 || new_origin_col != template.origin_col)
13506 });
13507 let dirty_region =
13508 Self::structural_dirty_region_for_domain(&new_domain, op);
13509 shift_plans.push(ShiftPlan {
13510 span_ref,
13511 template_id: span.template_id,
13512 new_origin_row,
13513 new_origin_col,
13514 new_domain,
13515 new_read_summary,
13516 binding_set_id: span.binding_set_id,
13517 force_binding_residual_axes,
13518 rewrite: None,
13519 dirty_region,
13520 });
13521 } else {
13522 demote_refs.push(span_ref);
13523 }
13524 } else {
13525 demote_refs.push(span_ref);
13526 }
13527 }
13528 SpanShiftPlan::Demote { .. } => {
13529 demote_refs.push(span_ref);
13530 }
13531 SpanShiftPlan::Split => {
13532 match plan_span_split(authority, span, span_ref, read_summary, op) {
13533 Some(mut plan) => {
13534 plan.upper_dirty_region =
13535 Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
13536 plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
13537 &plan.lower_new_domain,
13538 op,
13539 );
13540 split_plans.push(plan);
13541 }
13542 None => demote_refs.push(span_ref),
13545 }
13546 }
13547 SpanShiftPlan::Shift {
13548 row_delta,
13549 col_delta,
13550 origin_row_delta,
13551 origin_col_delta,
13552 rewrite_absolute_reads,
13553 } => {
13554 let Some(template) = authority.plane.templates.get(span.template_id) else {
13555 return Err(ExcelError::new(ExcelErrorKind::Ref)
13556 .with_message("FormulaPlane shift found a span with a missing template")
13557 .into());
13558 };
13559 let Some(new_origin_row) =
13560 checked_shift_u32(template.origin_row, origin_row_delta)
13561 else {
13562 return Err(ExcelError::new(ExcelErrorKind::Ref)
13563 .with_message("FormulaPlane shift overflowed template origin row")
13564 .into());
13565 };
13566 let Some(new_origin_col) =
13567 checked_shift_u32(template.origin_col, origin_col_delta)
13568 else {
13569 return Err(ExcelError::new(ExcelErrorKind::Ref)
13570 .with_message("FormulaPlane shift overflowed template origin column")
13571 .into());
13572 };
13573 let Some(new_domain) =
13574 span.domain.project_through_axis_shift(row_delta, col_delta)
13575 else {
13576 return Err(ExcelError::new(ExcelErrorKind::Ref)
13577 .with_message("FormulaPlane shift overflowed span domain")
13578 .into());
13579 };
13580 let new_result_region = Region::from_domain(&new_domain);
13581 let force_binding_residual_axes = span
13582 .binding_set_id
13583 .and_then(|id| authority.plane.binding_sets.get(id))
13584 .is_some_and(|binding_set| {
13585 !binding_set.value_ref_slots.is_empty()
13586 && (origin_row_delta != 0 || origin_col_delta != 0)
13587 });
13588 let rewrite_and_summary = if rewrite_absolute_reads {
13589 let frame_sound = read_summary.is_some_and(|summary| {
13610 crate::formula_plane::structural_shift::rewrite_frame_is_sound(
13611 summary,
13612 template.origin_row,
13613 template.origin_col,
13614 op,
13615 )
13616 });
13617 if !frame_sound {
13618 demote_refs.push(span_ref);
13619 continue;
13620 }
13621 let Some(ast) = self
13622 .graph
13623 .data_store()
13624 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
13625 else {
13626 demote_refs.push(span_ref);
13627 continue;
13628 };
13629 let shift_op = shift_operation_for(op);
13630 let adjuster =
13631 crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
13632 let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
13633 span.sheet_id,
13634 self.graph.sheet_reg(),
13635 );
13636 let Some(adjusted_ast) =
13637 adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
13638 else {
13639 demote_refs.push(span_ref);
13644 continue;
13645 };
13646 let canonical = crate::formula_plane::template_canonical::canonicalize_template(
13647 &adjusted_ast,
13648 new_origin_row,
13649 new_origin_col,
13650 );
13651 if !canonical.labels.is_authority_supported() {
13652 demote_refs.push(span_ref);
13653 continue;
13654 }
13655 let literal_slots_compatible = span
13660 .binding_set_id
13661 .and_then(|id| authority.plane.binding_sets.get(id))
13662 .is_none_or(|binding_set| {
13663 binding_set.literal_slots.as_ref()
13664 == canonical.literal_slot_descriptors.as_ref()
13665 });
13666 if !literal_slots_compatible {
13667 demote_refs.push(span_ref);
13668 continue;
13669 }
13670 let value_ref_slots = Arc::from(
13678 crate::formula_plane::placement::value_ref_slot_descriptors(
13679 &canonical.expr,
13680 )
13681 .into_boxed_slice(),
13682 );
13683 let dependency_summary =
13694 crate::formula_plane::dependency_summary::summarize_canonical_template(
13695 &canonical,
13696 );
13697 let rewritten_result_region =
13698 ResultRegion::scalar_cells(new_domain.clone());
13699 let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
13700 span.sheet_id,
13701 &rewritten_result_region,
13702 &dependency_summary,
13703 self.graph.sheet_reg(),
13704 ) else {
13705 demote_refs.push(span_ref);
13706 continue;
13707 };
13708 let formula_text = Some(Arc::<str>::from(
13709 formualizer_parse::pretty::canonical_formula(&adjusted_ast),
13710 ));
13711 Some((
13712 SpanTemplateRewrite {
13713 exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
13714 parameterized_canonical_key: Arc::<str>::from(
13715 canonical.parameterized_key.payload(),
13716 ),
13717 formula_text,
13718 canonical_expr: canonical.expr,
13719 value_ref_slots,
13720 adjusted_ast,
13721 },
13722 rederived_summary,
13723 ))
13724 } else {
13725 None
13726 };
13727 let (rewrite, new_read_summary) = match rewrite_and_summary {
13728 Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
13729 None => {
13730 let new_read_summary = if let Some(summary) = read_summary {
13731 Some(
13732 shifted_read_summary(
13733 summary,
13734 new_result_region,
13735 op,
13736 row_delta,
13737 col_delta,
13738 origin_row_delta,
13739 origin_col_delta,
13740 )
13741 .ok_or_else(|| {
13742 ExcelError::new(ExcelErrorKind::Ref).with_message(
13743 "FormulaPlane shift could not project read summary",
13744 )
13745 })?,
13746 )
13747 } else {
13748 None
13749 };
13750 (None, new_read_summary)
13751 }
13752 };
13753 let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
13754 shift_plans.push(ShiftPlan {
13755 span_ref,
13756 template_id: span.template_id,
13757 new_origin_row,
13758 new_origin_col,
13759 new_domain,
13760 new_read_summary,
13761 binding_set_id: span.binding_set_id,
13762 force_binding_residual_axes,
13763 rewrite,
13764 dirty_region,
13765 });
13766 }
13767 }
13768 }
13769 let span_geometry_changed =
13770 !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
13771 let mut span_dirty_deltas = Vec::new();
13772 if span_geometry_changed {
13773 let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
13777 for plan in shift_plans {
13778 let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
13779 let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
13780 let template_slot_map =
13781 crate::formula_plane::placement::build_template_slot_map(
13782 ast_id,
13783 self.graph.data_store(),
13784 &rewrite.canonical_expr,
13785 );
13786 (ast_id, template_slot_map)
13787 });
13788 prepared_shift_plans.push((plan, rewrite_prepared));
13789 }
13790 let authority = self.graph.formula_authority_mut();
13791 for span_ref in remove_refs {
13792 authority.plane.remove_overlays_for_source_span(span_ref);
13793 authority.plane.remove_span(span_ref);
13794 }
13795 for (plan, rewrite_prepared) in prepared_shift_plans {
13796 let dirty_region = plan.dirty_region;
13797 let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
13798 let Some((ast_id, template_slot_map)) = rewrite_prepared else {
13799 return Err(ExcelError::new(ExcelErrorKind::Ref)
13800 .with_message("FormulaPlane shift lost a rewritten template AST")
13801 .into());
13802 };
13803 let Some(template_id) = authority.plane.intern_rewritten_template(
13804 plan.template_id,
13805 ast_id,
13806 rewrite.exact_canonical_key,
13807 rewrite.parameterized_canonical_key,
13808 rewrite.formula_text,
13809 plan.new_origin_row,
13810 plan.new_origin_col,
13811 ) else {
13812 return Err(ExcelError::new(ExcelErrorKind::Ref)
13813 .with_message("FormulaPlane shift could not intern rewritten template")
13814 .into());
13815 };
13816 (
13817 template_id,
13818 Some((template_slot_map, rewrite.value_ref_slots)),
13819 )
13820 } else {
13821 let Some(template_id) = authority.plane.intern_shifted_template_origin(
13822 plan.template_id,
13823 plan.new_origin_row,
13824 plan.new_origin_col,
13825 ) else {
13826 return Err(ExcelError::new(ExcelErrorKind::Ref)
13827 .with_message("FormulaPlane shift could not clone template origin")
13828 .into());
13829 };
13830 (template_id, None)
13831 };
13832 if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
13833 && let Some(binding_set_id) = plan.binding_set_id
13834 {
13835 authority.plane.set_binding_template_slots(
13836 binding_set_id,
13837 template_slot_map,
13838 value_ref_slots,
13839 );
13840 }
13841 if let Some(binding_set_id) = plan.binding_set_id {
13842 let Some(template) = authority.plane.templates.get(template_id) else {
13843 return Err(ExcelError::new(ExcelErrorKind::Ref)
13844 .with_message("FormulaPlane shift could not find shifted template")
13845 .into());
13846 };
13847 let (ast_id, origin_row, origin_col) =
13848 (template.ast_id, template.origin_row, template.origin_col);
13849 authority.plane.set_binding_template_anchor(
13850 binding_set_id,
13851 ast_id,
13852 origin_row,
13853 origin_col,
13854 );
13855 }
13856 let read_summary_id = plan
13857 .new_read_summary
13858 .map(|summary| authority.plane.insert_span_read_summary(summary));
13859 let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
13860 if !authority.plane.replace_span_geometry(
13861 plan.span_ref,
13862 template_id,
13863 plan.new_domain,
13864 result_region,
13865 read_summary_id,
13866 ) {
13867 return Err(ExcelError::new(ExcelErrorKind::Ref)
13868 .with_message("FormulaPlane shift could not update span geometry")
13869 .into());
13870 }
13871 if let Some(region) = dirty_region {
13872 span_dirty_deltas.push((plan.span_ref, region));
13873 }
13874 if plan.force_binding_residual_axes
13875 && let Some(binding_set_id) = plan.binding_set_id
13876 {
13877 authority.plane.force_binding_residual_axes(binding_set_id);
13886 }
13887 }
13888 for plan in split_plans {
13889 let upper_dirty_region = plan.upper_dirty_region;
13890 let lower_dirty_region = plan.lower_dirty_region;
13891 let Some(split_relocation) = authority
13892 .plane
13893 .spans
13894 .get(plan.span_ref)
13895 .map(|span| span.ast_relocation)
13896 else {
13897 return Err(ExcelError::new(ExcelErrorKind::Ref)
13898 .with_message("FormulaPlane split lost its anchor AST state")
13899 .into());
13900 };
13901 let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
13904 plan.template_id,
13905 plan.lower_new_origin_row,
13906 plan.lower_new_origin_col,
13907 ) else {
13908 return Err(ExcelError::new(ExcelErrorKind::Ref)
13909 .with_message("FormulaPlane split could not clone template origin")
13910 .into());
13911 };
13912 let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
13913 let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
13914 return Err(ExcelError::new(ExcelErrorKind::Ref)
13915 .with_message(
13916 "FormulaPlane split found a span with a missing binding set",
13917 )
13918 .into());
13919 };
13920 let clone = source.clone();
13924 let new_binding_set_id = authority.plane.insert_binding_set(clone);
13925 let Some(template) = authority.plane.templates.get(lower_template_id) else {
13926 return Err(ExcelError::new(ExcelErrorKind::Ref)
13927 .with_message("FormulaPlane split could not find shifted template")
13928 .into());
13929 };
13930 let (ast_id, origin_row, origin_col) =
13931 (template.ast_id, template.origin_row, template.origin_col);
13932 authority.plane.set_binding_template_anchor(
13933 new_binding_set_id,
13934 ast_id,
13935 origin_row,
13936 origin_col,
13937 );
13938 Some(new_binding_set_id)
13939 } else {
13940 None
13941 };
13942 let lower_read_summary_id = plan
13943 .lower_new_read_summary
13944 .map(|summary| authority.plane.insert_span_read_summary(summary));
13945 let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
13946 let lower_ref = authority.plane.insert_span_with_ast_relocation(
13947 crate::formula_plane::runtime::NewFormulaSpan {
13948 sheet_id: plan.sheet_id,
13949 template_id: lower_template_id,
13950 domain: plan.lower_new_domain,
13951 result_region: lower_result_region,
13952 intrinsic_mask_id: None,
13953 read_summary_id: lower_read_summary_id,
13954 binding_set_id: lower_binding_set_id,
13955 is_constant_result: plan.is_constant_result,
13956 },
13957 crate::formula_plane::runtime::SpanAstRelocation {
13958 ast_id: split_relocation.ast_id,
13959 anchor_row: plan.lower_new_origin_row,
13960 anchor_col: plan.lower_new_origin_col,
13961 },
13962 );
13963 if let Some(region) = lower_dirty_region {
13964 span_dirty_deltas.push((lower_ref, region));
13965 }
13966 if let Some(binding_set_id) = lower_binding_set_id {
13967 authority
13968 .plane
13969 .set_binding_span_ref(binding_set_id, lower_ref);
13970 if plan.lower_force_binding_residual_axes {
13971 authority.plane.force_binding_residual_axes(binding_set_id);
13975 }
13976 }
13977 let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
13981 if !name_keys.is_empty() {
13982 authority
13983 .plane
13984 .register_span_name_dependents(lower_ref, &name_keys);
13985 }
13986 for overlay_ref in plan.lower_overlay_refs {
13987 if !authority
13988 .plane
13989 .set_overlay_source_span(overlay_ref, Some(lower_ref))
13990 {
13991 return Err(ExcelError::new(ExcelErrorKind::Ref)
13992 .with_message(
13993 "FormulaPlane split could not re-point overlay provenance",
13994 )
13995 .into());
13996 }
13997 }
13998 let upper_read_summary_id = plan
14001 .upper_read_summary
14002 .map(|summary| authority.plane.insert_span_read_summary(summary));
14003 let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
14004 if !authority.plane.replace_span_geometry_with_ast_relocation(
14005 plan.span_ref,
14006 plan.template_id,
14007 split_relocation,
14008 plan.upper_domain,
14009 upper_result_region,
14010 upper_read_summary_id,
14011 ) {
14012 return Err(ExcelError::new(ExcelErrorKind::Ref)
14013 .with_message("FormulaPlane split could not update span geometry")
14014 .into());
14015 }
14016 if let Some(region) = upper_dirty_region {
14017 span_dirty_deltas.push((plan.span_ref, region));
14018 }
14019 }
14020 authority.rebuild_indexes();
14021 }
14022 for (span_ref, region) in span_dirty_deltas {
14023 self.graph.mark_formula_span_region_dirty(span_ref, region);
14024 }
14025
14026 let mut span_plans = Vec::new();
14027 for span_ref in demote_refs {
14028 let authority = self.graph.formula_authority();
14029 let Some(span) = authority.plane.spans.get(span_ref) else {
14030 continue;
14031 };
14032 let relocation = span.ast_relocation;
14033 let ast = self
14034 .graph
14035 .data_store()
14036 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14037 .ok_or_else(|| {
14038 ExcelError::new(ExcelErrorKind::Ref).with_message(
14039 "FormulaPlane demotion could not retrieve the span anchor AST",
14040 )
14041 })?;
14042 let placements = span
14043 .domain
14044 .iter()
14045 .map(|placement| (placement.row + 1, placement.col + 1))
14046 .collect();
14047 span_plans.push(SpanPlan {
14048 span_ref,
14049 sheet_id: span.sheet_id,
14050 ast,
14051 origin_row: relocation.anchor_row,
14052 origin_col: relocation.anchor_col,
14053 binding_set_id: span.binding_set_id,
14054 placements,
14055 });
14056 }
14057 if span_plans.is_empty() {
14058 return Ok(());
14059 }
14060
14061 let mut relocated = Vec::new();
14062 let mut placement_cells = Vec::new();
14063 for plan in &span_plans {
14064 for &(row, col) in &plan.placements {
14065 let row_delta = i64::from(row) - i64::from(plan.origin_row);
14066 let col_delta = i64::from(col) - i64::from(plan.origin_col);
14067 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14068 let authority = self.graph.formula_authority();
14069 if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14070 if binding_set.is_single_literal_binding() {
14071 plan.ast.clone()
14072 } else {
14073 let placement = crate::formula_plane::runtime::PlacementCoord::new(
14074 plan.sheet_id,
14075 row.saturating_sub(1),
14076 col.saturating_sub(1),
14077 );
14078 let binding =
14079 authority.plane.spans.get(plan.span_ref).and_then(|span| {
14080 binding_set
14081 .literal_bindings_for_placement(&span.domain, placement)
14082 });
14083 if let Some(binding) = binding {
14084 substitute_literal_slots_for_template_placement(
14085 &plan.ast,
14086 binding.as_ref(),
14087 )
14088 } else {
14089 plan.ast.clone()
14090 }
14091 }
14092 } else {
14093 plan.ast.clone()
14094 }
14095 } else {
14096 plan.ast.clone()
14097 };
14098 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14099 relocated.push((plan.sheet_id, row, col, ast));
14100 placement_cells.push((plan.sheet_id, row, col));
14101 }
14102 }
14103 let planned_by_sheet = {
14104 let mut pipeline = self.ingest_pipeline();
14105 let mut planned_by_sheet: BTreeMap<
14106 SheetId,
14107 Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14108 > = BTreeMap::new();
14109 for (formula_sheet_id, row, col, ast) in relocated {
14110 let placement =
14111 CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14112 let ingested =
14113 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14114 planned_by_sheet.entry(formula_sheet_id).or_default().push((
14115 row,
14116 col,
14117 ingested.ast_id,
14118 ingested.dep_plan,
14119 ));
14120 }
14121 planned_by_sheet
14122 };
14123 {
14124 let authority = self.graph.formula_authority_mut();
14125 for plan in &span_plans {
14126 authority
14127 .plane
14128 .remove_overlays_for_source_span(plan.span_ref);
14129 authority.plane.remove_span(plan.span_ref);
14130 }
14131 authority.rebuild_indexes();
14132 }
14133 if clear_computed_overlays {
14134 self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14141 }
14142 for (formula_sheet_id, planned) in planned_by_sheet {
14143 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14144 self.graph
14145 .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14146 }
14147 if !clear_computed_overlays {
14148 for (formula_sheet_id, row, col) in &placement_cells {
14149 let row0 = row.saturating_sub(1);
14150 let col0 = col.saturating_sub(1);
14151 if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14152 continue;
14153 }
14154 let cell =
14155 CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14156 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14157 self.graph.set_dirty(vertex_id, false);
14158 }
14159 }
14160 }
14161 Ok(())
14162 }
14163
14164 pub(crate) fn prepare_formula_span_demotion(
14168 &mut self,
14169 span_refs: &[FormulaSpanRef],
14170 ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14171 #[cfg(test)]
14172 let fault = self
14173 .formula_span_demotion_fault_for_test
14174 .take()
14175 .unwrap_or_default();
14176 #[cfg(not(test))]
14177 let fault = FormulaSpanDemotionFault::None;
14178
14179 if fault == FormulaSpanDemotionFault::AstPreparation {
14180 return Err(FormulaSpanDemotionError::Injected(fault));
14181 }
14182
14183 fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14184 fn visit(
14185 ast: &ASTNode,
14186 binding: &[LiteralValue],
14187 next: &mut usize,
14188 in_array: bool,
14189 ) -> ASTNode {
14190 let node_type = match &ast.node_type {
14191 ASTNodeType::Literal(_) if !in_array => {
14192 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14193 *next = next.saturating_add(1);
14194 ASTNodeType::Literal(value)
14195 }
14196 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14197 ASTNodeType::Omitted => ASTNodeType::Omitted,
14198 ASTNodeType::Reference {
14199 original,
14200 reference,
14201 } => ASTNodeType::Reference {
14202 original: original.clone(),
14203 reference: reference.clone(),
14204 },
14205 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14206 op: op.clone(),
14207 expr: Box::new(visit(expr, binding, next, in_array)),
14208 },
14209 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14210 op: op.clone(),
14211 left: Box::new(visit(left, binding, next, in_array)),
14212 right: Box::new(visit(right, binding, next, in_array)),
14213 },
14214 ASTNodeType::Function { name, args } => ASTNodeType::Function {
14215 name: name.clone(),
14216 args: args
14217 .iter()
14218 .map(|arg| visit(arg, binding, next, in_array))
14219 .collect(),
14220 },
14221 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14222 callee: Box::new(visit(callee, binding, next, in_array)),
14223 args: args
14224 .iter()
14225 .map(|arg| visit(arg, binding, next, in_array))
14226 .collect(),
14227 },
14228 ASTNodeType::Array(rows) => ASTNodeType::Array(
14229 rows.iter()
14230 .map(|row| {
14231 row.iter()
14232 .map(|cell| visit(cell, binding, next, true))
14233 .collect()
14234 })
14235 .collect(),
14236 ),
14237 };
14238 ASTNode::new(node_type, ast.source_token.clone())
14239 }
14240 let mut next = 0;
14241 visit(ast, binding, &mut next, false)
14242 }
14243
14244 struct SpanMaterialization {
14245 span_ref: FormulaSpanRef,
14246 sheet_id: SheetId,
14247 ast: ASTNode,
14248 origin_row: u32,
14249 origin_col: u32,
14250 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
14251 domain: PlacementDomain,
14252 }
14253
14254 let request_id = self
14255 .active_evaluation_resource_request
14256 .as_ref()
14257 .map(|request| request.request_id);
14258 let resource_error =
14259 |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
14260 FormulaSpanDemotionError::Resource(
14261 crate::engine::ResourceLedgerError::Exhausted(
14262 formualizer_common::ResourceExhaustionDetail {
14263 reason,
14264 limit,
14265 observed,
14266 request_id,
14267 },
14268 )
14269 .into_excel_error(),
14270 )
14271 };
14272 let authority = self.graph.formula_authority();
14273 let expected_plane_epoch = authority.plane.epoch().0;
14274 let expected_indexes_epoch = authority.indexes_epoch();
14275 let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
14276 let mut unique = Vec::new();
14277 unique.try_reserve_exact(span_refs.len()).map_err(|_| {
14278 resource_error(
14279 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14280 u64::MAX,
14281 span_refs.len() as u64,
14282 )
14283 })?;
14284 let mut plans = Vec::new();
14285 plans.try_reserve_exact(span_refs.len()).map_err(|_| {
14286 resource_error(
14287 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14288 u64::MAX,
14289 span_refs.len() as u64,
14290 )
14291 })?;
14292 let mut placement_count = 0usize;
14293 let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
14294 for &span_ref in span_refs {
14295 if unique.contains(&span_ref) {
14296 continue;
14297 }
14298 unique.push(span_ref);
14299 let span = authority
14300 .plane
14301 .spans
14302 .get(span_ref)
14303 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14304 let relocation = span.ast_relocation;
14305 let ast = self
14306 .graph
14307 .data_store()
14308 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14309 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14310 let count = usize::try_from(span.domain.cell_count())
14311 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
14312 placement_count = placement_count
14313 .checked_add(count)
14314 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14315 if u64::try_from(placement_count)
14316 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
14317 > self.workbook_load_limits.max_formula_plane_fallback_cells
14318 {
14319 return Err(resource_error(
14320 formualizer_common::ResourceExhaustionReason::MaterializationCells,
14321 self.workbook_load_limits.max_formula_plane_fallback_cells,
14322 placement_count as u64,
14323 ));
14324 }
14325 let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
14326 *sheet_count = sheet_count
14327 .checked_add(span.domain.cell_count())
14328 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14329 if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
14330 return Err(resource_error(
14331 formualizer_common::ResourceExhaustionReason::Admission,
14332 self.workbook_load_limits.max_sheet_logical_cells,
14333 *sheet_count,
14334 ));
14335 }
14336 let domain_in_bounds = match &span.domain {
14337 PlacementDomain::RowRun { row_end, col, .. } => {
14338 *row_end < self.workbook_load_limits.max_sheet_rows
14339 && *col < self.workbook_load_limits.max_sheet_cols
14340 }
14341 PlacementDomain::ColRun { row, col_end, .. } => {
14342 *row < self.workbook_load_limits.max_sheet_rows
14343 && *col_end < self.workbook_load_limits.max_sheet_cols
14344 }
14345 PlacementDomain::Rect {
14346 row_end, col_end, ..
14347 } => {
14348 *row_end < self.workbook_load_limits.max_sheet_rows
14349 && *col_end < self.workbook_load_limits.max_sheet_cols
14350 }
14351 };
14352 if !domain_in_bounds {
14353 return Err(resource_error(
14354 formualizer_common::ResourceExhaustionReason::Admission,
14355 u64::from(self.workbook_load_limits.max_sheet_rows)
14356 .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
14357 span.domain.cell_count(),
14358 ));
14359 }
14360 plans.push(SpanMaterialization {
14361 span_ref,
14362 sheet_id: span.sheet_id,
14363 ast,
14364 origin_row: relocation.anchor_row,
14365 origin_col: relocation.anchor_col,
14366 binding_set_id: span.binding_set_id,
14367 domain: span.domain.clone(),
14368 });
14369 }
14370 let mut relocated = Vec::new();
14371 relocated.try_reserve_exact(placement_count).map_err(|_| {
14372 resource_error(
14373 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14374 u64::MAX,
14375 placement_count as u64,
14376 )
14377 })?;
14378 for plan in &plans {
14379 for coord in plan.domain.iter() {
14380 let row = coord.row.saturating_add(1);
14381 let col = coord.col.saturating_add(1);
14382 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14383 let authority = self.graph.formula_authority();
14384 let binding_set = authority
14385 .plane
14386 .binding_sets
14387 .get(binding_set_id)
14388 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14389 if binding_set.is_single_literal_binding() {
14390 plan.ast.clone()
14391 } else {
14392 let placement = PlacementCoord::new(
14393 plan.sheet_id,
14394 row.saturating_sub(1),
14395 col.saturating_sub(1),
14396 );
14397 let span = authority
14398 .plane
14399 .spans
14400 .get(plan.span_ref)
14401 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14402 let binding = binding_set
14403 .literal_bindings_for_placement(&span.domain, placement)
14404 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14405 substitute_literal_slots(&plan.ast, binding.as_ref())
14406 }
14407 } else {
14408 plan.ast.clone()
14409 };
14410 let row_delta = i64::from(row) - i64::from(plan.origin_row);
14411 let col_delta = i64::from(col) - i64::from(plan.origin_col);
14412 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
14413 .map_err(FormulaSpanDemotionError::AstPreparation)?;
14414 relocated.push((plan.sheet_id, row, col, ast));
14415 }
14416 }
14417
14418 let mut analyzed = Vec::new();
14419 analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
14420 resource_error(
14421 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14422 u64::MAX,
14423 relocated.len() as u64,
14424 )
14425 })?;
14426 {
14427 let mut pipeline = self.ingest_pipeline();
14428 for (sheet_id, row, col, ast) in relocated {
14429 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
14430 let ingested = pipeline
14431 .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
14432 .map_err(FormulaSpanDemotionError::AstPreparation)?;
14433 analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
14434 }
14435 }
14436
14437 if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
14438 return Err(FormulaSpanDemotionError::Injected(fault));
14439 }
14440 let legacy_graph = self
14441 .graph
14442 .prepare_legacy_graph_plan_multi_sheet(analyzed)
14443 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14444 self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
14445 .map_err(FormulaSpanDemotionError::Resource)?;
14446
14447 Ok(PreparedFormulaSpanDemotion {
14448 span_refs: unique,
14449 expected_plane_epoch,
14450 expected_indexes_epoch,
14451 expected_indexed_plane_epoch,
14452 placement_count,
14453 legacy_graph,
14454 fault,
14455 })
14456 }
14457
14458 pub(crate) fn validate_prepared_formula_span_demotion(
14459 &self,
14460 prepared: &PreparedFormulaSpanDemotion,
14461 ) -> Result<(), FormulaSpanDemotionError> {
14462 let authority = self.graph.formula_authority();
14463 if authority.plane.epoch().0 != prepared.expected_plane_epoch
14464 || authority.indexes_epoch() != prepared.expected_indexes_epoch
14465 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14466 || prepared
14467 .span_refs
14468 .iter()
14469 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14470 {
14471 return Err(FormulaSpanDemotionError::StaleAuthority);
14472 }
14473 self.graph
14474 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14475 .map_err(FormulaSpanDemotionError::LegacyGraph)
14476 }
14477
14478 pub(crate) fn commit_prepared_formula_span_demotion(
14484 &mut self,
14485 prepared: PreparedFormulaSpanDemotion,
14486 ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
14487 self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
14488 .map_err(FormulaSpanDemotionError::Resource)?;
14489 if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
14490 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14491 }
14492 self.graph
14493 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14494 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14495 if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
14496 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14497 }
14498 {
14499 let authority = self.graph.formula_authority();
14500 if authority.plane.epoch().0 != prepared.expected_plane_epoch
14501 || authority.indexes_epoch() != prepared.expected_indexes_epoch
14502 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14503 || prepared
14504 .span_refs
14505 .iter()
14506 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14507 {
14508 return Err(FormulaSpanDemotionError::StaleAuthority);
14509 }
14510 }
14511 if matches!(
14512 prepared.fault,
14513 FormulaSpanDemotionFault::AllocationReservation
14514 | FormulaSpanDemotionFault::BeforeFirstMutation
14515 ) {
14516 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14517 }
14518
14519 let PreparedFormulaSpanDemotion {
14520 span_refs,
14521 placement_count,
14522 legacy_graph,
14523 ..
14524 } = prepared;
14525 let _ = self
14526 .graph
14527 .apply_prevalidated_legacy_graph_plan(legacy_graph);
14528 let authority = self.graph.formula_authority_mut();
14529 for span_ref in &span_refs {
14530 authority.plane.remove_overlays_for_source_span(*span_ref);
14531 let _ = authority.plane.remove_span(*span_ref);
14532 }
14533 let _ = authority.rebuild_indexes();
14534 self.mark_topology_edited();
14535 Ok(FormulaSpanDemotionReport {
14536 spans_demoted: span_refs.len(),
14537 placements_materialized: placement_count,
14538 })
14539 }
14540
14541 #[cfg(test)]
14542 pub(crate) fn set_formula_span_demotion_fault_for_test(
14543 &mut self,
14544 fault: FormulaSpanDemotionFault,
14545 ) {
14546 self.formula_span_demotion_fault_for_test = Some(fault);
14547 }
14548
14549 #[cfg(test)]
14550 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
14551 self.force_non_cycle_schedule_fallback_for_test = true;
14552 }
14553
14554 #[cfg(test)]
14555 pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
14556 self.mixed_topology_index_builds_for_test
14557 }
14558
14559 fn collect_cyclic_span_refs(
14564 &self,
14565 schedule: &MixedSchedule,
14566 span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
14567 ) -> Vec<FormulaSpanRef> {
14568 let mut refs = Vec::new();
14569 for fallback in &schedule.fallbacks {
14570 if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
14571 continue;
14572 }
14573 if let FormulaProducerId::Span(span_id) = fallback.producer
14574 && let Some(span_ref) = span_refs_by_id.get(&span_id)
14575 && !refs.contains(span_ref)
14576 {
14577 refs.push(*span_ref);
14578 }
14579 }
14580 refs
14581 }
14582
14583 fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
14591 let materialization_started = crate::instant::FzInstant::now();
14592 let prepared = self
14593 .prepare_formula_span_demotion(span_refs)
14594 .map_err(|error| {
14595 if let FormulaSpanDemotionError::Resource(error) = error {
14596 error
14597 } else {
14598 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14599 "FormulaPlane cycle-member span demotion preparation failed: {error}"
14600 ))
14601 }
14602 })?;
14603 let report = self
14604 .commit_prepared_formula_span_demotion(prepared)
14605 .map_err(|error| {
14606 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14607 "FormulaPlane cycle-member span demotion commit failed: {error}"
14608 ))
14609 })?;
14610 self.observe_materialization(
14611 report.placements_materialized,
14612 true,
14613 materialization_started.elapsed(),
14614 );
14615 self.formula_plane_cycle_member_span_demotions = self
14616 .formula_plane_cycle_member_span_demotions
14617 .saturating_add(report.spans_demoted as u64);
14618 Self::record_shadow_fallback_reason(
14619 &mut self.formula_ingest_report_total,
14620 PlacementFallbackReason::CycleMember,
14621 report.spans_demoted as u64,
14622 );
14623 Ok(())
14624 }
14625
14626 fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
14641 let dirty = self.graph.get_evaluation_vertices();
14642 if dirty.is_empty() {
14643 return Ok(0);
14644 }
14645 let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
14646 let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
14647 let mut cycle_errors = 0usize;
14648 let mut stamped_vertices: Vec<VertexId> = Vec::new();
14649 for &unit in &schedule.units {
14650 let ScheduleUnit::Cycle(i) = unit else {
14651 continue;
14652 };
14653 let members = schedule.unit_cycle(i);
14654 let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
14655 if stamped > 0 {
14656 cycle_errors += 1;
14657 }
14658 stamped_vertices.extend(members.iter().copied());
14659 }
14660 if !stamped_vertices.is_empty() {
14665 self.resource_checkpoint(0)?;
14666 self.graph.clear_dirty_flags(&stamped_vertices);
14667 }
14668 Ok(cycle_errors)
14669 }
14670
14671 fn materialize_deferred_sheet_before_structural_edit(
14672 &mut self,
14673 sheet: &str,
14674 ) -> Result<(), crate::engine::EditorError> {
14675 if self.staged_formulas.contains_key(sheet) {
14676 self.build_graph_for_sheets([sheet])?;
14677 }
14678 Ok(())
14679 }
14680
14681 pub fn insert_rows(
14683 &mut self,
14684 sheet: &str,
14685 before: u32,
14686 count: u32,
14687 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14688 {
14689 if count == 0 {
14690 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14691 }
14692 self.observe_function_semantic_epoch()
14693 .map_err(crate::engine::EditorError::Excel)?;
14694 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14695 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14696 let sheet_id = self.ensure_known_sheet_id(sheet)?;
14697 let before0 = before.saturating_sub(1);
14698 let affected_region = Self::structural_row_region(sheet_id, before0);
14699 let op = StructuralOp::InsertRows {
14700 sheet_id,
14701 before: before0,
14702 count,
14703 };
14704 self.demote_spans_for_structural_op(op, affected_region)?;
14705 let summary = {
14706 let mut editor = VertexEditor::new(&mut self.graph);
14707 editor.insert_rows(sheet_id, before0, count)?
14708 };
14709 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14710 let before0 = before0 as usize;
14711 asheet.insert_rows(before0, count as usize);
14712 }
14713 self.mark_moved_formula_vertices_dirty(&summary);
14714 self.clear_computed_overlay_after_row(sheet, before0 as usize);
14715 self.shift_row_visibility_insert(sheet_id, before0, count);
14716 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14717 self.mark_topology_edited();
14718 Ok(summary)
14719 }
14720
14721 pub fn delete_rows(
14723 &mut self,
14724 sheet: &str,
14725 start: u32,
14726 count: u32,
14727 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14728 {
14729 if count == 0 {
14730 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14731 }
14732 self.observe_function_semantic_epoch()
14733 .map_err(crate::engine::EditorError::Excel)?;
14734 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14735 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14736 let sheet_id = self.ensure_known_sheet_id(sheet)?;
14737 let start0 = start.saturating_sub(1);
14738 let affected_region = Self::structural_row_region(sheet_id, start0);
14739 let op = StructuralOp::DeleteRows {
14740 sheet_id,
14741 start: start0,
14742 count,
14743 };
14744 self.demote_spans_for_structural_op(op, affected_region)?;
14745 let summary = {
14746 let mut editor = VertexEditor::new(&mut self.graph);
14747 editor.delete_rows(sheet_id, start0, count)?
14748 };
14749 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14750 let start0 = start0 as usize;
14751 asheet.delete_rows(start0, count as usize);
14752 }
14753 self.mark_moved_formula_vertices_dirty(&summary);
14754 self.clear_computed_overlay_after_row(sheet, start0 as usize);
14755 self.shift_row_visibility_delete(sheet_id, start0, count);
14756 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14757 self.mark_topology_edited();
14758 Ok(summary)
14759 }
14760
14761 pub fn insert_columns(
14763 &mut self,
14764 sheet: &str,
14765 before: u32,
14766 count: u32,
14767 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14768 {
14769 if count == 0 {
14770 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14771 }
14772 self.observe_function_semantic_epoch()
14773 .map_err(crate::engine::EditorError::Excel)?;
14774 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14775 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14776 let sheet_id = self.graph.sheet_id(sheet).ok_or(
14777 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14778 name: sheet.to_string(),
14779 reason: "Unknown sheet".to_string(),
14780 },
14781 )?;
14782 let before0 = before.saturating_sub(1);
14783 let affected_region = Self::structural_col_region(sheet_id, before0);
14784 let op = StructuralOp::InsertColumns {
14785 sheet_id,
14786 before: before0,
14787 count,
14788 };
14789 self.demote_spans_for_structural_op(op, affected_region)?;
14790 let summary = {
14791 let mut editor = VertexEditor::new(&mut self.graph);
14792 editor.insert_columns(sheet_id, before0, count)?
14793 };
14794 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14795 let before0 = before0 as usize;
14796 asheet.insert_columns(before0, count as usize);
14797 }
14798 self.mark_moved_formula_vertices_dirty(&summary);
14799 self.clear_computed_overlay_after_col(sheet, before0 as usize);
14800 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14801 self.mark_topology_edited();
14802 Ok(summary)
14803 }
14804
14805 pub fn delete_columns(
14807 &mut self,
14808 sheet: &str,
14809 start: u32,
14810 count: u32,
14811 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14812 {
14813 if count == 0 {
14814 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14815 }
14816 self.observe_function_semantic_epoch()
14817 .map_err(crate::engine::EditorError::Excel)?;
14818 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14819 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14820 let sheet_id = self.graph.sheet_id(sheet).ok_or(
14821 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14822 name: sheet.to_string(),
14823 reason: "Unknown sheet".to_string(),
14824 },
14825 )?;
14826 let start0 = start.saturating_sub(1);
14827 let affected_region = Self::structural_col_region(sheet_id, start0);
14828 let op = StructuralOp::DeleteColumns {
14829 sheet_id,
14830 start: start0,
14831 count,
14832 };
14833 self.demote_spans_for_structural_op(op, affected_region)?;
14834 let summary = {
14835 let mut editor = VertexEditor::new(&mut self.graph);
14836 editor.delete_columns(sheet_id, start0, count)?
14837 };
14838 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14839 let start0 = start0 as usize;
14840 asheet.delete_columns(start0, count as usize);
14841 }
14842 self.mark_moved_formula_vertices_dirty(&summary);
14843 self.clear_computed_overlay_after_col(sheet, start0 as usize);
14844 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14845 self.mark_topology_edited();
14846 Ok(summary)
14847 }
14848 fn arrow_used_row_bounds(
14850 &self,
14851 sheet: &str,
14852 start_col: u32,
14853 end_col: u32,
14854 ) -> Option<(u32, u32)> {
14855 let a = self.sheet_store().sheet(sheet)?;
14856 if a.columns.is_empty() {
14857 return None;
14858 }
14859 let sc0 = start_col.saturating_sub(1) as usize;
14860 let ec0 = end_col.saturating_sub(1) as usize;
14861 let col_hi = a.columns.len().saturating_sub(1);
14862 if sc0 > col_hi {
14863 return None;
14864 }
14865 let ec0 = ec0.min(col_hi);
14866 let snap = self.data_snapshot_id();
14868 let mut min_r0: Option<usize> = None;
14869 for ci in sc0..=ec0 {
14870 let sheet_id = self.graph.sheet_id(sheet)?;
14871 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
14872 g.as_ref()
14873 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14874 }) {
14875 let mv = mv as usize;
14876 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
14877 continue;
14878 }
14879 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14881 if let Ok(mut g) = self.row_bounds_cache.write() {
14882 g.get_or_insert_with(|| RowBoundsCache::new(snap))
14883 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
14884 }
14885 if let Some(m) = min_c {
14886 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
14887 }
14888 }
14889 min_r0?;
14890 let mut max_r0: Option<usize> = None;
14891 for ci in sc0..=ec0 {
14892 let sheet_id = self.graph.sheet_id(sheet)?;
14893 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
14894 g.as_ref()
14895 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14896 }) {
14897 let mv = mv as usize;
14898 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
14899 continue;
14900 }
14901 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14902 if let Ok(mut g) = self.row_bounds_cache.write() {
14903 g.get_or_insert_with(|| RowBoundsCache::new(snap))
14904 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
14905 }
14906 if let Some(m) = max_c {
14907 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
14908 }
14909 }
14910 match (min_r0, max_r0) {
14911 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
14912 _ => None,
14913 }
14914 }
14915
14916 fn scan_column_used_bounds(
14917 a: &crate::arrow_store::ArrowSheet,
14918 ci: usize,
14919 ) -> (Option<u32>, Option<u32>) {
14920 let col = &a.columns[ci];
14921
14922 let mut min_r0: Option<u32> = None;
14924 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
14925 let tags = chunk.type_tag.values();
14926 for (off, &t) in tags.iter().enumerate() {
14927 let overlay_non_empty = chunk
14928 .overlay
14929 .get(off)
14930 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14931 .unwrap_or(false)
14932 || chunk
14933 .computed_overlay
14934 .get(off)
14935 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14936 .unwrap_or(false);
14937 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14938 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14939 break;
14940 };
14941 let row0 = chunk_start + off;
14942 min_r0 = Some(row0 as u32);
14943 break;
14944 }
14945 }
14946 if min_r0.is_some() {
14947 break;
14948 }
14949 }
14950 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
14951 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14952 sparse_idxs.sort_unstable();
14953 for chunk_idx in sparse_idxs {
14954 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14955 continue;
14956 };
14957 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14958 continue;
14959 };
14960 let tags = chunk.type_tag.values();
14961 for (off, &t) in tags.iter().enumerate() {
14962 let overlay_non_empty = chunk
14963 .overlay
14964 .get(off)
14965 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14966 .unwrap_or(false)
14967 || chunk
14968 .computed_overlay
14969 .get(off)
14970 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14971 .unwrap_or(false);
14972 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14973 let row0 = chunk_start + off;
14974 min_r0 = Some(row0 as u32);
14975 break;
14976 }
14977 }
14978 if min_r0.is_some() {
14979 break;
14980 }
14981 }
14982 }
14983
14984 let mut max_r0: Option<u32> = None;
14986 if !col.sparse_chunks.is_empty() {
14987 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14988 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
14989 for chunk_idx in sparse_idxs {
14990 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14991 continue;
14992 };
14993 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14994 continue;
14995 };
14996 let tags = chunk.type_tag.values();
14997 for (rev_idx, &t) in tags.iter().enumerate().rev() {
14998 let overlay_non_empty = chunk
14999 .overlay
15000 .get(rev_idx)
15001 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15002 .unwrap_or(false)
15003 || chunk
15004 .computed_overlay
15005 .get(rev_idx)
15006 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15007 .unwrap_or(false);
15008 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15009 let row0 = chunk_start + rev_idx;
15010 max_r0 = Some(row0 as u32);
15011 break;
15012 }
15013 }
15014 if max_r0.is_some() {
15015 break;
15016 }
15017 }
15018 }
15019 if max_r0.is_none() {
15020 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
15021 let tags = chunk.type_tag.values();
15022 for (rev_idx, &t) in tags.iter().enumerate().rev() {
15023 let overlay_non_empty = chunk
15024 .overlay
15025 .get(rev_idx)
15026 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15027 .unwrap_or(false)
15028 || chunk
15029 .computed_overlay
15030 .get(rev_idx)
15031 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15032 .unwrap_or(false);
15033 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15034 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15035 break;
15036 };
15037 let row0 = chunk_start + rev_idx;
15038 max_r0 = Some(row0 as u32);
15039 break;
15040 }
15041 }
15042 if max_r0.is_some() {
15043 break;
15044 }
15045 }
15046 }
15047
15048 (min_r0, max_r0)
15049 }
15050
15051 fn arrow_used_col_bounds(
15053 &self,
15054 sheet: &str,
15055 start_row: u32,
15056 end_row: u32,
15057 ) -> Option<(u32, u32)> {
15058 let a = self.sheet_store().sheet(sheet)?;
15059 if a.columns.is_empty() {
15060 return None;
15061 }
15062 let sr0 = start_row.saturating_sub(1) as usize;
15063 let er0 = end_row.saturating_sub(1) as usize;
15064 if sr0 > er0 {
15065 return None;
15066 }
15067 let mut min_c0: Option<usize> = None;
15070 let mut max_c0: Option<usize> = None;
15071 for (ci, col) in a.columns.iter().enumerate() {
15073 let mut any_in_range = false;
15074
15075 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15076 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15077 return false;
15078 };
15079 let chunk_len = chunk.type_tag.len();
15080 if chunk_len == 0 {
15081 return false;
15082 }
15083 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15084 if sr0 > chunk_end || er0 < chunk_start {
15086 return false;
15087 }
15088 let start_off = sr0.max(chunk_start) - chunk_start;
15089 let end_off = er0.min(chunk_end) - chunk_start;
15090 let tags = chunk.type_tag.values();
15091 for off in start_off..=end_off {
15092 let overlay_non_empty = chunk
15093 .overlay
15094 .get(off)
15095 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15096 .unwrap_or(false)
15097 || chunk
15098 .computed_overlay
15099 .get(off)
15100 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15101 .unwrap_or(false);
15102 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15103 return true;
15104 }
15105 }
15106 false
15107 };
15108
15109 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15110 if scan_chunk(chunk_idx, chunk) {
15111 any_in_range = true;
15112 break;
15113 }
15114 }
15115
15116 if !any_in_range && !col.sparse_chunks.is_empty() {
15117 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15118 if scan_chunk(chunk_idx, chunk) {
15119 any_in_range = true;
15120 break;
15121 }
15122 }
15123 }
15124
15125 if any_in_range {
15126 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
15127 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
15128 }
15129 }
15130 match (min_c0, max_c0) {
15131 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15132 _ => None,
15133 }
15134 }
15135
15136 fn formula_row_bounds_for_columns(
15137 &self,
15138 sheet: &str,
15139 start_col: u32,
15140 end_col: u32,
15141 ) -> Option<(u32, u32)> {
15142 let sheet_id = self.graph.sheet_id(sheet)?;
15143 let sc0 = start_col.saturating_sub(1);
15144 let ec0 = end_col.saturating_sub(1);
15145 let mut min_r0: Option<u32> = None;
15146 let mut max_r0: Option<u32> = None;
15147
15148 if let Some(index) = self.graph.sheet_index(sheet_id) {
15149 for vid in index.vertices_in_col_range(sc0, ec0) {
15150 if !matches!(
15151 self.graph.get_vertex_kind(vid),
15152 VertexKind::FormulaScalar | VertexKind::FormulaArray
15153 ) {
15154 continue;
15155 }
15156 let row0 = self.graph.vertex_coord(vid).row();
15157 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15158 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15159 }
15160 } else {
15161 for vid in self.graph.vertices_in_sheet(sheet_id) {
15162 if !matches!(
15163 self.graph.get_vertex_kind(vid),
15164 VertexKind::FormulaScalar | VertexKind::FormulaArray
15165 ) {
15166 continue;
15167 }
15168 let coord = self.graph.vertex_coord(vid);
15169 let col0 = coord.col();
15170 if col0 < sc0 || col0 > ec0 {
15171 continue;
15172 }
15173 let row0 = coord.row();
15174 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15175 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15176 }
15177 }
15178
15179 match (min_r0, max_r0) {
15180 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15181 _ => None,
15182 }
15183 }
15184
15185 fn formula_col_bounds_for_rows(
15186 &self,
15187 sheet: &str,
15188 start_row: u32,
15189 end_row: u32,
15190 ) -> Option<(u32, u32)> {
15191 let sheet_id = self.graph.sheet_id(sheet)?;
15192 let sr0 = start_row.saturating_sub(1);
15193 let er0 = end_row.saturating_sub(1);
15194 let mut min_c0: Option<u32> = None;
15195 let mut max_c0: Option<u32> = None;
15196
15197 if let Some(index) = self.graph.sheet_index(sheet_id) {
15198 for vid in index.vertices_in_row_range(sr0, er0) {
15199 if !matches!(
15200 self.graph.get_vertex_kind(vid),
15201 VertexKind::FormulaScalar | VertexKind::FormulaArray
15202 ) {
15203 continue;
15204 }
15205 let col0 = self.graph.vertex_coord(vid).col();
15206 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15207 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15208 }
15209 } else {
15210 for vid in self.graph.vertices_in_sheet(sheet_id) {
15211 if !matches!(
15212 self.graph.get_vertex_kind(vid),
15213 VertexKind::FormulaScalar | VertexKind::FormulaArray
15214 ) {
15215 continue;
15216 }
15217 let coord = self.graph.vertex_coord(vid);
15218 let row0 = coord.row();
15219 if row0 < sr0 || row0 > er0 {
15220 continue;
15221 }
15222 let col0 = coord.col();
15223 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15224 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15225 }
15226 }
15227
15228 match (min_c0, max_c0) {
15229 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15230 _ => None,
15231 }
15232 }
15233
15234 fn union_used_bounds(
15235 first: Option<(u32, u32)>,
15236 second: Option<(u32, u32)>,
15237 ) -> Option<(u32, u32)> {
15238 match (first, second) {
15239 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
15240 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
15241 (None, None) => None,
15242 }
15243 }
15244
15245 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
15248 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15249 return;
15250 }
15251 if self.arrow_sheets.sheet(sheet).is_none() {
15252 self.arrow_sheets
15253 .sheets
15254 .push(crate::arrow_store::ArrowSheet {
15255 name: std::sync::Arc::<str>::from(sheet),
15256 date_system: self.config.date_system,
15257 columns: Vec::new(),
15258 nrows: 0,
15259 chunk_starts: Vec::new(),
15260 chunk_rows: 32 * 1024,
15261 });
15262 }
15263
15264 let row0 = row.saturating_sub(1) as usize;
15265 let col0 = col.saturating_sub(1) as usize;
15266
15267 let asheet = self
15268 .arrow_sheets
15269 .sheet_mut(sheet)
15270 .expect("ArrowSheet must exist");
15271
15272 let cur_cols = asheet.columns.len();
15273 if col0 >= cur_cols {
15274 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15275 }
15276
15277 if row0 >= asheet.nrows as usize {
15278 if asheet.columns.is_empty() {
15279 asheet.insert_columns(0, 1);
15280 }
15281 asheet.ensure_row_capacity(row0 + 1);
15282 }
15283 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
15284 let ov =
15285 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
15286 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
15287 let _ = ch.overlay.set(in_off, ov);
15288 ch.computed_overlay.remove(in_off)
15293 } else {
15294 return;
15295 };
15296 let abs_threshold = 1024usize;
15298 let frac_den = 50usize;
15299 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
15300 if freed > 0 {
15301 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
15302 }
15303 self.adjust_computed_overlay_bytes(computed_delta);
15304 }
15305 }
15306
15307 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
15312 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15313 return;
15314 }
15315 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15316 return;
15317 };
15318 let row0 = row.saturating_sub(1) as usize;
15319 let col0 = col.saturating_sub(1) as usize;
15320 if row0 >= asheet.nrows as usize {
15321 return;
15322 }
15323 if col0 >= asheet.columns.len() {
15324 return;
15325 }
15326 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15327 return;
15328 };
15329 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
15330 let _ = ch.overlay.remove(in_off);
15331 }
15332 }
15333
15334 fn clear_computed_overlay_col_row_range(
15335 &mut self,
15336 sheet: &str,
15337 col0: usize,
15338 start_row0: usize,
15339 end_row0_exclusive: usize,
15340 ) {
15341 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15342 return;
15343 }
15344 if start_row0 >= end_row0_exclusive {
15345 return;
15346 }
15347
15348 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15349 return;
15350 };
15351 if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
15352 return;
15353 }
15354 let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
15355 if start_row0 >= end_row0_exclusive {
15356 return;
15357 }
15358
15359 let starts = asheet.chunk_starts.clone();
15360 let nrows = asheet.nrows as usize;
15361 let mut delta = 0isize;
15362 let Some(col) = asheet.columns.get_mut(col0) else {
15363 return;
15364 };
15365 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15366 let Some(&chunk_start) = starts.get(chunk_idx) else {
15367 continue;
15368 };
15369 let chunk_end = starts
15370 .get(chunk_idx + 1)
15371 .copied()
15372 .unwrap_or(nrows)
15373 .min(chunk_start.saturating_add(ch.len()));
15374 let clear_start = start_row0.max(chunk_start);
15375 let clear_end = end_row0_exclusive.min(chunk_end);
15376 if clear_start >= clear_end {
15377 continue;
15378 }
15379 if clear_start == chunk_start && clear_end == chunk_end {
15380 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15381 } else {
15382 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15383 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15384 delta = delta.saturating_add(
15385 ch.computed_overlay
15386 .remove_range(start_in_chunk..end_in_chunk),
15387 );
15388 }
15389 }
15390 for (chunk_idx, ch) in &mut col.sparse_chunks {
15391 let Some(&chunk_start) = starts.get(*chunk_idx) else {
15392 continue;
15393 };
15394 let chunk_end = starts
15395 .get(*chunk_idx + 1)
15396 .copied()
15397 .unwrap_or(nrows)
15398 .min(chunk_start.saturating_add(ch.len()));
15399 let clear_start = start_row0.max(chunk_start);
15400 let clear_end = end_row0_exclusive.min(chunk_end);
15401 if clear_start >= clear_end {
15402 continue;
15403 }
15404 if clear_start == chunk_start && clear_end == chunk_end {
15405 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15406 } else {
15407 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15408 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15409 delta = delta.saturating_add(
15410 ch.computed_overlay
15411 .remove_range(start_in_chunk..end_in_chunk),
15412 );
15413 }
15414 }
15415 self.adjust_computed_overlay_bytes(delta);
15416 }
15417
15418 fn clear_computed_overlay_cells_in_region(
15419 &mut self,
15420 cells: &[(SheetId, u32, u32)],
15421 affected_region: &Region,
15422 ) {
15423 let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
15424 for (formula_sheet_id, row, col) in cells {
15425 let row0 = row.saturating_sub(1);
15426 let col0 = col.saturating_sub(1);
15427 let placement_region = Region::point(*formula_sheet_id, row0, col0);
15428 if placement_region.intersects(affected_region) {
15429 by_col
15430 .entry((*formula_sheet_id, col0))
15431 .or_default()
15432 .push(row0);
15433 }
15434 }
15435
15436 for ((formula_sheet_id, col0), mut rows) in by_col {
15437 rows.sort_unstable();
15438 rows.dedup();
15439 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
15440 let mut start = rows[0];
15441 let mut prev = rows[0];
15442 for row in rows.into_iter().skip(1) {
15443 if row == prev.saturating_add(1) {
15444 prev = row;
15445 continue;
15446 }
15447 self.clear_computed_overlay_col_row_range(
15448 &sheet_name,
15449 col0 as usize,
15450 start as usize,
15451 prev.saturating_add(1) as usize,
15452 );
15453 start = row;
15454 prev = row;
15455 }
15456 self.clear_computed_overlay_col_row_range(
15457 &sheet_name,
15458 col0 as usize,
15459 start as usize,
15460 prev.saturating_add(1) as usize,
15461 );
15462 }
15463 }
15464
15465 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
15466 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15467 return;
15468 }
15469
15470 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15471 return;
15472 };
15473 if start_row0 >= asheet.nrows as usize {
15474 return;
15475 }
15476
15477 let starts = asheet.chunk_starts.clone();
15478 let nrows = asheet.nrows as usize;
15479 let mut delta = 0isize;
15480 for col in &mut asheet.columns {
15481 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15482 let Some(&chunk_start) = starts.get(chunk_idx) else {
15483 continue;
15484 };
15485 let chunk_end = starts
15486 .get(chunk_idx + 1)
15487 .copied()
15488 .unwrap_or(nrows)
15489 .min(chunk_start.saturating_add(ch.len()));
15490 if chunk_end <= start_row0 {
15491 continue;
15492 }
15493 if chunk_start >= start_row0 {
15494 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15495 } else {
15496 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15497 delta = delta
15498 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15499 }
15500 }
15501
15502 for (chunk_idx, ch) in &mut col.sparse_chunks {
15503 let Some(&chunk_start) = starts.get(*chunk_idx) else {
15504 continue;
15505 };
15506 let chunk_end = starts
15507 .get(*chunk_idx + 1)
15508 .copied()
15509 .unwrap_or(nrows)
15510 .min(chunk_start.saturating_add(ch.len()));
15511 if chunk_end <= start_row0 {
15512 continue;
15513 }
15514 if chunk_start >= start_row0 {
15515 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15516 } else {
15517 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15518 delta = delta
15519 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15520 }
15521 }
15522 }
15523 self.adjust_computed_overlay_bytes(delta);
15524 }
15525
15526 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
15527 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15528 return;
15529 }
15530
15531 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15532 return;
15533 };
15534 if start_col0 >= asheet.columns.len() {
15535 return;
15536 }
15537
15538 let mut delta = 0isize;
15539 for col in asheet.columns.iter_mut().skip(start_col0) {
15540 for ch in &mut col.chunks {
15541 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15542 }
15543 for ch in col.sparse_chunks.values_mut() {
15544 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15545 }
15546 }
15547 self.adjust_computed_overlay_bytes(delta);
15548 }
15549
15550 #[inline]
15551 fn literal_to_overlay_value(
15552 value: &LiteralValue,
15553 date_system: crate::engine::DateSystem,
15554 ) -> crate::arrow_store::OverlayValue {
15555 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
15556 }
15557
15558 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
15559 self.arrow_sheets
15560 .sheet(sheet)
15561 .map(|sheet| sheet.date_system)
15562 .unwrap_or(self.config.date_system)
15563 }
15564
15565 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15568 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15569 return None;
15570 }
15571 let asheet = self.arrow_sheets.sheet(sheet)?;
15572 let row0 = row.saturating_sub(1) as usize;
15573 let col0 = col.saturating_sub(1) as usize;
15574 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15575 return None;
15576 }
15577 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15578 let ch = asheet.columns[col0].chunk(ch_idx)?;
15579 ch.overlay
15580 .get_scalar(in_off)
15581 .map(|ov| ov.to_literal_for(asheet.date_system))
15582 }
15583
15584 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15587 if !(self.config.arrow_storage_enabled
15588 && self.config.delta_overlay_enabled
15589 && self.config.write_formula_overlay_enabled)
15590 {
15591 return None;
15592 }
15593 let asheet = self.arrow_sheets.sheet(sheet)?;
15594 let row0 = row.saturating_sub(1) as usize;
15595 let col0 = col.saturating_sub(1) as usize;
15596 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15597 return None;
15598 }
15599 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15600 let ch = asheet.columns[col0].chunk(ch_idx)?;
15601 ch.computed_overlay
15602 .get_scalar(in_off)
15603 .map(|ov| ov.to_literal_for(asheet.date_system))
15604 }
15605
15606 fn set_delta_overlay_cell_raw(
15607 &mut self,
15608 sheet: &str,
15609 row: u32,
15610 col: u32,
15611 value: Option<LiteralValue>,
15612 ) {
15613 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15614 return;
15615 }
15616
15617 self.ensure_arrow_sheet(sheet);
15618 let date_system = self.arrow_sheet_date_system(sheet);
15619 let ov_opt = value
15620 .as_ref()
15621 .map(|value| Self::literal_to_overlay_value(value, date_system));
15622 let row0 = row.saturating_sub(1) as usize;
15623 let col0 = col.saturating_sub(1) as usize;
15624 let asheet = self
15625 .arrow_sheets
15626 .sheet_mut(sheet)
15627 .expect("ArrowSheet must exist");
15628
15629 let cur_cols = asheet.columns.len();
15630 if col0 >= cur_cols {
15631 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15632 }
15633 if row0 >= asheet.nrows as usize {
15634 if asheet.columns.is_empty() {
15635 asheet.insert_columns(0, 1);
15636 }
15637 asheet.ensure_row_capacity(row0 + 1);
15638 }
15639
15640 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15641 return;
15642 };
15643 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15644 return;
15645 };
15646
15647 if let Some(ov) = ov_opt {
15648 let _ = ch.overlay.set(in_off, ov);
15649 } else {
15650 let _ = ch.overlay.remove(in_off);
15651 }
15652 }
15653
15654 fn set_computed_overlay_cell_raw(
15655 &mut self,
15656 sheet: &str,
15657 row: u32,
15658 col: u32,
15659 value: Option<LiteralValue>,
15660 ) {
15661 if !(self.config.arrow_storage_enabled
15662 && self.config.delta_overlay_enabled
15663 && self.config.write_formula_overlay_enabled)
15664 {
15665 return;
15666 }
15667
15668 self.ensure_arrow_sheet(sheet);
15669 let date_system = self.arrow_sheet_date_system(sheet);
15670 let ov_opt = value
15671 .as_ref()
15672 .map(|value| Self::literal_to_overlay_value(value, date_system));
15673 let row0 = row.saturating_sub(1) as usize;
15674 let col0 = col.saturating_sub(1) as usize;
15675 let asheet = self
15676 .arrow_sheets
15677 .sheet_mut(sheet)
15678 .expect("ArrowSheet must exist");
15679
15680 let cur_cols = asheet.columns.len();
15681 if col0 >= cur_cols {
15682 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15683 }
15684 if row0 >= asheet.nrows as usize {
15685 if asheet.columns.is_empty() {
15686 asheet.insert_columns(0, 1);
15687 }
15688 asheet.ensure_row_capacity(row0 + 1);
15689 }
15690
15691 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15692 return;
15693 };
15694 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15695 return;
15696 };
15697
15698 let delta = if let Some(ov) = ov_opt {
15699 ch.computed_overlay.set(in_off, ov)
15700 } else {
15701 ch.computed_overlay.remove(in_off)
15702 };
15703 self.adjust_computed_overlay_bytes(delta);
15704 }
15705
15706 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
15707 use crate::engine::ArrowOp;
15708
15709 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
15710 Box::new(batch.ops.iter().rev())
15711 } else {
15712 Box::new(batch.ops.iter())
15713 };
15714
15715 for op in iter {
15716 match op {
15717 ArrowOp::SetDeltaCell {
15718 sheet_id,
15719 row0,
15720 col0,
15721 old,
15722 new,
15723 } => {
15724 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15725 let v = if undo { old.clone() } else { new.clone() };
15726 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15727 }
15728 ArrowOp::SetComputedCell {
15729 sheet_id,
15730 row0,
15731 col0,
15732 old,
15733 new,
15734 } => {
15735 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15736 let v = if undo { old.clone() } else { new.clone() };
15737 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15738 }
15739 ArrowOp::RestoreComputedRect {
15740 sheet_id,
15741 sr0,
15742 sc0,
15743 er0,
15744 ec0,
15745 old,
15746 new,
15747 } => {
15748 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15749 let vals = if undo { old } else { new };
15750 let height = (*er0).saturating_sub(*sr0) as usize + 1;
15751 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
15752 for r in 0..height {
15753 for c in 0..width {
15754 let v = vals
15755 .get(r)
15756 .and_then(|row| row.get(c))
15757 .cloned()
15758 .unwrap_or(LiteralValue::Empty);
15759 self.set_computed_overlay_cell_raw(
15760 &sheet,
15761 *sr0 + 1 + r as u32,
15762 *sc0 + 1 + c as u32,
15763 Some(v),
15764 );
15765 }
15766 }
15767 }
15768 ArrowOp::InsertRows {
15769 sheet_id,
15770 before0,
15771 count,
15772 } => {
15773 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15774 self.ensure_arrow_sheet(&sheet);
15775 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15776 if undo {
15777 asheet.delete_rows(*before0 as usize, *count as usize);
15778 } else {
15779 asheet.insert_rows(*before0 as usize, *count as usize);
15780 }
15781 }
15782 }
15783 ArrowOp::InsertCols {
15784 sheet_id,
15785 before0,
15786 count,
15787 } => {
15788 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15789 self.ensure_arrow_sheet(&sheet);
15790 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15791 if undo {
15792 asheet.delete_columns(*before0 as usize, *count as usize);
15793 } else {
15794 asheet.insert_columns(*before0 as usize, *count as usize);
15795 }
15796 }
15797 }
15798 }
15799 }
15800 }
15801
15802 fn record_spill_ops_into_arrow_undo(
15803 &mut self,
15804 undo: &mut crate::engine::ArrowUndoBatch,
15805 events: &[crate::engine::ChangeEvent],
15806 ) {
15807 use crate::engine::ChangeEvent;
15808 use formualizer_common::LiteralValue;
15809
15810 #[allow(clippy::type_complexity)]
15811 let rect_from_snapshot =
15812 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
15813 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
15814 if snap.target_cells.is_empty() {
15815 return None;
15816 }
15817 let sheet_id = snap.target_cells[0].sheet_id;
15818 let sr0 = snap.target_cells[0].coord.row();
15819 let sc0 = snap.target_cells[0].coord.col();
15820 if snap.values.is_empty() || snap.values[0].is_empty() {
15821 return None;
15822 }
15823 let h = snap.values.len() as u32;
15824 let w = snap.values[0].len() as u32;
15825 let er0 = sr0.saturating_add(h.saturating_sub(1));
15826 let ec0 = sc0.saturating_add(w.saturating_sub(1));
15827 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
15828 };
15829
15830 for ev in events {
15831 match ev {
15832 ChangeEvent::SpillCommitted { old, new, .. } => {
15833 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
15834 let old_vals = if let Some(old_snap) = old {
15835 rect_from_snapshot(old_snap)
15836 .map(|(_, _, _, _, _, v)| v)
15837 .unwrap_or_else(|| {
15838 vec![
15839 vec![LiteralValue::Empty; new_vals[0].len()];
15840 new_vals.len()
15841 ]
15842 })
15843 } else {
15844 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
15845 };
15846 undo.record_restore_computed_rect(
15847 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15848 );
15849 }
15850 }
15851 ChangeEvent::SpillCleared { old, .. } => {
15852 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
15853 let new_vals =
15854 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
15855 undo.record_restore_computed_rect(
15856 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15857 );
15858 }
15859 }
15860 _ => {}
15861 }
15862 }
15863 }
15864
15865 fn mirror_value_to_computed_overlay(
15870 &mut self,
15871 sheet: &str,
15872 row: u32,
15873 col: u32,
15874 value: &LiteralValue,
15875 ) {
15876 if !(self.config.arrow_storage_enabled
15877 && self.config.delta_overlay_enabled
15878 && self.config.write_formula_overlay_enabled)
15879 {
15880 return;
15881 }
15882 if self.computed_overlay_mirroring_disabled {
15883 return;
15884 }
15885
15886 let date_system = self.arrow_sheet_date_system(sheet);
15887 let ov = Self::literal_to_overlay_value(value, date_system);
15888 self.write_computed_overlay_value_0based(
15889 sheet,
15890 row.saturating_sub(1),
15891 col.saturating_sub(1),
15892 ov,
15893 );
15894 }
15895
15896 fn write_computed_overlay_value_0based(
15897 &mut self,
15898 sheet: &str,
15899 row0: u32,
15900 col0: u32,
15901 value: OverlayValue,
15902 ) {
15903 if !(self.config.arrow_storage_enabled
15904 && self.config.delta_overlay_enabled
15905 && self.config.write_formula_overlay_enabled)
15906 {
15907 return;
15908 }
15909 if self.computed_overlay_mirroring_disabled {
15910 return;
15911 }
15912
15913 self.ensure_arrow_sheet(sheet);
15914
15915 let row0 = row0 as usize;
15916 let col0 = col0 as usize;
15917 let asheet = self
15918 .arrow_sheets
15919 .sheet_mut(sheet)
15920 .expect("ArrowSheet must exist");
15921
15922 let cur_cols = asheet.columns.len();
15923 if col0 >= cur_cols {
15924 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15925 }
15926
15927 if row0 >= asheet.nrows as usize {
15928 if asheet.columns.is_empty() {
15929 asheet.insert_columns(0, 1);
15930 }
15931 asheet.ensure_row_capacity(row0 + 1);
15932 }
15933
15934 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15935 return;
15936 };
15937 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15938 return;
15939 };
15940
15941 let delta = ch.computed_overlay.set_scalar(in_off, value);
15942 self.adjust_computed_overlay_bytes(delta);
15943
15944 if let Some(cap) = self.config.max_overlay_memory_bytes
15945 && self.computed_overlay_bytes_estimate > cap
15946 {
15947 self.disable_computed_overlay_mirroring_due_to_budget(cap);
15948 }
15949 }
15950
15951 pub(crate) fn plan_computed_write_coalescing(
15952 &self,
15953 buffer: &ComputedWriteBuffer,
15954 ) -> ComputedWriteCoalescingPlan {
15955 self.plan_computed_write_coalescing_from_writes(buffer.writes().iter().cloned())
15956 }
15957
15958 fn plan_owned_computed_write_coalescing(
15959 &self,
15960 writes: Vec<ComputedWrite>,
15961 ) -> ComputedWriteCoalescingPlan {
15962 self.plan_computed_write_coalescing_from_writes(writes)
15963 }
15964
15965 fn plan_computed_write_coalescing_from_writes(
15966 &self,
15967 writes: impl IntoIterator<Item = ComputedWrite>,
15968 ) -> ComputedWriteCoalescingPlan {
15969 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
15970 BTreeMap::new();
15971 let mut input_cells = 0usize;
15972
15973 for write in writes {
15974 match write {
15975 ComputedWrite::Cell {
15976 seq,
15977 sheet_id,
15978 row0,
15979 col0,
15980 value,
15981 } => {
15982 input_cells = input_cells.saturating_add(1);
15983 self.push_computed_write_plan_entry(
15984 &mut groups,
15985 seq,
15986 sheet_id,
15987 row0,
15988 col0,
15989 value,
15990 );
15991 }
15992 ComputedWrite::Rect {
15993 seq,
15994 sheet_id,
15995 sr0,
15996 sc0,
15997 values,
15998 } => {
15999 for (r_off, row) in values.into_iter().enumerate() {
16000 for (c_off, value) in row.into_iter().enumerate() {
16001 input_cells = input_cells.saturating_add(1);
16002 self.push_computed_write_plan_entry(
16003 &mut groups,
16004 seq,
16005 sheet_id,
16006 sr0.saturating_add(r_off as u32),
16007 sc0.saturating_add(c_off as u32),
16008 value,
16009 );
16010 }
16011 }
16012 }
16013 }
16014 }
16015
16016 let mut plan = ComputedWriteCoalescingPlan {
16017 chunks: Vec::with_capacity(groups.len()),
16018 input_cells,
16019 coalesced_cells: 0,
16020 overwritten_cells: 0,
16021 };
16022 for (key, entries) in groups {
16023 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(key, entries);
16024 plan.coalesced_cells = plan
16025 .coalesced_cells
16026 .saturating_add(chunk_plan.entries.len());
16027 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
16028 plan.chunks.push(chunk_plan);
16029 }
16030 debug_assert_eq!(
16031 plan.input_cells,
16032 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
16033 );
16034 plan
16035 }
16036
16037 fn push_computed_write_plan_entry(
16038 &self,
16039 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
16040 seq: u64,
16041 sheet_id: SheetId,
16042 row0: u32,
16043 col0: u32,
16044 value: OverlayValue,
16045 ) {
16046 let (chunk_idx, chunk_start_row0, row_in_chunk) =
16047 self.locate_computed_write_chunk(sheet_id, row0);
16048 let key = ComputedWriteChunkKey {
16049 sheet_id,
16050 col0,
16051 chunk_idx,
16052 chunk_start_row0,
16053 };
16054 groups
16055 .entry(key)
16056 .or_default()
16057 .push(ComputedWriteChunkEntryPlan {
16058 row_in_chunk,
16059 seq,
16060 value,
16061 });
16062 }
16063
16064 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
16065 let sheet_name = self.graph.sheet_name(sheet_id);
16066 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
16067 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
16068 }
16069 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
16070 }
16071
16072 fn locate_row_in_sheet_for_computed_write_plan(
16073 sheet: &crate::arrow_store::ArrowSheet,
16074 row0: usize,
16075 ) -> (usize, u32, usize) {
16076 if row0 < sheet.nrows as usize
16077 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
16078 {
16079 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
16080 return (chunk_idx, chunk_start as u32, row_in_chunk);
16081 }
16082
16083 let chunk_rows = sheet.chunk_rows.max(1);
16084 if sheet.chunk_starts.is_empty() {
16085 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
16086 }
16087
16088 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
16089 let mut chunk_start = sheet.chunk_starts[chunk_idx];
16090 while chunk_start.saturating_add(chunk_rows) <= row0 {
16091 chunk_idx = chunk_idx.saturating_add(1);
16092 chunk_start = chunk_start.saturating_add(chunk_rows);
16093 }
16094 (
16095 chunk_idx,
16096 chunk_start as u32,
16097 row0.saturating_sub(chunk_start),
16098 )
16099 }
16100
16101 fn locate_row_in_empty_sheet_for_computed_write_plan(
16102 row0: usize,
16103 chunk_rows: usize,
16104 ) -> (usize, u32, usize) {
16105 let chunk_rows = chunk_rows.max(1);
16106 let chunk_idx = row0 / chunk_rows;
16107 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
16108 (
16109 chunk_idx,
16110 chunk_start as u32,
16111 row0.saturating_sub(chunk_start),
16112 )
16113 }
16114
16115 #[cfg(test)]
16116 pub(crate) fn debug_plan_computed_write_coalescing(
16117 &self,
16118 buffer: &ComputedWriteBuffer,
16119 ) -> ComputedWriteCoalescingPlan {
16120 self.plan_computed_write_coalescing(buffer)
16121 }
16122
16123 fn record_computed_write_buffer_delta(
16124 &self,
16125 buffer: &ComputedWriteBuffer,
16126 delta: &mut DeltaCollector,
16127 ) {
16128 if delta.mode == DeltaMode::Off {
16129 return;
16130 }
16131 let mut final_values = BTreeMap::new();
16132 for write in buffer.writes() {
16133 match write {
16134 ComputedWrite::Cell {
16135 sheet_id,
16136 row0,
16137 col0,
16138 value,
16139 ..
16140 } => {
16141 final_values.insert((*sheet_id, *row0, *col0), value.clone());
16142 }
16143 ComputedWrite::Rect {
16144 sheet_id,
16145 sr0,
16146 sc0,
16147 values,
16148 ..
16149 } => {
16150 for (row_offset, row) in values.iter().enumerate() {
16151 for (col_offset, value) in row.iter().enumerate() {
16152 final_values.insert(
16153 (
16154 *sheet_id,
16155 sr0.saturating_add(row_offset as u32),
16156 sc0.saturating_add(col_offset as u32),
16157 ),
16158 value.clone(),
16159 );
16160 }
16161 }
16162 }
16163 }
16164 }
16165 for ((sheet_id, row0, col0), value) in final_values {
16166 let old = self
16167 .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
16168 .unwrap_or(LiteralValue::Empty);
16169 if old != value.to_literal_for(self.config.date_system) {
16170 delta.record_cell(sheet_id, row0, col0);
16171 }
16172 }
16173 }
16174
16175 pub(crate) fn flush_computed_write_buffer(
16176 &mut self,
16177 buffer: &mut ComputedWriteBuffer,
16178 ) -> Result<(), ExcelError> {
16179 if buffer.is_empty() {
16180 return Ok(());
16181 }
16182
16183 self.resource_checkpoint(0)?;
16188 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
16189 let plan = self.plan_owned_computed_write_coalescing(buffer.take_writes());
16190 self.flush_computed_write_plan(plan);
16191 self.observe_evaluation_commit_window(commit_started);
16192
16193 Ok(())
16194 }
16195
16196 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
16197 for chunk in plan.chunks {
16198 self.flush_computed_write_chunk_plan(chunk);
16199 }
16200 }
16201
16202 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
16203 match &chunk.shape {
16204 ComputedWriteChunkPlanShape::Point => {
16205 self.flush_computed_write_chunk_plan_as_points(chunk);
16206 }
16207 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
16208 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
16209 }
16210 ComputedWriteChunkPlanShape::DenseRange { .. } => {
16211 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16212 }
16213 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
16214 if Self::should_emit_computed_run_fragment(*len, *runs) {
16215 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
16216 } else {
16217 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16218 }
16219 }
16220 }
16221 }
16222
16223 #[inline]
16224 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
16225 runs <= len / 2
16226 }
16227
16228 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
16229 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
16230 for entry in chunk.entries {
16231 let row0 = chunk
16232 .chunk_start_row0
16233 .saturating_add(entry.row_in_chunk as u32);
16234 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
16235 }
16236 }
16237
16238 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
16239 &mut self,
16240 chunk: ComputedWriteChunkPlan,
16241 ) {
16242 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
16243 let sheet_id = chunk.sheet_id;
16244 let col0 = chunk.col0;
16245 let chunk_idx = chunk.chunk_idx;
16246 let chunk_start_row0 = chunk.chunk_start_row0;
16247 let items: Vec<(usize, OverlayValue)> = chunk
16248 .entries
16249 .into_iter()
16250 .map(|entry| (entry.row_in_chunk, entry.value))
16251 .collect();
16252 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
16253 items,
16254 point_estimate,
16255 ) {
16256 Some(Ok(fragment)) => {
16257 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
16258 }
16259 Some(Err(cells)) => {
16260 self.flush_computed_overlay_cells_as_points(
16261 sheet_id,
16262 col0,
16263 chunk_start_row0,
16264 cells,
16265 );
16266 }
16267 None => {}
16268 }
16269 }
16270
16271 #[inline]
16272 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
16273 chunk
16274 .entries
16275 .iter()
16276 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
16277 .fold(0usize, usize::saturating_add)
16278 }
16279
16280 fn flush_computed_overlay_cells_as_points(
16281 &mut self,
16282 sheet_id: SheetId,
16283 col0: u32,
16284 chunk_start_row0: u32,
16285 cells: Vec<(usize, OverlayValue)>,
16286 ) {
16287 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16288 for (row_in_chunk, value) in cells {
16289 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
16290 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
16291 }
16292 }
16293
16294 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16295 if chunk.entries.is_empty() {
16296 return;
16297 }
16298 let start = chunk.entries[0].row_in_chunk;
16299 let values: Vec<OverlayValue> =
16300 chunk.entries.into_iter().map(|entry| entry.value).collect();
16301 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
16302 self.apply_computed_overlay_fragment(
16303 chunk.sheet_id,
16304 chunk.col0,
16305 chunk.chunk_idx,
16306 fragment,
16307 );
16308 }
16309 }
16310
16311 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16312 if chunk.entries.is_empty() {
16313 return;
16314 }
16315 let start = chunk.entries[0].row_in_chunk;
16316 let values: Vec<OverlayValue> =
16317 chunk.entries.into_iter().map(|entry| entry.value).collect();
16318 if let Some(fragment) = OverlayFragment::run_range(start, values) {
16319 self.apply_computed_overlay_fragment(
16320 chunk.sheet_id,
16321 chunk.col0,
16322 chunk.chunk_idx,
16323 fragment,
16324 );
16325 }
16326 }
16327
16328 fn apply_computed_overlay_fragment(
16329 &mut self,
16330 sheet_id: SheetId,
16331 col0: u32,
16332 chunk_idx: usize,
16333 fragment: OverlayFragment,
16334 ) {
16335 if !(self.config.arrow_storage_enabled
16336 && self.config.delta_overlay_enabled
16337 && self.config.write_formula_overlay_enabled)
16338 {
16339 return;
16340 }
16341 if self.computed_overlay_mirroring_disabled {
16342 return;
16343 }
16344
16345 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16346 self.ensure_arrow_sheet(&sheet_name);
16347
16348 let col0 = col0 as usize;
16349 let asheet = self
16350 .arrow_sheets
16351 .sheet_mut(&sheet_name)
16352 .expect("ArrowSheet must exist");
16353
16354 let cur_cols = asheet.columns.len();
16355 if col0 >= cur_cols {
16356 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16357 }
16358
16359 let start_row0 = asheet
16360 .chunk_starts
16361 .get(chunk_idx)
16362 .copied()
16363 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
16364 let required_rows =
16365 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
16366 if required_rows > asheet.nrows as usize {
16367 if asheet.columns.is_empty() {
16368 asheet.insert_columns(0, 1);
16369 }
16370 asheet.ensure_row_capacity(required_rows);
16371 }
16372
16373 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
16374 return;
16375 };
16376 let delta = ch.computed_overlay.apply_fragment(fragment);
16377 self.adjust_computed_overlay_bytes(delta);
16378
16379 if let Some(cap) = self.config.max_overlay_memory_bytes
16380 && self.computed_overlay_bytes_estimate > cap
16381 {
16382 self.disable_computed_overlay_mirroring_due_to_budget(cap);
16383 }
16384 }
16385
16386 #[inline]
16387 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
16388 if delta >= 0 {
16389 self.computed_overlay_bytes_estimate = self
16390 .computed_overlay_bytes_estimate
16391 .saturating_add(delta as usize);
16392 } else {
16393 self.computed_overlay_bytes_estimate = self
16394 .computed_overlay_bytes_estimate
16395 .saturating_sub((-delta) as usize);
16396 }
16397 }
16398
16399 fn clear_all_computed_overlays(&mut self) {
16400 let mut freed_total = 0usize;
16401 for sh in self.arrow_sheets.sheets.iter_mut() {
16402 for col in sh.columns.iter_mut() {
16403 for ch in col.chunks.iter_mut() {
16404 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16405 }
16406 for ch in col.sparse_chunks.values_mut() {
16407 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16408 }
16409 }
16410 }
16411 self.computed_overlay_bytes_estimate = self
16412 .computed_overlay_bytes_estimate
16413 .saturating_sub(freed_total);
16414 }
16415
16416 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
16417 self.compact_all_computed_overlays();
16420 }
16421
16422 fn compact_all_computed_overlays(&mut self) {
16425 let mut freed_total = 0usize;
16426 for sheet in self.arrow_sheets.sheets.iter_mut() {
16427 for col_idx in 0..sheet.columns.len() {
16428 let num_dense = sheet.columns[col_idx].chunks.len();
16430 for ch_idx in 0..num_dense {
16431 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
16432 }
16433 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
16435 .sparse_chunks
16436 .keys()
16437 .copied()
16438 .collect();
16439 for ch_idx in sparse_keys {
16440 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
16441 }
16442 }
16443 }
16444 self.computed_overlay_bytes_estimate = self
16445 .computed_overlay_bytes_estimate
16446 .saturating_sub(freed_total);
16447 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16448 }
16449
16450 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
16451 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
16452 }
16453
16454 fn record_vertex_value_to_overlay(
16455 &mut self,
16456 vertex_id: VertexId,
16457 value: &LiteralValue,
16458 computed_writes: Option<&mut ComputedWriteBuffer>,
16459 ) -> Result<(), ExcelError> {
16460 if !(self.config.arrow_storage_enabled
16461 && self.config.delta_overlay_enabled
16462 && self.config.write_formula_overlay_enabled)
16463 {
16464 return Ok(());
16465 }
16466 if self.computed_overlay_mirroring_disabled {
16467 return Ok(());
16468 }
16469 if !matches!(
16470 self.graph.get_vertex_kind(vertex_id),
16471 VertexKind::FormulaScalar | VertexKind::FormulaArray
16472 ) {
16473 return Ok(());
16474 }
16475 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
16476 return Ok(());
16477 };
16478 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
16479 let date_system = self.arrow_sheet_date_system(&sheet_name);
16480 let ov = Self::literal_to_overlay_value(value, date_system);
16481 if let Some(buffer) = computed_writes {
16482 buffer.push_cell(cell.sheet_id, cell.coord.row(), cell.coord.col(), ov);
16483 if self.should_flush_computed_write_buffer(buffer) {
16484 self.flush_computed_write_buffer(buffer)?;
16485 }
16486 } else {
16487 self.write_computed_overlay_value_0based(
16488 &sheet_name,
16489 cell.coord.row(),
16490 cell.coord.col(),
16491 ov,
16492 );
16493 }
16494 Ok(())
16495 }
16496
16497 #[inline]
16498 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
16499 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
16500 if cap == 0 {
16501 return false;
16502 }
16503 self.computed_overlay_bytes_estimate
16504 .saturating_add(buffer.estimated_bytes())
16505 > cap
16506 })
16507 }
16508
16509 pub fn overlay_memory_usage(&self) -> usize {
16511 self.computed_overlay_bytes_estimate
16512 }
16513
16514 #[cfg(test)]
16515 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
16516 self.overlay_compactions
16517 }
16518
16519 #[cfg(test)]
16520 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
16521 let mut total = 0usize;
16522 for sheet in &self.arrow_sheets.sheets {
16523 for column in &sheet.columns {
16524 for chunk in &column.chunks {
16525 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16526 }
16527 for chunk in column.sparse_chunks.values() {
16528 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16529 }
16530 }
16531 }
16532 self.computed_overlay_bytes_estimate = total;
16533 total
16534 }
16535
16536 fn resolve_sheet_locator_for_write(
16537 &mut self,
16538 loc: formualizer_common::SheetLocator<'_>,
16539 current_sheet: &str,
16540 ) -> Result<SheetId, ExcelError> {
16541 Ok(match loc {
16542 formualizer_common::SheetLocator::Id(id) => id,
16543 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
16544 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
16545 })
16546 }
16547
16548 fn resolve_sheet_locator_for_read(
16549 &self,
16550 loc: formualizer_common::SheetLocator<'_>,
16551 current_sheet: &str,
16552 ) -> Result<SheetId, ExcelError> {
16553 match loc {
16554 formualizer_common::SheetLocator::Id(id) => Ok(id),
16555 formualizer_common::SheetLocator::Name(name) => self
16556 .graph
16557 .sheet_id(name.as_ref())
16558 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16559 formualizer_common::SheetLocator::Current => self
16560 .graph
16561 .sheet_id(current_sheet)
16562 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16563 }
16564 }
16565
16566 pub fn set_cell_value(
16568 &mut self,
16569 sheet: &str,
16570 row: u32,
16571 col: u32,
16572 value: LiteralValue,
16573 ) -> Result<(), ExcelError> {
16574 self.observe_function_semantic_epoch()?;
16575 let sheet_existed = self.graph.sheet_id(sheet).is_some();
16576 let sheet_id = self.graph.sheet_id_mut(sheet);
16577 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16578 let replaced_formula =
16579 self.graph
16580 .get_vertex_id_for_address(&cell_ref)
16581 .is_some_and(|vertex| {
16582 matches!(
16583 self.graph.get_vertex_kind(*vertex),
16584 VertexKind::FormulaScalar | VertexKind::FormulaArray
16585 )
16586 });
16587 self.demote_span_containing_cell_for_write(
16588 sheet_id,
16589 row.saturating_sub(1),
16590 col.saturating_sub(1),
16591 )
16592 .map_err(Self::editor_error_to_excel)?;
16593 self.graph.set_cell_value(sheet, row, col, value.clone())?;
16594 self.record_formula_plane_changed_cell(sheet, row, col);
16595 if !sheet_existed || replaced_formula {
16596 self.mark_topology_edited();
16597 }
16598 self.mirror_value_to_overlay(sheet, row, col, &value);
16600 self.snapshot_id
16602 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16603 self.has_edited = true;
16604 Ok(())
16605 }
16606
16607 fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
16612 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16613 return;
16614 }
16615 let sheet_id = self.graph.sheet_id_mut(sheet);
16616 self.record_formula_plane_structural_change(StructuralScope::Cell {
16617 sheet: sheet_id,
16618 row: row.saturating_sub(1),
16619 col: col.saturating_sub(1),
16620 });
16621 }
16622
16623 fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
16624 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16625 return;
16626 }
16627
16628 match event {
16629 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16630 self.record_formula_plane_structural_change(StructuralScope::Cell {
16631 sheet: addr.sheet_id,
16632 row: addr.coord.row(),
16633 col: addr.coord.col(),
16634 });
16635 }
16636 ChangeEvent::SpillCommitted { new, .. } => {
16637 if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
16638 self.record_formula_plane_structural_change(scope);
16639 }
16640 }
16641 ChangeEvent::SpillCleared { old, .. } => {
16642 if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
16643 self.record_formula_plane_structural_change(scope);
16644 }
16645 }
16646 ChangeEvent::DefineName { .. }
16647 | ChangeEvent::UpdateName { .. }
16648 | ChangeEvent::DeleteName { .. }
16649 | ChangeEvent::NamedRangeAdjusted { .. } => {
16650 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
16654 }
16655 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
16656 }
16659 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
16660 self.record_formula_plane_structural_change(StructuralScope::Region(
16661 Region::whole_row(*sheet_id, *row0),
16662 ));
16663 }
16664 ChangeEvent::AddVertex { .. }
16665 | ChangeEvent::RemoveVertex { .. }
16666 | ChangeEvent::EdgeAdded { .. }
16667 | ChangeEvent::EdgeRemoved { .. }
16668 | ChangeEvent::CompoundStart { .. }
16669 | ChangeEvent::CompoundEnd { .. }
16670 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
16671 }
16672 }
16673
16674 fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
16675 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16676 return;
16677 }
16678
16679 match scope {
16680 StructuralScope::Cell { sheet, row, col } => {
16681 self.graph
16682 .mark_formula_region_dirty(Region::point(sheet, row, col));
16683 }
16684 StructuralScope::Region(region) => {
16685 self.graph.mark_formula_region_dirty(region);
16686 }
16687 StructuralScope::Sheet(sheet_id) => {
16688 self.graph
16689 .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
16690 }
16691 StructuralScope::RemovedSheet(sheet_id) => {
16692 let removed_refs = {
16693 let authority = self.graph.formula_authority();
16694 authority
16695 .active_span_refs()
16696 .into_iter()
16697 .filter(|span_ref| {
16698 authority
16699 .plane
16700 .spans
16701 .get(*span_ref)
16702 .map(|span| span.sheet_id == sheet_id)
16703 .unwrap_or(false)
16704 })
16705 .collect::<Vec<_>>()
16706 };
16707
16708 {
16709 let authority = self.graph.formula_authority_mut();
16710 for span_ref in removed_refs {
16711 authority.plane.remove_span(span_ref);
16712 }
16713 let _ = authority.rebuild_indexes();
16714 }
16715 self.graph
16716 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16717 }
16718 StructuralScope::OpaqueGlobal => {
16719 let _ = self.graph.formula_authority_mut().rebuild_indexes();
16720 self.graph
16721 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16722 }
16723 StructuralScope::AllSheets => {
16724 let _ = self.graph.formula_authority_mut().rebuild_indexes();
16728 }
16729 }
16730 }
16731
16732 fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
16733 let first = cells.first()?;
16734 let sheet_id = first.sheet_id;
16735 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
16736 return Some(StructuralScope::OpaqueGlobal);
16737 }
16738 let mut row_start = first.coord.row();
16739 let mut row_end = row_start;
16740 let mut col_start = first.coord.col();
16741 let mut col_end = col_start;
16742 for cell in cells.iter().skip(1) {
16743 row_start = row_start.min(cell.coord.row());
16744 row_end = row_end.max(cell.coord.row());
16745 col_start = col_start.min(cell.coord.col());
16746 col_end = col_end.max(cell.coord.col());
16747 }
16748 Some(StructuralScope::Region(Region::rect(
16749 sheet_id, row_start, row_end, col_start, col_end,
16750 )))
16751 }
16752
16753 pub fn set_cell_value_ref(
16754 &mut self,
16755 cell: formualizer_common::SheetCellRef<'_>,
16756 current_sheet: &str,
16757 value: LiteralValue,
16758 ) -> Result<(), ExcelError> {
16759 let owned = cell.into_owned();
16760 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16761 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16762 self.set_cell_value(
16763 &sheet_name,
16764 owned.coord.row() + 1,
16765 owned.coord.col() + 1,
16766 value,
16767 )
16768 }
16769
16770 pub fn set_cell_formula_ref(
16771 &mut self,
16772 cell: formualizer_common::SheetCellRef<'_>,
16773 current_sheet: &str,
16774 ast: ASTNode,
16775 ) -> Result<(), ExcelError> {
16776 let owned = cell.into_owned();
16777 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16778 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16779 self.set_cell_formula(
16780 &sheet_name,
16781 owned.coord.row() + 1,
16782 owned.coord.col() + 1,
16783 ast,
16784 )
16785 }
16786
16787 pub fn get_cell_value_ref(
16788 &self,
16789 cell: formualizer_common::SheetCellRef<'_>,
16790 current_sheet: &str,
16791 ) -> Result<Option<LiteralValue>, ExcelError> {
16792 let owned = cell.into_owned();
16793 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
16794 let sheet_name = self.graph.sheet_name(sheet_id);
16795 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
16796 }
16797
16798 pub fn resolve_range_view_sheet_ref<'c>(
16799 &'c self,
16800 r: &formualizer_common::SheetRef<'_>,
16801 current_sheet: &str,
16802 ) -> Result<RangeView<'c>, ExcelError> {
16803 use formualizer_common::SheetLocator;
16804
16805 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
16806 match loc {
16807 SheetLocator::Current => Ok(None),
16808 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
16809 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
16810 }
16811 };
16812
16813 let rt = match r {
16814 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
16815 sheet: sheet_to_opt_name(cell.sheet.clone())?,
16816 row: cell.coord.row() + 1,
16817 col: cell.coord.col() + 1,
16818 row_abs: cell.coord.row_abs(),
16819 col_abs: cell.coord.col_abs(),
16820 },
16821 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
16822 sheet: sheet_to_opt_name(range.sheet.clone())?,
16823 start_row: range.start_row.map(|b| b.index + 1),
16824 start_col: range.start_col.map(|b| b.index + 1),
16825 end_row: range.end_row.map(|b| b.index + 1),
16826 end_col: range.end_col.map(|b| b.index + 1),
16827 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
16828 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
16829 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
16830 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
16831 },
16832 };
16833
16834 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
16835 }
16836
16837 pub fn set_cell_formula(
16839 &mut self,
16840 sheet: &str,
16841 row: u32,
16842 col: u32,
16843 ast: ASTNode,
16844 ) -> Result<(), ExcelError> {
16845 self.observe_function_semantic_epoch()?;
16846 let sheet_id = self.graph.sheet_id_mut(sheet);
16847 self.demote_span_containing_cell_for_write(
16848 sheet_id,
16849 row.saturating_sub(1),
16850 col.saturating_sub(1),
16851 )
16852 .map_err(Self::editor_error_to_excel)?;
16853 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16854 let ingested = {
16855 let mut pipeline = self.ingest_pipeline();
16856 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
16857 };
16858 self.graph.set_cell_formula_with_plan(
16859 sheet,
16860 row,
16861 col,
16862 ingested.ast_id,
16863 &ingested.dep_plan,
16864 ingested.dep_plan.volatile,
16865 ingested.dep_plan.dynamic,
16866 )?;
16867 self.record_formula_plane_changed_cell(sheet, row, col);
16868
16869 self.clear_delta_overlay_cell(sheet, row, col);
16874
16875 self.mark_topology_edited();
16877 Ok(())
16878 }
16879
16880 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
16882 where
16883 I: IntoIterator<Item = (u32, u32, ASTNode)>,
16884 {
16885 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
16886 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
16887 let sheet_id = self.graph.sheet_id_mut(sheet);
16888 let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
16889 let placement =
16890 PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
16891 self.graph
16892 .formula_authority()
16893 .plane
16894 .spans
16895 .find_at(placement)
16896 .is_some()
16897 });
16898 if writes_inside_active_span {
16899 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
16900 .map_err(Self::editor_error_to_excel)?;
16901 }
16902 let ingested = {
16903 let mut pipeline = self.ingest_pipeline();
16904 let inputs = collected.into_iter().map(|(row, col, ast)| {
16905 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16906 (FormulaAstInput::Tree(ast), placement, None)
16907 });
16908 pipeline.ingest_batch(inputs)?
16909 };
16910 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
16911 .into_iter()
16912 .map(|formula| {
16913 (
16914 formula.placement.coord.row() + 1,
16915 formula.placement.coord.col() + 1,
16916 formula.ast_id,
16917 formula.dep_plan,
16918 )
16919 })
16920 .collect();
16921 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
16922 for (row, col) in edited_cells {
16923 self.record_formula_plane_changed_cell(sheet, row, col);
16924 }
16925 if n > 0 {
16927 self.mark_topology_edited();
16928 }
16929 Ok(n)
16930 }
16931
16932 #[inline]
16933 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
16934 match v {
16935 LiteralValue::Empty => None,
16936 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
16937 other => Some(other),
16938 }
16939 }
16940
16941 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16943 self.read_cell_value(sheet, row, col)
16944 .and_then(Self::normalize_public_cell_read)
16945 }
16946
16947 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16949 let asheet = self.sheet_store().sheet(sheet)?;
16950 let r0 = row.saturating_sub(1) as usize;
16951 let c0 = col.saturating_sub(1) as usize;
16952 let v = asheet.get_cell_value(r0, c0);
16953 if matches!(v, LiteralValue::Empty) {
16954 None
16955 } else {
16956 Some(v)
16957 }
16958 }
16959
16960 pub(crate) fn read_range_values(
16962 &self,
16963 sheet: &str,
16964 sr: u32,
16965 sc: u32,
16966 er: u32,
16967 ec: u32,
16968 ) -> RangeView<'_> {
16969 let Some(asheet) = self.sheet_store().sheet(sheet) else {
16970 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
16971 };
16972 if er < sr || ec < sc {
16973 return asheet.range_view(1, 1, 0, 0);
16974 }
16975 let sr0 = sr.saturating_sub(1) as usize;
16976 let sc0 = sc.saturating_sub(1) as usize;
16977 let er0 = er.saturating_sub(1) as usize;
16978 let ec0 = ec.saturating_sub(1) as usize;
16979 asheet.range_view(sr0, sc0, er0, ec0)
16980 }
16981
16982 pub(crate) fn formula_plane_span_ast_at(
16983 &self,
16984 span_ref: FormulaSpanRef,
16985 placement: PlacementCoord,
16986 ) -> Option<ASTNode> {
16987 let authority = self.graph.formula_authority();
16988 let span = authority.plane.spans.get(span_ref)?;
16989 let relocation = span.ast_relocation;
16990 let mut ast = self
16991 .graph
16992 .data_store()
16993 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
16994 if let Some(binding_set_id) = span.binding_set_id {
16995 let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
16996 if !binding_set.is_single_literal_binding() {
16997 let binding =
16998 binding_set.literal_bindings_for_placement(&span.domain, placement)?;
16999 ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
17000 }
17001 }
17002 let row = placement.row.checked_add(1)?;
17003 let col = placement.col.checked_add(1)?;
17004 let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
17005 let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
17006 relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
17007 }
17008
17009 pub fn get_cell(
17011 &self,
17012 sheet: &str,
17013 row: u32,
17014 col: u32,
17015 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
17016 let v = self.get_cell_value(sheet, row, col);
17017 let sheet_id = self.graph.sheet_id(sheet)?;
17018 let coord = Coord::from_excel(row, col, true, true);
17019 let cell = CellRef::new(sheet_id, coord);
17020 let placement =
17021 crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
17022 let handle = self
17023 .graph
17024 .formula_authority()
17025 .plane
17026 .resolve_formula_at(placement, None);
17027 match handle.resolution {
17028 crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
17029 let ast = self.formula_plane_span_ast_at(span, placement);
17033 return Some((ast, v));
17034 }
17035 crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
17036 let ast = self
17037 .graph
17038 .formula_authority()
17039 .plane
17040 .formula_overlay
17041 .get(overlay_ref)
17042 .and_then(|overlay| match overlay.kind {
17043 crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
17044 template_id,
17045 ) => self
17046 .graph
17047 .formula_authority()
17048 .plane
17049 .templates
17050 .get(template_id)
17051 .and_then(|template| {
17052 self.graph
17053 .data_store()
17054 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
17055 }),
17056 _ => None,
17057 });
17058 return Some((ast, v));
17059 }
17060 _ => {}
17061 }
17062
17063 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
17064 let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
17065 self.graph
17066 .data_store()
17067 .retrieve_ast(ast_id, self.graph.sheet_reg())
17068 });
17069 Some((ast, v))
17070 } else if v.is_some() {
17071 Some((None, v))
17072 } else {
17073 None
17074 }
17075 }
17076
17077 pub fn begin_batch(&mut self) {
17079 self.graph.begin_batch();
17080 }
17081
17082 pub fn end_batch(&mut self) {
17084 self.graph.end_batch();
17085 }
17086
17087 pub fn begin_deferred_dirty(&mut self) {
17096 self.graph.begin_deferred_dirty();
17097 }
17098
17099 pub fn end_deferred_dirty(&mut self) {
17102 let _ = self.graph.end_deferred_dirty();
17103 }
17104
17105 pub fn dirty_propagation_visits(&self) -> u64 {
17109 self.graph.dirty_propagation_visits()
17110 }
17111
17112 #[inline]
17115 fn record_cell_if_changed(
17116 delta: &mut DeltaCollector,
17117 cell: &CellRef,
17118 old: &LiteralValue,
17119 new: &LiteralValue,
17120 ) {
17121 if old != new {
17122 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17123 }
17124 }
17125
17126 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17127 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
17128 engine.observe_function_semantic_epoch()?;
17129 engine.resource_checkpoint(1)?;
17131 let is_formula = engine.graph.vertex_exists(vertex_id)
17132 && matches!(
17133 engine.graph.get_vertex_kind(vertex_id),
17134 VertexKind::FormulaScalar | VertexKind::FormulaArray
17135 );
17136 if is_formula {
17137 engine.begin_evaluation_request();
17138 engine.graph.flush_pending_edge_deltas();
17139 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
17140 vertex_id,
17141 )];
17142 if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17143 && engine.graph.formula_authority().active_span_count() > 0
17144 {
17145 engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
17146 } else {
17147 engine.evaluate_legacy_target_roots(&roots, None)?;
17148 }
17149 } else if engine.config.formula_plane_mode
17150 == FormulaPlaneMode::AuthoritativeExperimental
17151 && engine.graph.formula_authority().active_span_count() > 0
17152 {
17153 engine.begin_evaluation_request();
17154 engine.graph.flush_pending_edge_deltas();
17155 engine.evaluate_authoritative_formula_plane(None, None)?;
17156 }
17157 engine.evaluate_vertex_impl(vertex_id, None)
17158 })
17159 }
17160
17161 fn evaluate_vertex_impl(
17162 &mut self,
17163 vertex_id: VertexId,
17164 delta: Option<&mut DeltaCollector>,
17165 ) -> Result<LiteralValue, ExcelError> {
17166 if !self.graph.vertex_exists(vertex_id) {
17170 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17171 .with_message(format!("Vertex not found: {vertex_id:?}")));
17172 }
17173 if self.active_resource_ledger.is_some()
17174 && matches!(
17175 self.graph.get_vertex_kind(vertex_id),
17176 VertexKind::FormulaScalar | VertexKind::FormulaArray
17177 )
17178 {
17179 let value = self
17180 .evaluate_vertex_immutable(vertex_id)
17181 .unwrap_or_else(LiteralValue::Error);
17182 let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
17183 self.resource_checkpoint(0)?;
17185 let mut delta = delta;
17186 for effect in &effects {
17187 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
17188 }
17189 return Ok(value);
17190 }
17191
17192 let mut delta = delta;
17193
17194 let kind = self.graph.get_vertex_kind(vertex_id);
17196 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17197
17198 let ast_id = match kind {
17199 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17200 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
17201 ast_id
17202 } else {
17203 return Ok(LiteralValue::Number(0.0));
17204 }
17205 }
17206 VertexKind::Empty | VertexKind::Cell => {
17207 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17208 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17209 let row = cell_ref.coord.row() + 1;
17210 let col = cell_ref.coord.col() + 1;
17211 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17212 return Ok(v);
17213 }
17214 }
17215 return Ok(LiteralValue::Number(0.0));
17216 }
17217 VertexKind::NamedScalar => {
17218 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
17219 return Ok(value);
17220 }
17221 VertexKind::NamedArray => {
17222 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
17223 return Ok(value);
17224 }
17225 VertexKind::InfiniteRange
17226 | VertexKind::Range
17227 | VertexKind::External
17228 | VertexKind::Table => {
17229 return Ok(LiteralValue::Number(0.0));
17231 }
17232 };
17233
17234 let sheet_name = self.graph.sheet_name(sheet_id);
17236 let cell_ref = self
17237 .graph
17238 .get_cell_ref(vertex_id)
17239 .expect("cell ref for vertex");
17240 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17241
17242 let result =
17243 interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
17244
17245 match result {
17247 Ok(cv) => {
17248 let result_literal =
17249 crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
17250 match result_literal {
17251 LiteralValue::Array(rows) => {
17252 self.graph
17254 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
17255 let anchor = self
17257 .graph
17258 .get_cell_ref(vertex_id)
17259 .expect("cell ref for vertex");
17260 let sheet_id = anchor.sheet_id;
17261 let h = rows.len() as u32;
17262 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
17263
17264 let spill_cells = (h as u64).saturating_mul(w as u64);
17266 if spill_cells > self.config.spill.max_spill_cells as u64 {
17267 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17268 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17269 .with_message("SpillTooLarge")
17270 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17271 expected_rows: h,
17272 expected_cols: w,
17273 });
17274 let spill_val = LiteralValue::Error(spill_err.clone());
17275 if let Some(d) = delta.as_deref_mut() {
17276 let old = self
17277 .read_cell_value(
17278 self.graph.sheet_name(anchor.sheet_id),
17279 anchor.coord.row() + 1,
17280 anchor.coord.col() + 1,
17281 )
17282 .unwrap_or(LiteralValue::Empty);
17283 if old != spill_val {
17284 d.record_cell(
17285 anchor.sheet_id,
17286 anchor.coord.row(),
17287 anchor.coord.col(),
17288 );
17289 }
17290 }
17291 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17292 if self.config.arrow_storage_enabled
17293 && self.config.delta_overlay_enabled
17294 && self.config.write_formula_overlay_enabled
17295 {
17296 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17297 self.mirror_value_to_computed_overlay(
17298 &sheet_name,
17299 anchor.coord.row() + 1,
17300 anchor.coord.col() + 1,
17301 &spill_val,
17302 );
17303 }
17304 return Ok(spill_val);
17305 }
17306 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
17310 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
17311 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
17312 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17313 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17314 .with_message("Spill exceeds sheet bounds")
17315 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17316 expected_rows: h,
17317 expected_cols: w,
17318 });
17319 let spill_val = LiteralValue::Error(spill_err.clone());
17320 if let Some(d) = delta.as_deref_mut() {
17321 let old = self
17322 .read_cell_value(
17323 self.graph.sheet_name(anchor.sheet_id),
17324 anchor.coord.row() + 1,
17325 anchor.coord.col() + 1,
17326 )
17327 .unwrap_or(LiteralValue::Empty);
17328 if old != spill_val {
17329 d.record_cell(
17330 anchor.sheet_id,
17331 anchor.coord.row(),
17332 anchor.coord.col(),
17333 );
17334 }
17335 }
17336 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17337 if self.config.arrow_storage_enabled
17338 && self.config.delta_overlay_enabled
17339 && self.config.write_formula_overlay_enabled
17340 {
17341 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17342 self.mirror_value_to_computed_overlay(
17343 &sheet_name,
17344 anchor.coord.row() + 1,
17345 anchor.coord.col() + 1,
17346 &spill_val,
17347 );
17348 }
17349 return Ok(spill_val);
17350 }
17351 let mut targets = Vec::new();
17352 for r in 0..h {
17353 for c in 0..w {
17354 targets.push(self.graph.make_cell_ref_internal(
17355 sheet_id,
17356 anchor.coord.row() + r,
17357 anchor.coord.col() + c,
17358 ));
17359 }
17360 }
17361
17362 match self.spill_mgr.reserve(
17364 vertex_id,
17365 anchor,
17366 SpillShape { rows: h, cols: w },
17367 SpillMeta {
17368 epoch: self.recalc_epoch,
17369 config: self.config.spill,
17370 },
17371 ) {
17372 Ok(()) => {
17373 if let Err(e) = self.commit_spill_and_mirror(
17377 vertex_id,
17378 &targets,
17379 rows.clone(),
17380 delta.as_deref_mut(),
17381 None,
17382 ) {
17383 self.clear_spill_projection_and_mirror(
17385 vertex_id,
17386 delta.as_deref_mut(),
17387 );
17388 if let Some(d) = delta.as_deref_mut() {
17389 let old = self
17390 .read_cell_value(
17391 self.graph.sheet_name(anchor.sheet_id),
17392 anchor.coord.row() + 1,
17393 anchor.coord.col() + 1,
17394 )
17395 .unwrap_or(LiteralValue::Empty);
17396 let new = LiteralValue::Error(e.clone());
17397 if old != new {
17398 d.record_cell(
17399 anchor.sheet_id,
17400 anchor.coord.row(),
17401 anchor.coord.col(),
17402 );
17403 }
17404 }
17405 let err_val = LiteralValue::Error(e.clone());
17406 self.graph.update_vertex_value(vertex_id, err_val.clone());
17407 if self.config.arrow_storage_enabled
17408 && self.config.delta_overlay_enabled
17409 && self.config.write_formula_overlay_enabled
17410 {
17411 let sheet_name =
17412 self.graph.sheet_name(anchor.sheet_id).to_string();
17413 self.mirror_value_to_computed_overlay(
17414 &sheet_name,
17415 anchor.coord.row() + 1,
17416 anchor.coord.col() + 1,
17417 &err_val,
17418 );
17419 }
17420 return Ok(err_val);
17421 }
17422 let top_left = rows
17424 .first()
17425 .and_then(|r| r.first())
17426 .cloned()
17427 .unwrap_or(LiteralValue::Empty);
17428 self.graph.update_vertex_value(vertex_id, top_left.clone());
17429 Ok(top_left)
17430 }
17431 Err(e) => {
17432 self.clear_spill_projection_and_mirror(
17433 vertex_id,
17434 delta.as_deref_mut(),
17435 );
17436 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17437 .with_message(
17438 e.message.unwrap_or_else(|| "Spill blocked".to_string()),
17439 )
17440 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17441 expected_rows: h,
17442 expected_cols: w,
17443 });
17444 let spill_val = LiteralValue::Error(spill_err.clone());
17445 if let Some(d) = delta.as_deref_mut() {
17446 let old = self
17447 .read_cell_value(
17448 self.graph.sheet_name(anchor.sheet_id),
17449 anchor.coord.row() + 1,
17450 anchor.coord.col() + 1,
17451 )
17452 .unwrap_or(LiteralValue::Empty);
17453 if old != spill_val {
17454 d.record_cell(
17455 anchor.sheet_id,
17456 anchor.coord.row(),
17457 anchor.coord.col(),
17458 );
17459 }
17460 }
17461 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17462 if self.config.arrow_storage_enabled
17463 && self.config.delta_overlay_enabled
17464 && self.config.write_formula_overlay_enabled
17465 {
17466 let sheet_name =
17467 self.graph.sheet_name(anchor.sheet_id).to_string();
17468 self.mirror_value_to_computed_overlay(
17469 &sheet_name,
17470 anchor.coord.row() + 1,
17471 anchor.coord.col() + 1,
17472 &spill_val,
17473 );
17474 }
17475 Ok(spill_val)
17476 }
17477 }
17478 }
17479 other => {
17480 let spill_cells = self
17482 .graph
17483 .spill_cells_for_anchor(vertex_id)
17484 .map(|cells| cells.to_vec())
17485 .unwrap_or_default();
17486 if let Some(d) = delta.as_deref_mut()
17487 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17488 {
17489 if spill_cells.is_empty() {
17490 let old = self
17491 .read_cell_value(
17492 self.graph.sheet_name(anchor.sheet_id),
17493 anchor.coord.row() + 1,
17494 anchor.coord.col() + 1,
17495 )
17496 .unwrap_or(LiteralValue::Empty);
17497 if old != other {
17498 d.record_cell(
17499 anchor.sheet_id,
17500 anchor.coord.row(),
17501 anchor.coord.col(),
17502 );
17503 }
17504 } else {
17505 for cell in spill_cells.iter() {
17506 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17507 let old = self
17508 .get_cell_value(
17509 sheet_name,
17510 cell.coord.row() + 1,
17511 cell.coord.col() + 1,
17512 )
17513 .unwrap_or(LiteralValue::Empty);
17514 let new = if cell.sheet_id == anchor.sheet_id
17515 && cell.coord.row() == anchor.coord.row()
17516 && cell.coord.col() == anchor.coord.col()
17517 {
17518 other.clone()
17519 } else {
17520 LiteralValue::Empty
17521 };
17522 Self::record_cell_if_changed(d, cell, &old, &new);
17523 }
17524 }
17525 }
17526 self.graph.clear_spill_region(vertex_id);
17527 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17528 self.record_formula_plane_structural_change(scope);
17529 }
17530 if self.config.arrow_storage_enabled
17531 && self.config.delta_overlay_enabled
17532 && self.config.write_formula_overlay_enabled
17533 {
17534 let empty = LiteralValue::Empty;
17535 for cell in spill_cells.iter() {
17536 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17537 self.mirror_value_to_computed_overlay(
17538 &sheet_name,
17539 cell.coord.row() + 1,
17540 cell.coord.col() + 1,
17541 &empty,
17542 );
17543 }
17544 }
17545 self.graph.update_vertex_value(vertex_id, other.clone());
17546 if self.config.arrow_storage_enabled
17548 && self.config.delta_overlay_enabled
17549 && self.config.write_formula_overlay_enabled
17550 {
17551 let anchor = self
17552 .graph
17553 .get_cell_ref(vertex_id)
17554 .expect("cell ref for vertex");
17555 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17556 self.mirror_value_to_computed_overlay(
17557 &sheet_name,
17558 anchor.coord.row() + 1,
17559 anchor.coord.col() + 1,
17560 &other,
17561 );
17562 }
17563 Ok(other)
17564 }
17565 }
17566 }
17567 Err(e) => {
17568 let spill_cells = self
17571 .graph
17572 .spill_cells_for_anchor(vertex_id)
17573 .map(|cells| cells.to_vec())
17574 .unwrap_or_default();
17575 let err_val = LiteralValue::Error(e.clone());
17576 if let Some(d) = delta
17577 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17578 {
17579 if spill_cells.is_empty() {
17580 let old = self
17581 .read_cell_value(
17582 self.graph.sheet_name(anchor.sheet_id),
17583 anchor.coord.row() + 1,
17584 anchor.coord.col() + 1,
17585 )
17586 .unwrap_or(LiteralValue::Empty);
17587 if old != err_val {
17588 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
17589 }
17590 } else {
17591 for cell in spill_cells.iter() {
17592 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17593 let old = self
17594 .get_cell_value(
17595 sheet_name,
17596 cell.coord.row() + 1,
17597 cell.coord.col() + 1,
17598 )
17599 .unwrap_or(LiteralValue::Empty);
17600 let new = if cell.sheet_id == anchor.sheet_id
17601 && cell.coord.row() == anchor.coord.row()
17602 && cell.coord.col() == anchor.coord.col()
17603 {
17604 err_val.clone()
17605 } else {
17606 LiteralValue::Empty
17607 };
17608 Self::record_cell_if_changed(d, cell, &old, &new);
17609 }
17610 }
17611 }
17612 self.graph.clear_spill_region(vertex_id);
17613 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17614 self.record_formula_plane_structural_change(scope);
17615 }
17616 if self.config.arrow_storage_enabled
17617 && self.config.delta_overlay_enabled
17618 && self.config.write_formula_overlay_enabled
17619 {
17620 let empty = LiteralValue::Empty;
17621 for cell in spill_cells.iter() {
17622 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17623 self.mirror_value_to_computed_overlay(
17624 &sheet_name,
17625 cell.coord.row() + 1,
17626 cell.coord.col() + 1,
17627 &empty,
17628 );
17629 }
17630 }
17631 self.graph.update_vertex_value(vertex_id, err_val.clone());
17632 if self.config.arrow_storage_enabled
17633 && self.config.delta_overlay_enabled
17634 && self.config.write_formula_overlay_enabled
17635 {
17636 let anchor = self
17637 .graph
17638 .get_cell_ref(vertex_id)
17639 .expect("cell ref for vertex");
17640 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17641 self.mirror_value_to_computed_overlay(
17642 &sheet_name,
17643 anchor.coord.row() + 1,
17644 anchor.coord.col() + 1,
17645 &err_val,
17646 );
17647 }
17648 Ok(err_val)
17649 }
17650 }
17651 }
17652
17653 fn evaluate_named_scalar(
17654 &mut self,
17655 vertex_id: VertexId,
17656 sheet_id: SheetId,
17657 ) -> Result<LiteralValue, ExcelError> {
17658 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17659 ExcelError::new(ExcelErrorKind::Name)
17660 .with_message("Named range metadata missing".to_string())
17661 })?;
17662
17663 match &named_range.definition {
17664 NamedDefinition::Cell(cell_ref) => {
17665 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17666 let row = cell_ref.coord.row() + 1;
17667 let col = cell_ref.coord.col() + 1;
17668
17669 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
17670 && matches!(
17671 self.graph.get_vertex_kind(dep_vertex),
17672 VertexKind::FormulaScalar | VertexKind::FormulaArray
17673 )
17674 {
17675 let value = self.evaluate_vertex(dep_vertex)?;
17677 self.graph.update_vertex_value(vertex_id, value.clone());
17678 Ok(value)
17679 } else {
17680 let value = self
17681 .get_cell_value(sheet_name, row, col)
17682 .unwrap_or(LiteralValue::Empty);
17683 self.graph.update_vertex_value(vertex_id, value.clone());
17684 Ok(value)
17685 }
17686 }
17687 NamedDefinition::Literal(v) => {
17688 let out = v.clone();
17689 self.graph.update_vertex_value(vertex_id, out.clone());
17690 Ok(out)
17691 }
17692 NamedDefinition::Formula { ast, .. } => {
17693 let context_sheet = match named_range.scope {
17694 NameScope::Sheet(id) => id,
17695 NameScope::Workbook => sheet_id,
17696 };
17697 let sheet_name = self.graph.sheet_name(context_sheet);
17698 let cell_ref = self
17699 .graph
17700 .get_cell_ref(vertex_id)
17701 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17702 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17703 match interpreter.evaluate_ast(ast) {
17704 Ok(cv) => {
17705 let value = cv.into_literal();
17706 match value {
17707 LiteralValue::Array(_) => {
17708 let err = ExcelError::new(ExcelErrorKind::NImpl)
17709 .with_message("Array result in scalar named range".to_string());
17710 let err_val = LiteralValue::Error(err.clone());
17711 self.graph.update_vertex_value(vertex_id, err_val.clone());
17712 Ok(err_val)
17713 }
17714 other => {
17715 self.graph.update_vertex_value(vertex_id, other.clone());
17716 Ok(other)
17717 }
17718 }
17719 }
17720 Err(err) => {
17721 let err_val = LiteralValue::Error(err.clone());
17722 self.graph.update_vertex_value(vertex_id, err_val.clone());
17723 Ok(err_val)
17724 }
17725 }
17726 }
17727 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17728 .with_message("Range-valued name evaluated as scalar".to_string())),
17729 }
17730 }
17731
17732 fn evaluate_named_array(
17733 &mut self,
17734 vertex_id: VertexId,
17735 sheet_id: SheetId,
17736 ) -> Result<LiteralValue, ExcelError> {
17737 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17738 ExcelError::new(ExcelErrorKind::Name)
17739 .with_message("Named range metadata missing".to_string())
17740 })?;
17741
17742 let out = match &named_range.definition {
17743 NamedDefinition::Range(range_ref) => {
17744 if range_ref.start.sheet_id != range_ref.end.sheet_id {
17745 return Err(ExcelError::new(ExcelErrorKind::Ref)
17746 .with_message("Named range cannot span sheets".to_string()));
17747 }
17748
17749 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17750 let sr0 = range_ref.start.coord.row();
17751 let sc0 = range_ref.start.coord.col();
17752 let er0 = range_ref.end.coord.row();
17753 let ec0 = range_ref.end.coord.col();
17754 if sr0 > er0 || sc0 > ec0 {
17755 return Err(ExcelError::new(ExcelErrorKind::Ref)
17756 .with_message("Invalid named range bounds".to_string()));
17757 }
17758
17759 let h = (er0 - sr0 + 1) as usize;
17760 let w = (ec0 - sc0 + 1) as usize;
17761 let cell_count = (h as u64).saturating_mul(w as u64);
17762 if cell_count > self.config.spill.max_spill_cells as u64 {
17763 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17764 "Named range too large to materialize as an array".to_string(),
17765 ));
17766 }
17767
17768 let mut rows = Vec::with_capacity(h);
17769 for r0 in sr0..=er0 {
17770 let mut row = Vec::with_capacity(w);
17771 for c0 in sc0..=ec0 {
17772 let v = self
17773 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
17774 .unwrap_or(LiteralValue::Empty);
17775 row.push(v);
17776 }
17777 rows.push(row);
17778 }
17779 LiteralValue::Array(rows)
17780 }
17781 NamedDefinition::Cell(cell_ref) => {
17782 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17783 let row = cell_ref.coord.row() + 1;
17784 let col = cell_ref.coord.col() + 1;
17785 let v = self
17786 .get_cell_value(sheet_name, row, col)
17787 .unwrap_or(LiteralValue::Empty);
17788 LiteralValue::Array(vec![vec![v]])
17789 }
17790 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
17791 NamedDefinition::Formula { ast, .. } => {
17792 let context_sheet = match named_range.scope {
17793 NameScope::Sheet(id) => id,
17794 NameScope::Workbook => sheet_id,
17795 };
17796 let sheet_name = self.graph.sheet_name(context_sheet);
17797 let cell_ref = self
17798 .graph
17799 .get_cell_ref(vertex_id)
17800 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17801 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17802 match interpreter.evaluate_ast(ast) {
17803 Ok(cv) => {
17804 let v = cv.into_literal();
17805 match v {
17806 LiteralValue::Array(_) => v,
17807 other => LiteralValue::Array(vec![vec![other]]),
17808 }
17809 }
17810 Err(err) => LiteralValue::Error(err),
17811 }
17812 }
17813 };
17814
17815 self.graph.update_vertex_value(vertex_id, out.clone());
17816 Ok(out)
17817 }
17818
17819 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
17820 crate::engine::ResourceLedgerError::Exhausted(
17821 formualizer_common::ResourceExhaustionDetail {
17822 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
17823 limit: limit as u64,
17824 observed: limit.saturating_add(1) as u64,
17825 request_id: self
17826 .active_evaluation_resource_request
17827 .as_ref()
17828 .map(|stats| stats.request_id),
17829 },
17830 )
17831 .into_excel_error()
17832 .with_message(format!("{context} did not converge after {limit} replans"))
17833 }
17834
17835 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
17836 matches!(
17837 &error.extra,
17838 formualizer_common::ExcelErrorExtra::PreparationStale {
17839 reason: formualizer_common::PreparationStaleReason::Semantic
17840 | formualizer_common::PreparationStaleReason::Provider
17841 }
17842 )
17843 }
17844
17845 fn prepare_graph_for_routed_evaluation(
17846 &mut self,
17847 targets: &[crate::engine::EvaluationTarget],
17848 options: &crate::engine::TargetEvalOptions<'_>,
17849 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
17850 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
17851 let mut retries = 0usize;
17852 loop {
17853 match self.prepare_graph_for_targets_unobserved(targets, options) {
17854 Err(error)
17855 if Self::transient_target_preparation_stale(&error)
17856 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
17857 {
17858 retries = retries.saturating_add(1);
17859 }
17860 result => return result,
17861 }
17862 }
17863 }
17864
17865 fn evaluate_mixed_targets(
17866 &mut self,
17867 targets: &[crate::engine::EvaluationTarget],
17868 delta: Option<&mut DeltaCollector>,
17869 ) -> Result<EvalResult, ExcelError> {
17870 let _source_cache = self.source_cache_session();
17871 let cancel = self.active_cancel_flag.clone();
17872 let options = crate::engine::TargetEvalOptions {
17873 request_id: self
17874 .active_evaluation_resource_request
17875 .as_ref()
17876 .map(|stats| stats.request_id),
17877 cancel,
17878 deadline: None,
17879 budgets: None,
17880 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
17881 };
17882 self.prepare_and_execute_target_recipe(targets, &options, delta)
17883 }
17884
17885 fn prepare_and_execute_target_recipe(
17886 &mut self,
17887 targets: &[crate::engine::EvaluationTarget],
17888 options: &crate::engine::TargetEvalOptions<'_>,
17889 delta: Option<&mut DeltaCollector>,
17890 ) -> Result<EvalResult, ExcelError> {
17891 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
17892 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
17893 }
17894
17895 fn execute_prepared_target_recipe(
17896 &mut self,
17897 targets: &[crate::engine::EvaluationTarget],
17898 scope: &crate::engine::PrepareScope,
17899 delta: Option<&mut DeltaCollector>,
17900 ) -> Result<EvalResult, ExcelError> {
17901 if matches!(scope, crate::engine::PrepareScope::Workbook)
17902 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
17903 {
17904 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
17905 }
17906 let mut roots = self.resolve_target_producers(targets)?;
17907 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
17908 let request_id = self
17909 .active_evaluation_resource_request
17910 .as_ref()
17911 .map(|request| request.request_id);
17912 let root_count = roots.len();
17913 let mut widened_roots =
17914 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
17915 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
17916 let sheet_ids = sheets
17917 .iter()
17918 .filter_map(|sheet| self.graph.sheet_id(sheet))
17919 .collect::<FxHashSet<_>>();
17920 for vertex in self.graph.formula_vertices() {
17921 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
17922 widened_roots
17923 .push(crate::engine::target_preparation::TargetProducer::Legacy(
17924 vertex,
17925 ))
17926 .map_err(|_| {
17927 target_root_allocation_error(widened_roots.len() + 1, request_id)
17928 })?;
17929 }
17930 }
17931 let authority = self.graph.formula_authority();
17932 for span_ref in authority.active_span_refs() {
17933 let Some(span) = authority.plane.spans.get(span_ref) else {
17934 continue;
17935 };
17936 if sheet_ids.contains(&span.sheet_id) {
17937 widened_roots
17938 .push(crate::engine::target_preparation::TargetProducer::Span {
17939 span_ref,
17940 demanded: Region::from_domain(span.result_region.domain()),
17941 })
17942 .map_err(|_| {
17943 target_root_allocation_error(widened_roots.len() + 1, request_id)
17944 })?;
17945 }
17946 }
17947 roots = widened_roots.into_vec();
17948 }
17949 self.begin_evaluation_request();
17950 self.graph.flush_pending_edge_deltas();
17951 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
17952 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17953 && self.graph.formula_authority().active_span_count() > 0
17954 {
17955 if workbook_scope {
17956 self.evaluate_authoritative_formula_plane(None, delta)
17957 } else {
17958 self.evaluate_authoritative_formula_plane_targets(&roots, delta)
17959 }
17960 } else {
17961 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
17962 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
17963 }
17964 if workbook_scope {
17965 if let Some(delta) = delta {
17966 self.evaluate_all_with_delta_collector(delta)
17967 } else {
17968 self.evaluate_all_legacy_impl()
17969 }
17970 } else {
17971 self.evaluate_legacy_target_roots(&roots, delta)
17972 }
17973 }
17974 }
17975
17976 fn legacy_coordinate_targets(
17977 &mut self,
17978 targets: &[(&str, u32, u32)],
17979 ) -> Vec<crate::engine::EvaluationTarget> {
17980 targets
17981 .iter()
17982 .map(|(sheet, row, col)| {
17983 self.graph.sheet_id_mut(sheet);
17986 crate::engine::EvaluationTarget::Cell {
17987 sheet: (*sheet).to_string(),
17988 row: *row,
17989 col: *col,
17990 }
17991 })
17992 .collect()
17993 }
17994
17995 pub fn evaluate_targets(
17997 &mut self,
17998 targets: &[crate::engine::EvaluationTarget],
17999 ) -> Result<EvalResult, ExcelError> {
18000 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18001 engine.observe_function_semantic_epoch()?;
18002 engine.validate_deterministic_mode()?;
18003 engine.evaluate_mixed_targets(targets, None)
18004 })
18005 }
18006
18007 pub fn evaluate_targets_with_options(
18009 &mut self,
18010 targets: &[crate::engine::EvaluationTarget],
18011 options: crate::engine::TargetEvalOptions<'_>,
18012 ) -> Result<EvalResult, ExcelError> {
18013 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18014 engine.active_cancel_flag = options.cancel.clone();
18015 engine.active_evaluation_deadline = options.deadline;
18016 let result = (|| {
18017 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
18018 engine.observe_function_semantic_epoch()?;
18019 engine.validate_deterministic_mode()?;
18020 let _source_cache = engine.source_cache_session();
18021 engine.prepare_and_execute_target_recipe(targets, &options, None)
18022 })();
18023 engine.active_cancel_flag = None;
18024 engine.active_evaluation_deadline = None;
18025 result
18026 })
18027 }
18028
18029 pub fn evaluate_targets_with_delta(
18031 &mut self,
18032 targets: &[crate::engine::EvaluationTarget],
18033 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
18034 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
18035 engine.observe_function_semantic_epoch()?;
18036 engine.validate_deterministic_mode()?;
18037 let mut collector = DeltaCollector::new(DeltaMode::Cells);
18038 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
18039 Ok((result, collector.finish_target()))
18040 })
18041 }
18042
18043 pub fn evaluate_until(
18045 &mut self,
18046 targets: &[(&str, u32, u32)],
18047 ) -> Result<EvalResult, ExcelError> {
18048 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18049 engine.evaluate_until_unobserved(targets)
18050 })
18051 }
18052
18053 fn evaluate_until_unobserved(
18054 &mut self,
18055 targets: &[(&str, u32, u32)],
18056 ) -> Result<EvalResult, ExcelError> {
18057 self.observe_function_semantic_epoch()?;
18058 let targets = self.legacy_coordinate_targets(targets);
18059 self.evaluate_mixed_targets(&targets, None)
18060 }
18061
18062 fn evaluate_until_with_delta_collector(
18063 &mut self,
18064 targets: &[(&str, u32, u32)],
18065 delta: &mut DeltaCollector,
18066 ) -> Result<EvalResult, ExcelError> {
18067 let targets = self.legacy_coordinate_targets(targets);
18068 self.evaluate_mixed_targets(&targets, Some(delta))
18069 }
18070
18071 fn evaluate_legacy_target_roots(
18072 &mut self,
18073 roots: &[crate::engine::target_preparation::TargetProducer],
18074 mut delta: Option<&mut DeltaCollector>,
18075 ) -> Result<EvalResult, ExcelError> {
18076 use crate::engine::target_preparation::TargetProducer;
18077 let start = crate::instant::FzInstant::now();
18078 let root_vertices = roots
18079 .iter()
18080 .filter_map(|root| match root {
18081 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
18082 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
18083 })
18084 .collect::<Vec<_>>();
18085 let mut computed_vertices = 0usize;
18086 let mut cycle_errors = 0usize;
18087 let mut replans = 0usize;
18088 const MAX_REPLAN: usize = 5;
18089 loop {
18090 let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
18091 if precedents_to_eval.is_empty() {
18092 break;
18093 }
18094 let scheduler = Scheduler::new(&self.graph);
18095 let schedule =
18096 scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
18097 for &unit in &schedule.units {
18098 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
18099 match unit {
18100 ScheduleUnit::Cycle(index) => {
18101 if self.handle_cycle_unit(
18102 schedule.unit_cycle(index),
18103 delta.as_deref_mut(),
18104 None,
18105 None,
18106 )? > 0
18107 {
18108 cycle_errors = cycle_errors.saturating_add(1);
18109 }
18110 }
18111 ScheduleUnit::Layer(index) => {
18112 let layer = schedule.unit_layer(index);
18113 let evaluated = if let Some(delta) = delta.as_deref_mut() {
18114 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18115 self.evaluate_layer_parallel_with_delta(layer, delta)?
18116 } else {
18117 self.evaluate_layer_sequential_with_delta(layer, delta)?
18118 }
18119 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18120 self.evaluate_layer_parallel(layer)?
18121 } else {
18122 self.evaluate_layer_sequential(layer)?
18123 };
18124 computed_vertices = computed_vertices.saturating_add(evaluated);
18125 }
18126 }
18127 }
18128 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
18129 self.resource_checkpoint(0)?;
18130 self.graph.clear_dirty_flags(&precedents_to_eval);
18131 for vertex in &changed {
18132 self.graph.set_dirty(*vertex, true);
18133 }
18134 if changed.is_empty() {
18135 break;
18136 }
18137 if replans >= MAX_REPLAN {
18138 return Err(self.replan_exhausted_error(
18139 MAX_REPLAN,
18140 "targeted legacy dynamic dependency evaluation",
18141 ));
18142 }
18143 replans = replans.saturating_add(1);
18144 }
18145 self.redirty_for_next_recalc();
18146 Ok(EvalResult {
18147 computed_vertices,
18148 cycle_errors,
18149 elapsed: start.elapsed(),
18150 })
18151 }
18152
18153 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
18155 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
18156 return Err(
18157 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
18158 "compatibility recalculation plans require all staged formulas to be prepared",
18159 ),
18160 );
18161 }
18162 let key = self.recalc_plan_key();
18163 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
18164 vertices.sort_unstable();
18165 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
18166 let schedule = if vertices.is_empty() {
18167 crate::engine::Schedule {
18168 units: Vec::new(),
18169 layers: Vec::new(),
18170 cycles: Vec::new(),
18171 }
18172 } else {
18173 self.create_evaluation_schedule_uncached(&vertices)?.0
18174 };
18175 self.validate_recalc_plan_key(&key)?;
18176 Ok(RecalcPlan {
18177 key,
18178 kind: RecalcPlanKind::CompatibilityFull {
18179 schedule,
18180 has_dynamic_refs,
18181 },
18182 })
18183 }
18184
18185 pub fn build_recalc_plan_for_targets(
18187 &mut self,
18188 targets: &[crate::engine::EvaluationTarget],
18189 ) -> Result<RecalcPlan, ExcelError> {
18190 self.build_recalc_plan_for_targets_with_options(
18191 targets,
18192 crate::engine::TargetEvalOptions::default(),
18193 )
18194 }
18195
18196 pub fn build_recalc_plan_for_targets_with_options(
18197 &mut self,
18198 targets: &[crate::engine::EvaluationTarget],
18199 options: crate::engine::TargetEvalOptions<'_>,
18200 ) -> Result<RecalcPlan, ExcelError> {
18201 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18202 engine.observe_function_semantic_epoch()?;
18203 engine.validate_deterministic_mode()?;
18204 let _source_cache = engine.source_cache_session();
18205 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
18206 engine.graph.flush_pending_edge_deltas();
18207 let topology = if matches!(
18208 preparation.widened_scope,
18209 crate::engine::PrepareScope::Workbook
18210 ) {
18211 RecalcTopology::Workbook
18212 } else {
18213 RecalcTopology::RunLocalRecipe
18214 };
18215 Ok(RecalcPlan {
18216 key: engine.recalc_plan_key(),
18217 kind: RecalcPlanKind::Target {
18218 targets: targets.to_vec(),
18219 scope: preparation.widened_scope,
18220 topology,
18221 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
18222 },
18223 })
18224 })
18225 }
18226
18227 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
18229 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18230 engine.evaluate_recalc_plan_unobserved(plan)
18231 })
18232 }
18233
18234 pub fn evaluate_recalc_plan_with_controls(
18235 &mut self,
18236 plan: &RecalcPlan,
18237 cancel: Option<crate::engine::CancelToken>,
18238 deadline: Option<Instant>,
18239 ) -> Result<EvalResult, ExcelError> {
18240 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18241 engine.active_cancel_flag = cancel.clone();
18242 engine.active_evaluation_deadline = deadline;
18243 let result = engine.evaluate_recalc_plan_unobserved(plan);
18244 engine.active_cancel_flag = None;
18245 engine.active_evaluation_deadline = None;
18246 result
18247 })
18248 }
18249
18250 fn evaluate_recalc_plan_unobserved(
18251 &mut self,
18252 plan: &RecalcPlan,
18253 ) -> Result<EvalResult, ExcelError> {
18254 self.graph.flush_pending_edge_deltas();
18255 self.validate_recalc_plan_key(&plan.key)?;
18256 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
18257 self.validate_deterministic_mode()?;
18258
18259 match &plan.kind {
18260 RecalcPlanKind::Target {
18261 targets,
18262 scope,
18263 topology,
18264 dynamic_policy,
18265 } => {
18266 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
18267 debug_assert_eq!(
18268 matches!(topology, RecalcTopology::Workbook),
18269 matches!(scope, crate::engine::PrepareScope::Workbook)
18270 );
18271 let _source_cache = self.source_cache_session();
18272 self.execute_prepared_target_recipe(targets, scope, None)
18273 }
18274 RecalcPlanKind::CompatibilityFull {
18275 schedule,
18276 has_dynamic_refs,
18277 } => {
18278 let _source_cache = self.source_cache_session();
18279 self.begin_evaluation_request();
18280 if self.graph.formula_authority().active_span_count() > 0 {
18281 return self.evaluate_authoritative_formula_plane_all();
18282 }
18283 if *has_dynamic_refs {
18284 self.virtual_dep_fallback_activations =
18285 self.virtual_dep_fallback_activations.saturating_add(1);
18286 return self.evaluate_all_coordinator();
18287 }
18288
18289 let start = crate::instant::FzInstant::now();
18290 let dirty_vertices = self.graph.get_evaluation_vertices();
18291 if dirty_vertices.is_empty() {
18292 return Ok(EvalResult {
18293 computed_vertices: 0,
18294 cycle_errors: 0,
18295 elapsed: start.elapsed(),
18296 });
18297 }
18298
18299 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
18300 let mut computed_vertices = 0;
18301 let mut cycle_errors = 0;
18302 for &unit in &schedule.units {
18303 self.cancellation_checkpoint(
18304 "Evaluation cancelled before recalculation plan schedule unit",
18305 )?;
18306 match unit {
18307 ScheduleUnit::Cycle(i) => {
18308 let stamped = self.handle_cycle_unit(
18309 schedule.unit_cycle(i),
18310 None,
18311 Some(&dirty_set),
18312 None,
18313 )?;
18314 if stamped > 0 {
18315 cycle_errors += 1;
18316 }
18317 }
18318 ScheduleUnit::Layer(i) => {
18319 let work: Vec<VertexId> = schedule
18320 .unit_layer(i)
18321 .vertices
18322 .iter()
18323 .copied()
18324 .filter(|v| dirty_set.contains(v))
18325 .collect();
18326 if work.is_empty() {
18327 continue;
18328 }
18329 let temp_layer = crate::engine::scheduler::Layer { vertices: work };
18330 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
18331 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
18332 } else {
18333 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
18334 }
18335 }
18336 }
18337 }
18338
18339 self.resource_checkpoint(0)?;
18340 self.graph.clear_dirty_flags(&dirty_vertices);
18341 self.redirty_for_next_recalc();
18342 Ok(EvalResult {
18343 computed_vertices,
18344 cycle_errors,
18345 elapsed: start.elapsed(),
18346 })
18347 }
18348 }
18349 }
18350 fn evaluate_all_legacy_and_ack_dirty(
18351 &mut self,
18352 formula_dirty: FormulaDirtyLease,
18353 ) -> Result<EvalResult, ExcelError> {
18354 let result = self.evaluate_all_legacy_impl()?;
18355 self.checked_ack_formula_dirty_observed(formula_dirty)?;
18356 Ok(result)
18357 }
18358
18359 fn evaluate_formula_plane_capacity_fallback(
18360 &mut self,
18361 mut formula_dirty: FormulaDirtyLease,
18362 owned_dirty_events: &[usize],
18363 selected_span_refs: &[FormulaSpanRef],
18364 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18365 delta: Option<&mut DeltaCollector>,
18366 ) -> Result<EvalResult, ExcelError> {
18367 let leased_len = formula_dirty.len();
18368 if !selected_span_refs.is_empty() {
18369 let materialization_started = crate::instant::FzInstant::now();
18370 let prepared = self
18371 .prepare_formula_span_demotion(selected_span_refs)
18372 .map_err(|error| {
18373 if let FormulaSpanDemotionError::Resource(error) = error {
18374 error
18375 } else {
18376 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18377 "FormulaPlane capacity fallback demotion preparation failed: {error}"
18378 ))
18379 }
18380 })?;
18381 let report = self
18382 .commit_prepared_formula_span_demotion(prepared)
18383 .map_err(|error| {
18384 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18385 "FormulaPlane capacity fallback demotion commit failed: {error}"
18386 ))
18387 })?;
18388 self.observe_materialization(
18389 report.placements_materialized,
18390 false,
18391 materialization_started.elapsed(),
18392 );
18393 }
18394 #[cfg(test)]
18395 {
18396 self.last_formula_plane_span_eval_report = None;
18397 }
18398 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
18399 formula_dirty = extended;
18400 }
18401 let mut acknowledged = owned_dirty_events.to_vec();
18402 acknowledged.extend(leased_len..formula_dirty.len());
18403 let result = if let Some(target_roots) = target_roots {
18404 let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
18405 self.evaluate_legacy_target_roots(&refreshed, delta)?
18406 } else if let Some(delta) = delta {
18407 self.evaluate_all_with_delta_collector(delta)?
18408 } else {
18409 self.evaluate_all_legacy_impl()?
18410 };
18411 self.formula_plane_capacity_bailouts =
18412 self.formula_plane_capacity_bailouts.saturating_add(1);
18413 if target_roots.is_some() {
18414 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
18415 } else {
18416 self.checked_ack_formula_dirty_observed(formula_dirty)?;
18417 }
18418 Ok(result)
18419 }
18420
18421 fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
18422 self.evaluate_authoritative_formula_plane(None, None)
18423 }
18424
18425 fn evaluate_authoritative_formula_plane_targets(
18426 &mut self,
18427 roots: &[crate::engine::target_preparation::TargetProducer],
18428 delta: Option<&mut DeltaCollector>,
18429 ) -> Result<EvalResult, ExcelError> {
18430 self.evaluate_authoritative_formula_plane(Some(roots), delta)
18431 }
18432
18433 fn evaluate_authoritative_formula_plane(
18434 &mut self,
18435 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18436 mut delta: Option<&mut DeltaCollector>,
18437 ) -> Result<EvalResult, ExcelError> {
18438 let mut formula_dirty = self.graph.lease_formula_dirty();
18442 self.observe_dirty_lease_acquired(formula_dirty.is_empty());
18443
18444 if self.graph.formula_authority().active_span_count() == 0 {
18447 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
18448 #[cfg(test)]
18449 {
18450 self.last_formula_plane_span_eval_report = None;
18451 }
18452 if let Some(target_roots) = target_roots {
18453 let result =
18454 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18455 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18456 return Ok(result);
18457 }
18458 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18459 }
18460
18461 if formula_dirty.is_empty() {
18464 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
18465 #[cfg(test)]
18466 {
18467 self.last_formula_plane_span_eval_report = None;
18468 }
18469 if let Some(target_roots) = target_roots {
18470 let result =
18471 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18472 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18473 return Ok(result);
18474 }
18475 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18476 }
18477
18478 let start = crate::instant::FzInstant::now();
18479 let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
18480 let mut workbook_exact_attempted = target_roots.is_none();
18481 let mut prepass_cycle_errors = 0usize;
18484 const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
18485 let mut cycle_demote_iters = 0usize;
18486 let mut include_dirty_regions = true;
18487 let mut retry_whole_spans = Vec::new();
18488 let mut computed_vertices = 0usize;
18489 let mut runtime_replan_iterations = 0usize;
18490 const MAX_RUNTIME_REPLAN: usize = 5;
18491 let mut virtual_telemetry = self
18492 .config
18493 .enable_virtual_dep_telemetry
18494 .then(|| self.start_virtual_dep_telemetry());
18495 'virtual_replan: loop {
18496 self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
18497 let (
18498 schedule,
18499 span_refs_by_id,
18500 plane_epoch,
18501 legacy_vertices,
18502 owned_dirty_events,
18503 island_plan,
18504 ) = loop {
18505 let (
18506 schedule,
18507 span_refs_by_id,
18508 plane_epoch,
18509 legacy_vertices,
18510 owned_dirty_events,
18511 island_plan,
18512 ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
18513 self.build_formula_plane_target_schedule(
18514 &formula_dirty,
18515 &retry_whole_spans,
18516 include_dirty_regions,
18517 target_roots,
18518 )?
18519 } else {
18520 self.build_formula_plane_mixed_schedule(
18521 &formula_dirty,
18522 &retry_whole_spans,
18523 include_dirty_regions,
18524 )?
18525 };
18526 #[cfg(test)]
18527 let mut schedule = schedule;
18528 #[cfg(test)]
18529 if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
18530 let producer = schedule
18531 .layers
18532 .iter()
18533 .flat_map(|layer| layer.work.iter())
18534 .map(|work| work.producer)
18535 .next()
18536 .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
18537 schedule.fallbacks.push(
18538 crate::formula_plane::scheduler::MixedScheduleFallback {
18539 producer,
18540 reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
18541 },
18542 );
18543 }
18544
18545 if schedule.is_authoritative_safe() {
18546 break (
18547 schedule,
18548 span_refs_by_id,
18549 plane_epoch,
18550 legacy_vertices,
18551 owned_dirty_events,
18552 island_plan,
18553 );
18554 }
18555
18556 let has_cycle_fallback = schedule.stats.cycle_count > 0
18561 || schedule
18562 .fallbacks
18563 .iter()
18564 .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
18565 if !has_cycle_fallback {
18566 let selected_span_refs = schedule
18567 .layers
18568 .iter()
18569 .flat_map(|layer| layer.work.iter())
18570 .filter_map(|work| match work.producer {
18571 FormulaProducerId::Span(span_id) => {
18572 span_refs_by_id.get(&span_id).copied()
18573 }
18574 FormulaProducerId::Legacy(_) => None,
18575 })
18576 .collect::<Vec<_>>();
18577
18578 return self.evaluate_formula_plane_capacity_fallback(
18579 formula_dirty,
18580 &owned_dirty_events,
18581 &selected_span_refs,
18582 runtime_target_roots.as_deref(),
18583 delta.as_deref_mut(),
18584 );
18585 }
18586
18587 let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
18597 if !cyclic_spans.is_empty() {
18598 self.demote_cyclic_spans(&cyclic_spans)?;
18599 }
18600
18601 if self.graph.formula_authority().active_span_count() == 0 {
18602 if let Some(target_roots) = runtime_target_roots.as_deref() {
18605 let refreshed =
18606 self.resolve_target_producers_from_existing_roots(target_roots)?;
18607 let result =
18608 self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
18609 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18610 return Ok(result);
18611 }
18612 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18613 }
18614
18615 prepass_cycle_errors =
18619 prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
18620
18621 retry_whole_spans = if runtime_target_roots.is_some() {
18625 let active = self
18626 .graph
18627 .formula_authority()
18628 .active_span_refs()
18629 .into_iter()
18630 .map(|span_ref| (span_ref.id, span_ref))
18631 .collect::<BTreeMap<_, _>>();
18632 let mut refs = schedule
18633 .layers
18634 .iter()
18635 .flat_map(|layer| layer.work.iter())
18636 .filter_map(|work| match work.producer {
18637 FormulaProducerId::Span(id) => active.get(&id).copied(),
18638 FormulaProducerId::Legacy(_) => None,
18639 })
18640 .collect::<Vec<_>>();
18641 refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
18642 match fallback.producer {
18643 FormulaProducerId::Span(id) => active.get(&id).copied(),
18644 FormulaProducerId::Legacy(_) => None,
18645 }
18646 }));
18647 refs.sort_by_key(|span_ref| span_ref.id);
18648 refs.dedup();
18649 refs
18650 } else {
18651 self.graph.formula_authority().active_span_refs()
18652 };
18653 self.graph.mark_formula_spans_dirty(
18654 retry_whole_spans.iter().copied(),
18655 WholeSpanDirtyReason::CycleRetry,
18656 );
18657 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
18662 {
18663 formula_dirty = extended;
18664 #[cfg(test)]
18665 if std::mem::take(
18666 &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
18667 ) {
18668 self.graph.mark_formula_spans_dirty(
18669 retry_whole_spans.iter().copied().take(1),
18670 WholeSpanDirtyReason::GlobalInvalidation,
18671 );
18672 }
18673 }
18674 include_dirty_regions = false;
18675
18676 cycle_demote_iters += 1;
18677 if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
18678 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18679 "FormulaPlane cyclic demotion did not converge".to_string(),
18680 ));
18681 }
18682 };
18683 self.observe_formula_plane_route(
18684 &island_plan,
18685 FormulaPlaneRoutePhase::Planned,
18686 FormulaPlaneRouteTransitionReason::ProvenIsolated,
18687 cycle_demote_iters,
18688 runtime_replan_iterations,
18689 );
18690 if !island_plan.dirty_vertices.is_empty() {
18691 self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
18692 let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
18693 &island_plan.dirty_vertices,
18694 delta.as_deref_mut(),
18695 )?;
18696 computed_vertices = computed_vertices.saturating_add(island_computed);
18697 prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
18698 self.observe_formula_plane_route(
18699 &island_plan,
18700 FormulaPlaneRoutePhase::Executed,
18701 FormulaPlaneRouteTransitionReason::ProvenIsolated,
18702 cycle_demote_iters,
18703 runtime_replan_iterations,
18704 );
18705 self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
18706 }
18707 let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
18708 let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18709
18710 #[cfg(test)]
18711 {
18712 self.last_formula_plane_span_eval_report = None;
18713 }
18714 for layer in schedule.layers {
18715 self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
18716 let mut buffer = ComputedWriteBuffer::default();
18717 let mut sink = SpanComputedWriteSink::new(&mut buffer);
18718 let work_items = layer.work;
18719 let mut work_index = 0usize;
18720 while work_index < work_items.len() {
18721 match work_items[work_index].producer {
18722 FormulaProducerId::Span(span_id) => {
18723 let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
18724 ExcelError::new(ExcelErrorKind::NImpl)
18725 .with_message("FormulaPlane schedule referenced a stale span")
18726 })?;
18727 let sheet_id = {
18728 let authority = self.graph.formula_authority();
18729 let span =
18730 authority.plane.spans.get(span_ref).ok_or_else(|| {
18731 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18732 "FormulaPlane schedule referenced a stale span",
18733 )
18734 })?;
18735 span.sheet_id
18736 };
18737 let current_sheet = self.graph.sheet_name(sheet_id);
18738 let authority = self.graph.formula_authority();
18739 let evaluator = SpanEvaluator::new_with_cancel(
18740 &authority.plane,
18741 self,
18742 current_sheet,
18743 self.graph.data_store(),
18744 self.graph.sheet_reg(),
18745 self.active_cancel_flag.as_ref(),
18746 );
18747 #[cfg(test)]
18748 let mut last_group_report = None;
18749 let mut selected_group_work = 0_u64;
18750 while work_index < work_items.len() {
18751 let FormulaProducerId::Span(group_span_id) =
18752 work_items[work_index].producer
18753 else {
18754 break;
18755 };
18756 let group_span_ref =
18757 *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
18758 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18759 "FormulaPlane schedule referenced a stale span",
18760 )
18761 })?;
18762 let group_sheet_id = {
18763 let authority = self.graph.formula_authority();
18764 let span =
18765 authority.plane.spans.get(group_span_ref).ok_or_else(
18766 || {
18767 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18768 "FormulaPlane schedule referenced a stale span",
18769 )
18770 },
18771 )?;
18772 span.sheet_id
18773 };
18774 if group_sheet_id != sheet_id {
18775 break;
18776 }
18777
18778 let dirty = producer_dirty_to_span_dirty(
18779 work_items[work_index].dirty.clone(),
18780 group_span_ref,
18781 );
18782 let task = SpanEvalTask {
18783 span: group_span_ref,
18784 dirty,
18785 plane_epoch,
18786 };
18787 self.cancellation_checkpoint(
18788 "Evaluation cancelled during FormulaPlane span",
18789 )?;
18790 let report = evaluator.evaluate_task(&task, &mut sink).map_err(
18791 |err| {
18792 if err
18793 == crate::formula_plane::span_eval::SpanEvalError::Cancelled
18794 {
18795 ExcelError::new(ExcelErrorKind::Cancelled).with_message(
18796 "Evaluation cancelled during FormulaPlane span",
18797 )
18798 } else {
18799 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18800 format!(
18801 "FormulaPlane span evaluation failed: {err:?}"
18802 ),
18803 )
18804 }
18805 },
18806 )?;
18807 #[cfg(test)]
18808 {
18809 last_group_report = Some(report.clone());
18810 }
18811 selected_group_work = selected_group_work
18812 .saturating_add(report.span_eval_placement_count);
18813 computed_vertices = computed_vertices
18814 .saturating_add(report.span_eval_placement_count as usize);
18815 work_index = work_index.saturating_add(1);
18816 }
18817 self.charge_bounded_work(selected_group_work)?;
18820 #[cfg(test)]
18821 {
18822 if let Some(report) = last_group_report {
18823 self.last_formula_plane_span_eval_report = Some(report);
18824 }
18825 }
18826 }
18827 FormulaProducerId::Legacy(_) => {
18828 let mut vertices = Vec::new();
18837 while work_index < work_items.len() {
18838 let FormulaProducerId::Legacy(vertex_id) =
18839 work_items[work_index].producer
18840 else {
18841 break;
18842 };
18843 vertices.push(vertex_id);
18844 work_index = work_index.saturating_add(1);
18845 }
18846 let legacy_layer = crate::engine::scheduler::Layer { vertices };
18847 let evaluated = if let Some(delta) = delta.as_deref_mut() {
18848 if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
18849 self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
18850 } else {
18851 self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
18852 }
18853 } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
18854 {
18855 self.evaluate_layer_parallel(&legacy_layer)?
18856 } else {
18857 self.evaluate_layer_sequential(&legacy_layer)?
18858 };
18859 computed_vertices = computed_vertices.saturating_add(evaluated);
18860 }
18861 }
18862 }
18863 self.resource_checkpoint(0)?;
18864 self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
18865 if let Some(delta) = delta.as_deref_mut() {
18866 self.record_computed_write_buffer_delta(&buffer, delta);
18867 }
18868 self.flush_computed_write_buffer(&mut buffer)?;
18869 }
18870
18871 let mut changed_virtual =
18872 self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
18873 let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18874 let mut dynamic_candidates = old_dynamic_regions
18875 .keys()
18876 .chain(new_dynamic_regions.keys())
18877 .copied()
18878 .collect::<FxHashSet<_>>();
18879 for vertex in dynamic_candidates.drain() {
18880 if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
18881 && !changed_virtual.contains(&vertex)
18882 {
18883 changed_virtual.push(vertex);
18884 }
18885 }
18886 if let Some(telemetry) = virtual_telemetry.as_mut() {
18887 telemetry.changed_vdeps_total = telemetry
18888 .changed_vdeps_total
18889 .saturating_add(changed_virtual.len());
18890 }
18891 self.resource_checkpoint(0)?;
18892 self.graph.clear_dirty_flags(&legacy_vertices);
18893 if !changed_virtual.is_empty() {
18894 for vertex in &changed_virtual {
18895 self.graph.set_dirty(*vertex, true);
18896 }
18897 let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
18898 let can_attempt_workbook =
18899 !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
18900 if can_replan || can_attempt_workbook {
18901 if can_replan {
18902 if let Some(roots) = runtime_target_roots.as_mut() {
18903 self.extend_target_roots_with_dynamic_regions(
18904 roots,
18905 old_dynamic_regions
18906 .values()
18907 .flatten()
18908 .chain(new_dynamic_regions.values().flatten()),
18909 )?;
18910 }
18911 runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
18912 } else {
18913 runtime_target_roots = None;
18914 workbook_exact_attempted = true;
18915 }
18916 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
18917 if can_replan {
18918 stats.runtime_replan_rounds =
18919 stats.runtime_replan_rounds.saturating_add(1);
18920 }
18921 stats.runtime_widening_rounds =
18922 stats.runtime_widening_rounds.saturating_add(1);
18923 if can_attempt_workbook {
18924 stats.workbook_exact_attempts = 1;
18925 }
18926 }
18927 self.cached_mixed_topology = None;
18928 if let Some(ledger) = self.active_resource_ledger.as_mut() {
18929 ledger
18930 .account_mixed_cache(0)
18931 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
18932 }
18933 include_dirty_regions = true;
18934 continue 'virtual_replan;
18935 }
18936 if let Some(telemetry) = virtual_telemetry.as_mut() {
18937 telemetry.replan_iterations = runtime_replan_iterations;
18938 telemetry.bailout_reason = Some("max_replan");
18939 self.last_virtual_dep_telemetry = telemetry.clone();
18940 }
18941 return Err(self.replan_exhausted_error(
18942 MAX_RUNTIME_REPLAN,
18943 "mixed dynamic dependency evaluation",
18944 ));
18945 }
18946 if let Some(mut telemetry) = virtual_telemetry {
18947 telemetry.replan_iterations = runtime_replan_iterations;
18948 telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
18949 "converged"
18950 } else {
18951 "max_replan"
18952 });
18953 self.last_virtual_dep_telemetry = telemetry;
18954 }
18955 self.redirty_for_next_recalc();
18959 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
18960 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
18961 return Ok(EvalResult {
18962 computed_vertices,
18963 cycle_errors: prepass_cycle_errors,
18964 elapsed: start.elapsed(),
18965 });
18966 }
18967 }
18968
18969 fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
18970 let budgets = &self.evaluation_resource_budgets;
18971 let candidate_limit = budgets
18972 .optimization
18973 .mixed_cache_candidates
18974 .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
18975 limit.min(self.config.max_formula_plane_cache_candidates)
18976 });
18977 let edge_limit = budgets
18978 .optimization
18979 .mixed_cache_edges
18980 .map_or(self.config.max_formula_plane_cache_edges, |limit| {
18981 limit.min(self.config.max_formula_plane_cache_edges)
18982 });
18983 let mut byte_limit =
18984 u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
18985 if let Some(limit) = budgets.retained.total_bytes {
18986 byte_limit = byte_limit.min(limit);
18987 }
18988 if let Some(limit) = budgets.retained.mixed_cache_bytes {
18989 byte_limit = byte_limit.min(limit);
18990 }
18991 (
18992 candidate_limit,
18993 edge_limit,
18994 usize::try_from(byte_limit).unwrap_or(usize::MAX),
18995 )
18996 }
18997
18998 fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
18999 let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
19000 MixedTopologyCacheKey {
19001 engine_topology_epoch: self.topology_epoch,
19002 graph_topology_revision: self.graph.topology_revision(),
19003 authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
19004 legacy_island_revision: self.graph.topology_revision(),
19005 boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
19006 function_semantic_epoch: self.function_semantic_epoch_seen,
19007 function_provider_revision: self.function_provider_revision_seen,
19008 max_candidates,
19009 max_edges,
19010 max_memory_bytes,
19011 }
19012 }
19013
19014 fn mixed_topology_index_preflight_bytes(&self) -> u64 {
19015 use crate::formula_plane::region_index::AxisRange;
19016
19017 const FIXED_BYTES: u64 = 8 * 1024;
19018 const ENTRY_BYTES: u64 = 512;
19019 const INDEX_UNIT_BYTES: u64 = 1024;
19020 const BINDING_BYTES: u64 = 128;
19021 let index_units = |region: Region| -> u64 {
19022 match region.normalized().axis_ranges() {
19023 (AxisRange::Point(_), AxisRange::Point(_)) => 2,
19024 (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
19025 let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
19026 let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
19027 row_buckets.saturating_mul(col_buckets).saturating_mul(2)
19028 }
19029 _ => 1,
19030 }
19031 };
19032
19033 let authority = self.graph.formula_authority();
19034 let span_refs = authority.active_span_refs();
19035 let legacy_vertices = self.graph.formula_vertices();
19036 let mut entry_count = 0_u64;
19037 let mut units = 0_u64;
19038 for span_ref in &span_refs {
19039 if let Some(span) = authority.plane.spans.get(*span_ref) {
19040 let result_region = Region::from_domain(span.result_region.domain());
19041 entry_count = entry_count.saturating_add(1);
19042 units = units.saturating_add(index_units(result_region));
19043 if let Some(summary_id) = span.read_summary_id
19044 && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
19045 {
19046 for dependency in &summary.dependencies {
19047 entry_count = entry_count.saturating_add(1);
19048 units = units.saturating_add(index_units(dependency.read_region));
19049 }
19050 }
19051 }
19052 }
19053 for &vertex in &legacy_vertices {
19054 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
19055 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19056 entry_count = entry_count.saturating_add(1);
19057 units = units.saturating_add(index_units(point));
19058 }
19059 for dependency in self.graph.get_dependencies(vertex) {
19060 if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
19061 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19062 entry_count = entry_count.saturating_add(1);
19063 units = units.saturating_add(index_units(point));
19064 }
19065 }
19066 let vertex_sheet = self.graph.get_vertex_sheet_id(vertex);
19067 if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
19068 for range in ranges {
19069 if let Ok(Some(region)) =
19070 self.shared_range_to_region_pattern(range, vertex_sheet)
19071 {
19072 entry_count = entry_count.saturating_add(1);
19073 units = units.saturating_add(index_units(region));
19074 }
19075 }
19076 }
19077 }
19078 FIXED_BYTES
19079 .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
19080 .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
19081 .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
19082 }
19083
19084 fn prove_sheet_disjoint_legacy_island(
19085 &self,
19086 span_results: &FormulaProducerResultIndex,
19087 span_reads: &FormulaConsumerReadIndex,
19088 legacy_vertices: &[VertexId],
19089 ) -> Option<LegacyIslandPlan> {
19090 if !self.legacy_island_structural_summaries_trusted
19091 || self.graph.named_ranges_iter().next().is_some()
19092 || self.graph.sheet_named_ranges_iter().next().is_some()
19093 || self.graph.spill_registry_counts() != (0, 0)
19094 || legacy_vertices.iter().copied().any(|vertex| {
19095 self.graph.is_dynamic(vertex)
19096 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19097 })
19098 {
19099 return None;
19100 }
19101 let span_result_sheets = span_results
19102 .entries()
19103 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19104 .map(|entry| entry.result_region.sheet_id())
19105 .collect::<FxHashSet<_>>();
19106 let span_read_sheets = span_reads
19107 .entries()
19108 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
19109 .map(|entry| entry.read_region.sheet_id())
19110 .collect::<FxHashSet<_>>();
19111
19112 let span_boundary_sheets = span_result_sheets
19113 .union(&span_read_sheets)
19114 .copied()
19115 .collect::<FxHashSet<_>>();
19116 for vertex in legacy_vertices {
19117 let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
19118 if span_boundary_sheets.contains(&cell.sheet_id) {
19119 return None;
19120 }
19121 for dependency in self.graph.get_dependencies(*vertex) {
19122 let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
19123 if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
19124 return None;
19125 }
19126 }
19127 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19128 for range in ranges {
19129 let region = self
19130 .shared_range_to_region_pattern(range, cell.sheet_id)
19131 .ok()??;
19132 if span_boundary_sheets.contains(®ion.sheet_id()) {
19133 return None;
19134 }
19135 }
19136 }
19137 }
19138
19139 let mut membership = legacy_vertices.to_vec();
19140 membership.sort_unstable();
19141 if membership.is_empty() {
19142 return None;
19143 }
19144 let mut sheet_ids = membership
19145 .iter()
19146 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19147 .map(|cell| cell.sheet_id)
19148 .collect::<Vec<_>>();
19149 sheet_ids.sort_unstable();
19150 sheet_ids.dedup();
19151 let island_id = membership
19152 .iter()
19153 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19154 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19155 });
19156 let retained_bytes = membership
19157 .capacity()
19158 .saturating_mul(std::mem::size_of::<VertexId>())
19159 .saturating_add(
19160 sheet_ids
19161 .capacity()
19162 .saturating_mul(std::mem::size_of::<SheetId>()),
19163 );
19164 Some(LegacyIslandPlan {
19165 membership,
19166 dirty_vertices: Vec::new(),
19167 sheet_ids,
19168 island_id,
19169 retained_bytes,
19170 boundary_relationships: 0,
19171 omitted_relationships: legacy_vertices.len(),
19172 omitted_direct_relationships: 0,
19173 })
19174 }
19175
19176 fn contract_legacy_islands(
19177 &self,
19178 full_results: &FormulaProducerResultIndex,
19179 full_reads: &FormulaConsumerReadIndex,
19180 legacy_reads_complete: bool,
19181 candidate_cap: usize,
19182 ) -> Option<(
19183 FormulaProducerResultIndex,
19184 FormulaConsumerReadIndex,
19185 LegacyIslandPlan,
19186 )> {
19187 use crate::formula_plane::producer::ProjectionResult;
19188 use crate::formula_plane::region_index::BoundedRegionQueryResult;
19189
19190 if !self.legacy_island_structural_summaries_trusted
19191 || !legacy_reads_complete
19192 || self.graph.named_ranges_iter().next().is_some()
19193 || self.graph.sheet_named_ranges_iter().next().is_some()
19194 || self.graph.spill_registry_counts() != (0, 0)
19195 {
19196 return None;
19197 }
19198 let legacy_vertices = self.graph.formula_vertices();
19199 if legacy_vertices.iter().copied().any(|vertex| {
19200 self.graph.is_dynamic(vertex)
19201 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19202 }) {
19203 return None;
19204 }
19205
19206 let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
19207 for entry in full_reads.entries() {
19208 if let FormulaProducerId::Legacy(vertex) = entry.consumer {
19209 reads_by_consumer
19210 .entry(vertex)
19211 .or_default()
19212 .push(entry.read_region);
19213 }
19214 }
19215
19216 let mut connected = FxHashSet::default();
19217 let mut queue = VecDeque::new();
19218 let mut observed_candidates = 0usize;
19219 let mut boundary_relationships = 0usize;
19220 let add_connected = |vertex: VertexId,
19221 connected: &mut FxHashSet<VertexId>,
19222 queue: &mut VecDeque<VertexId>| {
19223 if connected.insert(vertex) {
19224 queue.push_back(vertex);
19225 }
19226 };
19227
19228 for result in full_results.entries() {
19230 if !matches!(result.producer, FormulaProducerId::Span(_)) {
19231 continue;
19232 }
19233 let remaining = candidate_cap.saturating_sub(observed_candidates);
19234 let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
19235 let BoundedRegionQueryResult::Complete(query) = query else {
19236 return None;
19237 };
19238 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19239 for matched in query.matches {
19240 if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
19241 match matched.value.dirty {
19242 ProjectionResult::NoIntersection => {}
19243 ProjectionResult::Unsupported(_) => return None,
19244 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19245 boundary_relationships = boundary_relationships.saturating_add(1);
19246 add_connected(vertex, &mut connected, &mut queue);
19247 }
19248 }
19249 }
19250 }
19251 }
19252
19253 for read in full_reads.entries() {
19255 if !matches!(read.consumer, FormulaProducerId::Span(_)) {
19256 continue;
19257 }
19258 let remaining = candidate_cap.saturating_sub(observed_candidates);
19259 let query = full_results.query_bounded(read.read_region, remaining);
19260 let BoundedRegionQueryResult::Complete(query) = query else {
19261 return None;
19262 };
19263 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19264 for matched in query.matches {
19265 if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
19266 boundary_relationships = boundary_relationships.saturating_add(1);
19267 add_connected(vertex, &mut connected, &mut queue);
19268 }
19269 }
19270 }
19271
19272 while let Some(vertex) = queue.pop_front() {
19276 let producer = FormulaProducerId::Legacy(vertex);
19277 if let Some(result_region) = full_results.producer_result_region(producer) {
19278 let remaining = candidate_cap.saturating_sub(observed_candidates);
19279 let query = full_reads.query_changed_region_bounded(result_region, remaining);
19280 let BoundedRegionQueryResult::Complete(query) = query else {
19281 return None;
19282 };
19283 observed_candidates =
19284 observed_candidates.saturating_add(query.stats.candidate_count);
19285 for matched in query.matches {
19286 if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
19287 match matched.value.dirty {
19288 ProjectionResult::NoIntersection => {}
19289 ProjectionResult::Unsupported(_) => return None,
19290 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19291 add_connected(consumer, &mut connected, &mut queue);
19292 }
19293 }
19294 }
19295 }
19296 }
19297 for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
19298 let remaining = candidate_cap.saturating_sub(observed_candidates);
19299 let query = full_results.query_bounded(*read_region, remaining);
19300 let BoundedRegionQueryResult::Complete(query) = query else {
19301 return None;
19302 };
19303 observed_candidates =
19304 observed_candidates.saturating_add(query.stats.candidate_count);
19305 for matched in query.matches {
19306 if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
19307 add_connected(precedent, &mut connected, &mut queue);
19308 }
19309 }
19310 }
19311 }
19312
19313 let mut membership = legacy_vertices
19314 .iter()
19315 .copied()
19316 .filter(|vertex| !connected.contains(vertex))
19317 .collect::<Vec<_>>();
19318 membership.sort_unstable();
19319 if membership.is_empty() {
19320 return None;
19321 }
19322
19323 let mut producer_results = FormulaProducerResultIndex::default();
19324 let mut consumer_reads = FormulaConsumerReadIndex::default();
19325 let retained_span_results = full_results
19326 .entries()
19327 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19328 .count();
19329 producer_results
19330 .try_reserve(retained_span_results.saturating_add(connected.len()))
19331 .ok()?;
19332 for entry in full_results.entries() {
19333 let retain = match entry.producer {
19334 FormulaProducerId::Span(_) => true,
19335 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19336 };
19337 if retain {
19338 producer_results.insert_producer(entry.producer, entry.result_region);
19339 }
19340 }
19341 for entry in full_reads.entries() {
19342 let retain = match entry.consumer {
19343 FormulaProducerId::Span(_) => true,
19344 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19345 };
19346 if retain {
19347 consumer_reads.try_reserve(1).ok()?;
19348 consumer_reads.insert_read(
19349 entry.consumer,
19350 entry.read_region,
19351 entry.consumer_result_region,
19352 entry.projection,
19353 );
19354 }
19355 }
19356
19357 let mut sheet_ids = membership
19358 .iter()
19359 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19360 .map(|cell| cell.sheet_id)
19361 .collect::<Vec<_>>();
19362 sheet_ids.sort_unstable();
19363 sheet_ids.dedup();
19364 let island_id = membership
19365 .iter()
19366 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19367 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19368 });
19369 let retained_bytes = membership
19370 .capacity()
19371 .saturating_mul(std::mem::size_of::<VertexId>())
19372 .saturating_add(
19373 sheet_ids
19374 .capacity()
19375 .saturating_mul(std::mem::size_of::<SheetId>()),
19376 );
19377 let omitted_relationships = full_reads
19378 .entries()
19379 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
19380 .count();
19381 let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
19382 let omitted_direct_relationships = membership
19383 .iter()
19384 .map(|vertex| {
19385 self.graph
19386 .get_dependencies(*vertex)
19387 .into_iter()
19388 .filter(|dependency| membership_set.contains(dependency))
19389 .count()
19390 })
19391 .sum();
19392 Some((
19393 producer_results,
19394 consumer_reads,
19395 LegacyIslandPlan {
19396 membership,
19397 dirty_vertices: Vec::new(),
19398 sheet_ids,
19399 island_id,
19400 retained_bytes,
19401 boundary_relationships,
19402 omitted_relationships,
19403 omitted_direct_relationships,
19404 },
19405 ))
19406 }
19407
19408 fn compile_formula_plane_mixed_topology(
19409 &mut self,
19410 key: MixedTopologyCacheKey,
19411 ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
19412 #[cfg(test)]
19413 {
19414 self.mixed_topology_index_builds_for_test =
19415 self.mixed_topology_index_builds_for_test.saturating_add(1);
19416 }
19417 let authority = self.graph.formula_authority();
19418 let mut producer_results = FormulaProducerResultIndex::default();
19419 let mut consumer_reads = FormulaConsumerReadIndex::default();
19420 let mut legacy_reads_complete = true;
19421 let span_refs = authority.active_span_refs();
19422 let legacy_vertices = self.graph.formula_vertices();
19423 let _producer_count = span_refs
19424 .len()
19425 .checked_add(legacy_vertices.len())
19426 .ok_or_else(|| {
19427 ExcelError::new(ExcelErrorKind::NImpl)
19428 .with_message("FormulaPlane cache producer count overflow")
19429 })?;
19430 producer_results.try_reserve(_producer_count).map_err(|_| {
19431 ExcelError::new(ExcelErrorKind::NImpl)
19432 .with_message("FormulaPlane cache producer index reservation failed")
19433 })?;
19434 let span_refs_by_id = span_refs
19435 .iter()
19436 .copied()
19437 .map(|span_ref| (span_ref.id, span_ref))
19438 .collect::<BTreeMap<_, _>>();
19439 for span_ref in &span_refs {
19440 let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
19441 ExcelError::new(ExcelErrorKind::NImpl)
19442 .with_message("FormulaPlane active span ref is stale")
19443 })?;
19444 let result_region = Region::from_domain(span.result_region.domain());
19445 producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
19446 let Some(read_summary_id) = span.read_summary_id else {
19447 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19448 .with_message("FormulaPlane active span is missing read summary"));
19449 };
19450 let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
19451 else {
19452 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19453 .with_message("FormulaPlane active span has stale read summary"));
19454 };
19455 if read_summary.result_region != result_region {
19456 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19457 .with_message("FormulaPlane active span read summary is stale"));
19458 }
19459 for dependency in &read_summary.dependencies {
19460 consumer_reads.try_reserve(1).map_err(|_| {
19461 ExcelError::new(ExcelErrorKind::NImpl)
19462 .with_message("FormulaPlane cache read-index reservation failed")
19463 })?;
19464 consumer_reads.insert_read(
19465 FormulaProducerId::Span(span.id),
19466 dependency.read_region,
19467 read_summary.result_region,
19468 dependency.projection,
19469 );
19470 }
19471 }
19472
19473 let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
19474 &producer_results,
19475 &consumer_reads,
19476 &legacy_vertices,
19477 );
19478 if sheet_disjoint_island.is_none() {
19479 for vertex in &legacy_vertices {
19480 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19481 continue;
19482 };
19483 producer_results.insert_producer(
19484 FormulaProducerId::Legacy(*vertex),
19485 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
19486 );
19487 }
19488 for vertex in &legacy_vertices {
19489 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19490 continue;
19491 };
19492 let result_region =
19493 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19494 let mut seen = rustc_hash::FxHashSet::default();
19495 for dep in self.graph.get_dependencies(*vertex) {
19496 let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
19497 legacy_reads_complete = false;
19500 continue;
19501 };
19502 let read_region = Region::point(
19503 dep_cell.sheet_id,
19504 dep_cell.coord.row(),
19505 dep_cell.coord.col(),
19506 );
19507 if seen.insert(read_region) {
19508 consumer_reads.try_reserve(1).map_err(|_| {
19509 ExcelError::new(ExcelErrorKind::NImpl)
19510 .with_message("FormulaPlane cache read-index reservation failed")
19511 })?;
19512 consumer_reads.insert_read(
19513 FormulaProducerId::Legacy(*vertex),
19514 read_region,
19515 result_region,
19516 DirtyProjectionRule::WholeResult,
19517 );
19518 }
19519 }
19520 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19521 for range in ranges {
19522 let Some(read_region) =
19523 self.shared_range_to_region_pattern(range, cell.sheet_id)?
19524 else {
19525 legacy_reads_complete = false;
19526 continue;
19527 };
19528 if seen.insert(read_region) {
19529 consumer_reads.try_reserve(1).map_err(|_| {
19530 ExcelError::new(ExcelErrorKind::NImpl).with_message(
19531 "FormulaPlane cache read-index reservation failed",
19532 )
19533 })?;
19534 consumer_reads.insert_read(
19535 FormulaProducerId::Legacy(*vertex),
19536 read_region,
19537 result_region,
19538 DirtyProjectionRule::WholeResult,
19539 );
19540 }
19541 }
19542 }
19543 let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
19544 for dependency in virtual_dependencies
19545 .get(vertex)
19546 .into_iter()
19547 .flatten()
19548 .copied()
19549 {
19550 let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
19551 continue;
19552 };
19553 let read_region =
19554 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19555 if seen.insert(read_region) {
19556 consumer_reads.try_reserve(1).map_err(|_| {
19557 ExcelError::new(ExcelErrorKind::NImpl)
19558 .with_message("FormulaPlane cache virtual read reservation failed")
19559 })?;
19560 consumer_reads.insert_read(
19561 FormulaProducerId::Legacy(*vertex),
19562 read_region,
19563 result_region,
19564 DirtyProjectionRule::WholeResult,
19565 );
19566 }
19567 }
19568 for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
19569 {
19570 if seen.insert(read_region) {
19571 consumer_reads.try_reserve(1).map_err(|_| {
19572 ExcelError::new(ExcelErrorKind::NImpl)
19573 .with_message("FormulaPlane dynamic-region reservation failed")
19574 })?;
19575 consumer_reads.insert_read(
19576 FormulaProducerId::Legacy(*vertex),
19577 read_region,
19578 result_region,
19579 DirtyProjectionRule::WholeResult,
19580 );
19581 }
19582 }
19583 }
19584 }
19585
19586 let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
19590 {
19591 (producer_results, consumer_reads, island)
19592 } else {
19593 self.contract_legacy_islands(
19594 &producer_results,
19595 &consumer_reads,
19596 legacy_reads_complete,
19597 key.max_candidates,
19598 )
19599 .unwrap_or((
19600 producer_results,
19601 consumer_reads,
19602 LegacyIslandPlan::default(),
19603 ))
19604 };
19605
19606 let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
19607 .checked_add(
19608 producer_results
19609 .estimated_memory_bytes()
19610 .unwrap_or(usize::MAX),
19611 )
19612 .and_then(|bytes| {
19613 bytes.checked_add(
19614 consumer_reads
19615 .estimated_memory_bytes()
19616 .unwrap_or(usize::MAX),
19617 )
19618 })
19619 .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
19620 .and_then(|bytes| bytes.checked_add(island.retained_bytes))
19621 .unwrap_or(usize::MAX);
19622 let config = MixedTopologyConfig {
19623 max_candidates: key.max_candidates,
19624 max_edges: key.max_edges,
19625 max_memory_bytes: key.max_memory_bytes,
19626 retained_memory_bytes,
19627 };
19628 let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
19629 match &mut compiled {
19633 MixedTopologyCompileResult::Cached(topology) => {
19634 topology.stats.producers = topology
19635 .stats
19636 .producers
19637 .saturating_add(island.membership.len());
19638 topology.stats.candidates = topology
19639 .stats
19640 .candidates
19641 .saturating_add(island.omitted_direct_relationships);
19642 topology.stats.relationships = topology
19643 .stats
19644 .relationships
19645 .saturating_add(island.omitted_direct_relationships);
19646 }
19647 MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
19648 observed.producers = observed.producers.saturating_add(island.membership.len());
19649 observed.candidates = observed
19650 .candidates
19651 .saturating_add(island.omitted_direct_relationships);
19652 observed.relationships = observed
19653 .relationships
19654 .saturating_add(island.omitted_direct_relationships);
19655 }
19656 }
19657 let plane_epoch = authority.plane.epoch().0;
19658 Ok(match compiled {
19659 MixedTopologyCompileResult::Cached(topology) => {
19660 FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
19661 key,
19662 topology,
19663 producer_results,
19664 consumer_reads,
19665 span_refs_by_id,
19666 plane_epoch,
19667 island,
19668 })
19669 }
19670 MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
19671 FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
19672 reason,
19673 observed,
19674 producer_results,
19675 consumer_reads,
19676 span_refs_by_id,
19677 plane_epoch,
19678 island,
19679 })
19680 }
19681 })
19682 }
19683
19684 fn build_formula_plane_mixed_schedule(
19685 &mut self,
19686 formula_dirty: &FormulaDirtyLease,
19687 retry_whole_spans: &[FormulaSpanRef],
19688 include_dirty_regions: bool,
19689 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19690 self.build_formula_plane_mixed_schedule_inner(
19691 formula_dirty,
19692 retry_whole_spans,
19693 include_dirty_regions,
19694 None,
19695 )
19696 }
19697
19698 fn build_formula_plane_target_schedule(
19699 &mut self,
19700 formula_dirty: &FormulaDirtyLease,
19701 retry_whole_spans: &[FormulaSpanRef],
19702 include_dirty_regions: bool,
19703 target_roots: &[crate::engine::target_preparation::TargetProducer],
19704 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19705 self.build_formula_plane_mixed_schedule_inner(
19706 formula_dirty,
19707 retry_whole_spans,
19708 include_dirty_regions,
19709 Some(target_roots),
19710 )
19711 }
19712
19713 fn build_formula_plane_mixed_schedule_inner(
19714 &mut self,
19715 formula_dirty: &FormulaDirtyLease,
19716 retry_whole_spans: &[FormulaSpanRef],
19717 include_dirty_regions: bool,
19718 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19719 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19720 let baseline = self.graph.baseline_stats();
19721 self.charge_bounded_work(
19722 baseline
19723 .graph_vertex_count
19724 .saturating_add(baseline.graph_edge_count) as u64,
19725 )?;
19726 let key = self.mixed_topology_cache_key();
19727 let active_refs = self
19728 .graph
19729 .formula_authority()
19730 .active_span_refs()
19731 .into_iter()
19732 .map(|span_ref| (span_ref.id, span_ref))
19733 .collect::<BTreeMap<_, _>>();
19734 let has_dynamic_legacy = self
19735 .graph
19736 .formula_vertices()
19737 .into_iter()
19738 .any(|vertex| self.graph.is_dynamic(vertex));
19739 let cache_hit = !has_dynamic_legacy
19740 && self
19741 .cached_mixed_topology
19742 .as_ref()
19743 .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
19744 let mut skipped_topology = None;
19745 let mut index_scratch_reserved = 0_u64;
19746 let mut demand_scratch_reserved = 0_u64;
19747 let topology_started = crate::instant::FzInstant::now();
19748 let (cache_outcome, strategy, compile_stats) = if cache_hit {
19749 self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
19750 self.mixed_topology_cache_skip_streak = 0;
19751 let cached = self
19752 .cached_mixed_topology
19753 .as_ref()
19754 .expect("cache hit has topology");
19755 let stats = cached.topology.stats.clone();
19756 let strategy = if cached.topology.is_complete() {
19757 FormulaPlaneTopologyStrategy::Cached
19758 } else {
19759 FormulaPlaneTopologyStrategy::ExactPagedIndexed
19760 };
19761 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19762 ledger
19763 .account_mixed_cache(stats.estimated_memory_bytes as u64)
19764 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19765 }
19766 (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
19767 } else {
19768 self.cached_mixed_topology = None;
19771 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19772 ledger
19773 .account_mixed_cache(0)
19774 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19775 }
19776
19777 let preflight = self.mixed_topology_index_preflight_bytes();
19778 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19779 ledger
19780 .reserve_schedule_discovery(preflight)
19781 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19782 }
19783 index_scratch_reserved = preflight;
19784 let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
19785 Ok(compiled) => compiled,
19786 Err(error) => {
19787 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19788 let _ = ledger.release_scratch(index_scratch_reserved);
19789 }
19790 return Err(error);
19791 }
19792 };
19793 let actual_index_bytes = match &compiled {
19794 FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
19795 .producer_results
19796 .estimated_memory_bytes()
19797 .and_then(|bytes| {
19798 bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
19799 })
19800 .and_then(|bytes| {
19801 bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
19802 })
19803 .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
19804 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
19805 .producer_results
19806 .estimated_memory_bytes()
19807 .and_then(|bytes| {
19808 bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
19809 })
19810 .and_then(|bytes| {
19811 bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
19812 })
19813 .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
19814 }
19815 .unwrap_or(usize::MAX) as u64;
19816 debug_assert!(
19817 actual_index_bytes <= index_scratch_reserved,
19818 "mixed topology index preflight underestimated actual allocation"
19819 );
19820 if actual_index_bytes > index_scratch_reserved {
19821 let additional = actual_index_bytes - index_scratch_reserved;
19822 let reserve_result = self
19823 .active_resource_ledger
19824 .as_mut()
19825 .map_or(Ok(()), |ledger| {
19826 ledger.reserve_schedule_discovery(additional)
19827 });
19828 if let Err(error) = reserve_result {
19829 drop(compiled);
19830 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19831 let _ = ledger.release_scratch(index_scratch_reserved);
19832 }
19833 return Err(error.into_excel_error());
19834 }
19835 } else if actual_index_bytes < index_scratch_reserved
19836 && let Some(ledger) = self.active_resource_ledger.as_mut()
19837 {
19838 ledger
19839 .release_scratch(index_scratch_reserved - actual_index_bytes)
19840 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19841 }
19842 index_scratch_reserved = actual_index_bytes;
19843
19844 self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
19845 match compiled {
19846 FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
19847 let stats = compiled.topology.stats.clone();
19848 self.mixed_topology_cache_skip_streak =
19849 self.mixed_topology_cache_skip_streak.saturating_add(1);
19850 skipped_topology = Some(SkippedMixedTopology {
19851 reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
19852 observed: stats.clone(),
19853 producer_results: compiled.producer_results,
19854 consumer_reads: compiled.consumer_reads,
19855 span_refs_by_id: compiled.span_refs_by_id,
19856 plane_epoch: compiled.plane_epoch,
19857 island: compiled.island,
19858 });
19859 (
19860 FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
19861 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19862 stats,
19863 )
19864 }
19865 FormulaPlaneTopologyCompileResult::Cached(compiled) => {
19866 let stats = compiled.topology.stats.clone();
19867 let strategy = if compiled.topology.is_complete() {
19868 FormulaPlaneTopologyStrategy::CompiledAndCached
19869 } else {
19870 FormulaPlaneTopologyStrategy::ExactPagedIndexed
19871 };
19872 let retained_result = self
19873 .active_resource_ledger
19874 .as_mut()
19875 .map_or(Ok(()), |ledger| {
19876 ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
19877 });
19878 if let Err(error) = retained_result {
19879 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19880 let _ = ledger.release_scratch(index_scratch_reserved);
19881 }
19882 return Err(error.into_excel_error());
19883 }
19884 self.mixed_topology_cache_skip_streak = 0;
19885 self.cached_mixed_topology = Some(compiled);
19886 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19887 ledger
19888 .release_scratch(index_scratch_reserved)
19889 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19890 }
19891 index_scratch_reserved = 0;
19892 (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
19893 }
19894 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
19895 debug_assert!(self.cached_mixed_topology.is_none());
19896 self.mixed_topology_cache_overflows =
19897 self.mixed_topology_cache_overflows.saturating_add(1);
19898 self.mixed_topology_cache_skip_streak =
19899 self.mixed_topology_cache_skip_streak.saturating_add(1);
19900 let stats = skipped.observed.clone();
19901 skipped_topology = Some(skipped);
19902 (
19903 FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
19904 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19905 stats,
19906 )
19907 }
19908 }
19909 };
19910 self.observe_topology(
19911 cache_outcome,
19912 strategy,
19913 &compile_stats,
19914 topology_started.elapsed(),
19915 );
19916 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
19917 stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
19918 stats.topology.candidate_cap = Some(key.max_candidates as u64);
19919 stats.topology.edge_cap = Some(key.max_edges as u64);
19920 stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
19921 if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
19922 stats.topology.operator_guidance =
19923 Some("dynamic legacy dependencies require an exact request topology rebuild");
19924 } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
19925 stats.topology.operator_guidance =
19926 Some(if compile_stats.candidate_overflow_count > 0 {
19927 "raise the mixed-cache candidate limit to avoid exact request rebuilds"
19928 } else if compile_stats.edge_overflow_count > 0 {
19929 "raise the mixed-cache edge limit to avoid exact request rebuilds"
19930 } else {
19931 "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
19932 });
19933 }
19934 }
19935
19936 let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
19937 let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19938 let (
19939 producer_results,
19940 consumer_reads,
19941 span_refs_by_id,
19942 plane_epoch,
19943 cached_topology,
19944 partial_topology,
19945 island,
19946 ) = if let Some(skipped) = skipped_topology.as_ref() {
19947 (
19948 &skipped.producer_results,
19949 &skipped.consumer_reads,
19950 &skipped.span_refs_by_id,
19951 skipped.plane_epoch,
19952 None,
19953 None,
19954 &skipped.island,
19955 )
19956 } else {
19957 let cached = self
19958 .cached_mixed_topology
19959 .as_ref()
19960 .expect("mixed topology available for request");
19961 let (complete, partial) = if cached.topology.is_complete() {
19962 (Some(&cached.topology), None)
19963 } else {
19964 (None, Some(&cached.topology))
19965 };
19966 (
19967 &cached.producer_results,
19968 &cached.consumer_reads,
19969 &cached.span_refs_by_id,
19970 cached.plane_epoch,
19971 complete,
19972 partial,
19973 &cached.island,
19974 )
19975 };
19976 let demand = if let Some(target_roots) = target_roots {
19977 use crate::engine::target_preparation::TargetProducer;
19978 let mut roots = Vec::new();
19979 let mut graph_roots = Vec::new();
19980 for root in target_roots {
19981 match *root {
19982 TargetProducer::Span { span_ref, demanded } => {
19983 roots.push((FormulaProducerId::Span(span_ref.id), demanded))
19984 }
19985 TargetProducer::Legacy(vertex) => {
19986 graph_roots.push(vertex);
19987 if let Some(region) = producer_results
19988 .producer_result_region(FormulaProducerId::Legacy(vertex))
19989 {
19990 roots.push((FormulaProducerId::Legacy(vertex), region));
19991 }
19992 }
19993 TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
19994 TargetProducer::ValueOnly(_) => {}
19995 }
19996 }
19997 if !graph_roots.is_empty() {
19998 let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
19999 for vertex in legacy_demand {
20000 if let Some(region) = producer_results
20001 .producer_result_region(FormulaProducerId::Legacy(vertex))
20002 {
20003 roots.push((FormulaProducerId::Legacy(vertex), region));
20004 }
20005 }
20006 }
20007 let roots = roots;
20008 let closure_base = (producer_results.len() as u64)
20009 .saturating_mul(128)
20010 .saturating_add(4096);
20011 let scratch_estimates =
20012 exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
20013 let native_allowed = native_exact_demand_allowed(
20014 self.active_resource_ledger
20015 .as_ref()
20016 .and_then(ResourceLedger::disk_scratch_policy),
20017 !cfg!(target_arch = "wasm32"),
20018 );
20019 let (selected_strategy, reservation) = if cached_topology.is_some() {
20020 (None, closure_base)
20021 } else {
20022 let (strategy, reservation) =
20023 select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
20024 self.can_reserve_topology_scratch(bytes)
20025 });
20026 (Some(strategy), reservation)
20027 };
20028 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20029 ledger
20030 .reserve_schedule_discovery(reservation)
20031 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20032 }
20033 demand_scratch_reserved = reservation;
20034 #[allow(unused_mut)]
20035 let mut native_disk_bytes = 0_u64;
20036 let (demand, actual_strategy, passes) = match selected_strategy {
20037 None => (
20038 build_demand_closure_cached(
20039 roots,
20040 producer_results,
20041 cached_topology.expect("cached demand strategy has topology"),
20042 ),
20043 None,
20044 0,
20045 ),
20046 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20047 let (demand, passes) = build_demand_closure_paged(
20048 roots,
20049 producer_results,
20050 consumer_reads,
20051 |units| {
20052 Self::resource_loop_checkpoint(
20053 &mut self.active_resource_ledger,
20054 units,
20055 )
20056 },
20057 )?;
20058 (demand, selected_strategy, passes)
20059 }
20060 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20061 let (demand, passes) = build_demand_closure_in_memory_runs(
20062 roots,
20063 producer_results,
20064 consumer_reads,
20065 |units| {
20066 Self::resource_loop_checkpoint(
20067 &mut self.active_resource_ledger,
20068 units,
20069 )
20070 },
20071 )?;
20072 (demand, selected_strategy, passes)
20073 }
20074 #[cfg(not(target_arch = "wasm32"))]
20075 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20076 let request_id = self
20077 .active_evaluation_resource_request
20078 .as_ref()
20079 .map_or(0, |stats| stats.request_id);
20080 match NativeTopologyScratch::create(request_id) {
20081 Ok(mut native) => match build_demand_closure_native(
20082 roots.clone(),
20083 producer_results,
20084 consumer_reads,
20085 native.file.as_mut().expect("native scratch file"),
20086 native
20087 .auxiliary_file
20088 .as_mut()
20089 .expect("native auxiliary scratch file"),
20090 |units| {
20091 Self::resource_loop_checkpoint(
20092 &mut self.active_resource_ledger,
20093 units,
20094 )
20095 },
20096 ) {
20097 Ok((demand, passes, bytes)) => {
20098 native_disk_bytes = bytes;
20099 (demand, selected_strategy, passes)
20100 }
20101 Err(NativeExactDemandError::Work(error)) => return Err(error),
20102 Err(NativeExactDemandError::Io(_)) => {
20103 let (demand, passes) = build_demand_closure_repeated_passes(
20104 roots,
20105 producer_results,
20106 consumer_reads,
20107 |units| {
20108 Self::resource_loop_checkpoint(
20109 &mut self.active_resource_ledger,
20110 units,
20111 )
20112 },
20113 )?;
20114 (
20115 demand,
20116 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20117 passes,
20118 )
20119 }
20120 },
20121 Err(_) => {
20122 let (demand, passes) = build_demand_closure_repeated_passes(
20123 roots,
20124 producer_results,
20125 consumer_reads,
20126 |units| {
20127 Self::resource_loop_checkpoint(
20128 &mut self.active_resource_ledger,
20129 units,
20130 )
20131 },
20132 )?;
20133 (
20134 demand,
20135 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20136 passes,
20137 )
20138 }
20139 }
20140 }
20141 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20142 let (demand, passes) = build_demand_closure_repeated_passes(
20143 roots,
20144 producer_results,
20145 consumer_reads,
20146 |units| {
20147 Self::resource_loop_checkpoint(
20148 &mut self.active_resource_ledger,
20149 units,
20150 )
20151 },
20152 )?;
20153 (demand, selected_strategy, passes)
20154 }
20155 _ => unreachable!("exact demand strategy selection is exhaustive"),
20156 };
20157 let strategy_memory = match actual_strategy {
20158 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20159 scratch_estimates.paged.saturating_sub(closure_base)
20160 }
20161 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20162 scratch_estimates.runs.saturating_sub(closure_base)
20163 }
20164 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20165 scratch_estimates.native.saturating_sub(closure_base)
20166 }
20167 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20168 scratch_estimates.repeated.saturating_sub(closure_base)
20169 }
20170 None => 0,
20171 _ => 0,
20172 };
20173 let actual_scratch = demand
20174 .estimated_memory_bytes()
20175 .saturating_add(strategy_memory);
20176 if actual_scratch > demand_scratch_reserved {
20177 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20178 ledger
20179 .reserve_schedule_discovery(
20180 actual_scratch.saturating_sub(demand_scratch_reserved),
20181 )
20182 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20183 }
20184 } else if actual_scratch < demand_scratch_reserved
20185 && let Some(ledger) = self.active_resource_ledger.as_mut()
20186 {
20187 ledger
20188 .release_scratch(demand_scratch_reserved - actual_scratch)
20189 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20190 }
20191 demand_scratch_reserved = actual_scratch;
20192 if let Some(strategy) = actual_strategy
20193 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20194 {
20195 if strategy.severity() > stats.topology.strategy.severity() {
20196 stats.topology.strategy = strategy;
20197 }
20198 stats.topology.exact_pass_count = stats
20199 .topology
20200 .exact_pass_count
20201 .saturating_add(passes.max(1));
20202 stats.topology.native_topology_disk_bytes = stats
20203 .topology
20204 .native_topology_disk_bytes
20205 .saturating_add(native_disk_bytes);
20206 }
20207 Some(demand)
20208 } else {
20209 None
20210 };
20211
20212 let dirty_legacy = self.graph.get_evaluation_vertices();
20213 let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
20214 let island_target_demand = target_roots.map(|roots| {
20215 use crate::engine::target_preparation::TargetProducer;
20216 let graph_roots = roots
20217 .iter()
20218 .filter_map(|root| match *root {
20219 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
20220 Some(vertex)
20221 }
20222 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
20223 })
20224 .collect::<Vec<_>>();
20225 self.build_demand_subgraph(&graph_roots)
20226 .0
20227 .into_iter()
20228 .collect::<FxHashSet<_>>()
20229 });
20230 let mut island_plan = island.clone();
20231 island_plan.dirty_vertices = island_plan
20232 .membership
20233 .iter()
20234 .copied()
20235 .filter(|vertex| dirty_legacy_set.contains(vertex))
20236 .filter(|vertex| {
20237 island_target_demand
20238 .as_ref()
20239 .is_none_or(|demand| demand.contains(vertex))
20240 })
20241 .collect();
20242 let mut work = Vec::new();
20243 let mut scheduled_legacy_vertices = Vec::new();
20244 let demanded_dirty = |producer: FormulaProducerId,
20245 dirty: ProducerDirtyDomain|
20246 -> Option<ProducerDirtyDomain> {
20247 let Some(demand) = demand.as_ref() else {
20248 return Some(dirty);
20249 };
20250 let result = producer_results.producer_result_region(producer)?;
20251 dirty.intersect_demanded(result, demand.regions(producer))
20252 };
20253 for (span_ref, region) in formula_dirty.span_regions() {
20254 let producer = FormulaProducerId::Span(span_ref.id);
20255 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20256 && let Some(dirty) =
20257 demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
20258 {
20259 work.push(FormulaProducerWork { producer, dirty });
20260 }
20261 }
20262 for span_ref in formula_dirty
20263 .whole_spans()
20264 .chain(retry_whole_spans.iter().copied())
20265 {
20266 let producer = FormulaProducerId::Span(span_ref.id);
20267 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20268 && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20269 {
20270 work.push(FormulaProducerWork { producer, dirty });
20271 }
20272 }
20273 for vertex in dirty_legacy {
20274 let producer = FormulaProducerId::Legacy(vertex);
20275 if producer_results.producer_result_region(producer).is_some()
20276 && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20277 {
20278 scheduled_legacy_vertices.push(vertex);
20279 work.push(FormulaProducerWork { producer, dirty });
20280 }
20281 }
20282
20283 let pending_changed_regions = formula_dirty
20284 .regions()
20285 .chain(formula_dirty.span_regions().map(|(_, region)| region))
20286 .collect::<Vec<_>>();
20287 if include_dirty_regions && !pending_changed_regions.is_empty() {
20288 use crate::formula_plane::producer::compute_dirty_closure_checked;
20289 let closure = compute_dirty_closure_checked(
20290 consumer_reads,
20291 pending_changed_regions.iter().copied(),
20292 |producer| producer_results.producer_result_region(producer),
20293 |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
20294 )?;
20295 if closure.incomplete {
20296 work.clear();
20299 scheduled_legacy_vertices.clear();
20300 if let Some(demand) = demand.as_ref() {
20301 for producer in demand.producers() {
20302 if let Some(dirty) =
20303 demanded_dirty(producer, ProducerDirtyDomain::Whole)
20304 {
20305 if let FormulaProducerId::Legacy(vertex) = producer {
20306 scheduled_legacy_vertices.push(vertex);
20307 }
20308 work.push(FormulaProducerWork { producer, dirty });
20309 }
20310 }
20311 } else {
20312 work.extend(span_refs_by_id.keys().copied().map(|id| {
20313 FormulaProducerWork {
20314 producer: FormulaProducerId::Span(id),
20315 dirty: ProducerDirtyDomain::Whole,
20316 }
20317 }));
20318 for vertex in self.graph.formula_vertices() {
20319 let producer = FormulaProducerId::Legacy(vertex);
20320 if producer_results.producer_result_region(producer).is_some() {
20321 scheduled_legacy_vertices.push(vertex);
20322 work.push(FormulaProducerWork {
20323 producer,
20324 dirty: ProducerDirtyDomain::Whole,
20325 });
20326 }
20327 }
20328 }
20329 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20330 stats.topology.incomplete_reason =
20331 Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
20332 }
20333 } else if let Some(demand) = demand.as_ref() {
20334 for item in closure.work {
20335 if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
20336 work.push(FormulaProducerWork {
20337 producer: item.producer,
20338 dirty,
20339 });
20340 }
20341 }
20342 } else {
20343 work.extend(closure.work);
20344 }
20345 if !closure.incomplete && !closure.fallbacks.is_empty() {
20346 let mut already_whole = work
20347 .iter()
20348 .filter_map(|item| match (item.producer, &item.dirty) {
20349 (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
20350 _ => None,
20351 })
20352 .collect::<rustc_hash::FxHashSet<_>>();
20353 for fallback in closure.fallbacks {
20354 if let FormulaProducerId::Span(id) = fallback.consumer
20355 && already_whole.insert(id)
20356 {
20357 work.push(FormulaProducerWork {
20358 producer: FormulaProducerId::Span(id),
20359 dirty: ProducerDirtyDomain::Whole,
20360 });
20361 }
20362 }
20363 }
20364 }
20365
20366 let owned_dirty_events = if let Some(demand) = demand.as_ref() {
20367 use crate::formula_plane::producer::compute_dirty_closure_checked;
20368 let mut owned = Vec::new();
20369 for (index, event) in formula_dirty.events() {
20370 let (seed_regions, initial) = match event {
20371 FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
20372 FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
20373 vec![region],
20374 Some(FormulaProducerWork {
20375 producer: FormulaProducerId::Span(span_ref.id),
20376 dirty: ProducerDirtyDomain::Regions(vec![region]),
20377 }),
20378 ),
20379 FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
20380 let producer = FormulaProducerId::Span(span_ref.id);
20381 let Some(result) = producer_results.producer_result_region(producer)
20382 else {
20383 continue;
20384 };
20385 (
20386 vec![result],
20387 Some(FormulaProducerWork {
20388 producer,
20389 dirty: ProducerDirtyDomain::Whole,
20390 }),
20391 )
20392 }
20393 };
20394 let closure = compute_dirty_closure_checked(
20395 consumer_reads,
20396 seed_regions,
20397 |producer| producer_results.producer_result_region(producer),
20398 |units| {
20399 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
20400 },
20401 )?;
20402 if closure.incomplete || !closure.fallbacks.is_empty() {
20403 continue;
20404 }
20405 let fully_demanded =
20406 initial
20407 .into_iter()
20408 .chain(closure.work.into_iter())
20409 .all(|item| {
20410 let Some(result) =
20411 producer_results.producer_result_region(item.producer)
20412 else {
20413 return false;
20414 };
20415 item.dirty
20416 .result_regions(result)
20417 .into_iter()
20418 .all(|dirty_region| {
20419 demand
20420 .regions(item.producer)
20421 .iter()
20422 .any(|demanded| demanded.contains_region(dirty_region))
20423 })
20424 });
20425 if fully_demanded {
20426 owned.push(index);
20427 }
20428 }
20429 owned
20430 } else {
20431 (0..formula_dirty.len()).collect()
20432 };
20433
20434 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
20435 let producer_count = producer_results.len();
20436 let dirty_units = work.iter().fold(0usize, |total, item| {
20437 total.saturating_add(match &item.dirty {
20438 ProducerDirtyDomain::Whole => 0,
20439 ProducerDirtyDomain::Cells(cells) => cells.len(),
20440 ProducerDirtyDomain::Regions(regions) => regions.len(),
20441 })
20442 });
20443 let work_bytes = work
20444 .len()
20445 .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
20446 .saturating_add(
20447 dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
20448 std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
20449 )),
20450 ) as u64;
20451 let binding_bytes =
20452 estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
20453 let schedule_bytes = work_bytes
20454 .saturating_mul(4)
20455 .saturating_add((producer_count as u64).saturating_mul(192))
20456 .saturating_add(binding_bytes)
20457 .saturating_add(1024);
20458 let maximum_edges =
20459 producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
20460 let paged_bytes = schedule_bytes
20464 .saturating_add(maximum_edges.saturating_mul(160))
20465 .saturating_add(16 * 1024);
20466 let runs_bytes = schedule_bytes
20467 .saturating_add(maximum_edges.saturating_mul(48))
20468 .saturating_add(8192);
20469 let native_bytes = schedule_bytes
20470 .saturating_add((producer_count as u64).saturating_mul(128))
20471 .saturating_add(2048);
20472 let repeated_bytes = schedule_bytes
20473 .saturating_add((producer_count as u64).saturating_mul(96))
20474 .saturating_add(1024);
20475 let native_allowed = self
20476 .active_resource_ledger
20477 .as_ref()
20478 .and_then(ResourceLedger::disk_scratch_policy)
20479 == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
20480 && !cfg!(target_arch = "wasm32");
20481
20482 let cached_schedule_bytes = schedule_bytes
20483 .saturating_add(
20484 cached_topology
20485 .map_or(0, |topology| topology.stats.relationships as u64)
20486 .saturating_mul(160),
20487 )
20488 .saturating_add(1024);
20489 let use_cached_schedule = cached_topology.is_some()
20490 && self.can_reserve_topology_scratch(cached_schedule_bytes);
20491 let (selected_strategy, scratch) = if use_cached_schedule {
20492 (None, cached_schedule_bytes)
20493 } else if self.can_reserve_topology_scratch(paged_bytes) {
20494 (
20495 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
20496 paged_bytes,
20497 )
20498 } else if self.can_reserve_topology_scratch(runs_bytes) {
20499 (
20500 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
20501 runs_bytes,
20502 )
20503 } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
20504 (
20505 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20506 native_bytes,
20507 )
20508 } else if self.can_reserve_topology_scratch(repeated_bytes) {
20509 (
20510 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20511 repeated_bytes,
20512 )
20513 } else {
20514 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20515 ledger
20516 .reserve_schedule_discovery(repeated_bytes)
20517 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20518 }
20519 unreachable!("failed scratch reservation must return a typed error")
20520 };
20521 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20522 ledger
20523 .reserve_schedule_discovery(scratch)
20524 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20525 }
20526 let request_id = self
20527 .active_evaluation_resource_request
20528 .as_ref()
20529 .map_or(0, |stats| stats.request_id);
20530 let build_result = (|| -> Result<_, ExcelError> {
20531 let (schedule, actual_strategy, passes, native_disk_bytes) =
20532 match (cached_topology, selected_strategy) {
20533 (Some(topology), None) => (
20534 schedule_dirty_work(work, producer_results, topology, 256),
20535 None,
20536 0,
20537 0,
20538 ),
20539 (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
20540 let (schedule, passes) = schedule_dirty_work_paged_hybrid(
20541 work,
20542 producer_results,
20543 consumer_reads,
20544 partial_topology.filter(|_| reuse_partial_topology),
20545 256,
20546 |units| {
20547 Self::resource_loop_checkpoint(
20548 &mut self.active_resource_ledger,
20549 units,
20550 )
20551 },
20552 )?;
20553 (schedule, selected_strategy, passes.max(1), 0)
20554 }
20555 (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
20556 let (schedule, passes) = schedule_dirty_work_in_memory_runs(
20557 work,
20558 producer_results,
20559 consumer_reads,
20560 256,
20561 |units| {
20562 Self::resource_loop_checkpoint(
20563 &mut self.active_resource_ledger,
20564 units,
20565 )
20566 },
20567 )?;
20568 (schedule, selected_strategy, passes.max(1), 0)
20569 }
20570 #[cfg(not(target_arch = "wasm32"))]
20571 (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
20572 let native = NativeTopologyScratch::create(request_id);
20573 if let Ok(mut native) = native {
20574 match schedule_dirty_work_native(
20575 work.clone(),
20576 producer_results,
20577 consumer_reads,
20578 256,
20579 native.file.as_mut().expect("native scratch file"),
20580 |units| {
20581 Self::resource_loop_checkpoint(
20582 &mut self.active_resource_ledger,
20583 units,
20584 )
20585 },
20586 ) {
20587 Ok((schedule, passes, topology_bytes)) => (
20588 schedule,
20589 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20590 passes.max(1),
20591 topology_bytes,
20592 ),
20593 Err(NativeExactScheduleError::Work(error)) => {
20594 return Err(error);
20595 }
20596 Err(NativeExactScheduleError::Io(_)) => {
20597 let (schedule, passes) =
20598 schedule_dirty_work_repeated_passes(
20599 work,
20600 producer_results,
20601 consumer_reads,
20602 256,
20603 |units| {
20604 Self::resource_loop_checkpoint(
20605 &mut self.active_resource_ledger,
20606 units,
20607 )
20608 },
20609 )?;
20610 (
20611 schedule,
20612 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20613 passes.max(1),
20614 0,
20615 )
20616 }
20617 }
20618 } else {
20619 let (schedule, passes) = schedule_dirty_work_repeated_passes(
20620 work,
20621 producer_results,
20622 consumer_reads,
20623 256,
20624 |units| {
20625 Self::resource_loop_checkpoint(
20626 &mut self.active_resource_ledger,
20627 units,
20628 )
20629 },
20630 )?;
20631 (
20632 schedule,
20633 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20634 passes.max(1),
20635 0,
20636 )
20637 }
20638 }
20639 (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
20640 let (schedule, passes) = schedule_dirty_work_repeated_passes(
20641 work,
20642 producer_results,
20643 consumer_reads,
20644 256,
20645 |units| {
20646 Self::resource_loop_checkpoint(
20647 &mut self.active_resource_ledger,
20648 units,
20649 )
20650 },
20651 )?;
20652 (schedule, selected_strategy, passes.max(1), 0)
20653 }
20654 _ => unreachable!(
20655 "topology cache and exact strategy selection are exhaustive"
20656 ),
20657 };
20658 Ok((
20659 schedule,
20660 actual_strategy,
20661 passes,
20662 native_disk_bytes,
20663 span_refs_by_id.clone(),
20664 ))
20665 })();
20666 let release_result = self
20667 .active_resource_ledger
20668 .as_mut()
20669 .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
20670 .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20671 let (schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
20672 match build_result {
20673 Ok(output) => {
20674 release_result?;
20675 output
20676 }
20677 Err(error) => {
20678 let _ = release_result;
20679 return Err(error);
20680 }
20681 };
20682 if let Some(strategy) = exact_strategy
20683 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20684 {
20685 if strategy.severity() > stats.topology.strategy.severity() {
20686 stats.topology.strategy = strategy;
20687 }
20688 stats.topology.exact_pass_count = pass_count;
20689 stats.topology.native_topology_disk_bytes = stats
20690 .topology
20691 .native_topology_disk_bytes
20692 .saturating_add(native_disk_bytes);
20693 }
20694 Ok((
20695 schedule,
20696 result_span_refs,
20697 plane_epoch,
20698 scheduled_legacy_vertices,
20699 owned_dirty_events,
20700 island_plan,
20701 ))
20702 })();
20703 let scratch_release_result = self
20704 .active_resource_ledger
20705 .as_mut()
20706 .map_or(Ok(()), |ledger| {
20707 ledger
20708 .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
20709 })
20710 .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20711 match schedule_result {
20712 Ok(output) => {
20713 scratch_release_result?;
20714 Ok(output)
20715 }
20716 Err(error) => {
20717 let _ = scratch_release_result;
20718 Err(error)
20719 }
20720 }
20721 }
20722}
20723
20724impl<R> Engine<R>
20725where
20726 R: EvaluationContext,
20727{
20728 fn shared_range_to_region_pattern(
20737 &self,
20738 range: &crate::reference::SharedRangeRef<'static>,
20739 context_sheet: SheetId,
20740 ) -> Result<Option<Region>, ExcelError> {
20741 assert!(
20742 matches!(range.sheet, crate::reference::SharedSheetLocator::Id(_)),
20743 "PROBE: non-Id locator reached shared_range_to_region_pattern: {:?}",
20744 range.sheet
20745 );
20746 let Ok(sheet_id) = self.resolve_sheet_locator(&range.sheet, context_sheet) else {
20747 return Ok(None);
20749 };
20750 match (
20751 range.start_row,
20752 range.end_row,
20753 range.start_col,
20754 range.end_col,
20755 ) {
20756 (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
20757 sheet_id, sr.index, er.index, sc.index, ec.index,
20758 ))),
20759 (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
20760 Ok(Some(Region::whole_col(sheet_id, sc.index)))
20761 }
20762 (Some(sr), Some(er), None, None) if sr.index == er.index => {
20763 Ok(Some(Region::whole_row(sheet_id, sr.index)))
20764 }
20765 _ => Ok(None),
20766 }
20767 }
20768
20769 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
20771 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
20772 engine.evaluate_all_unobserved()
20773 })
20774 }
20775
20776 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
20777 debug_assert!(
20778 !self.graph.deferred_dirty_active(),
20779 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
20780 was not balanced by end_deferred_dirty"
20781 );
20782 self.observe_function_semantic_epoch()?;
20783 self.lookup_index_cache.reset_counters();
20784 let _source_cache = self.source_cache_session();
20785 self.validate_deterministic_mode()?;
20786 if self.config.defer_graph_building {
20787 self.build_graph_all()?;
20789 }
20790 self.evaluate_all_coordinator()
20791 }
20792
20793 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
20798 self.begin_evaluation_request();
20799 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
20800 return self.evaluate_authoritative_formula_plane_all();
20801 }
20802 self.evaluate_all_legacy_impl()
20803 }
20804
20805 fn legacy_pass_run_units(
20811 &mut self,
20812 schedule: &crate::engine::scheduler::Schedule,
20813 ) -> Result<(usize, usize), ExcelError> {
20814 let mut computed_vertices = 0;
20815 let mut cycle_count = 0;
20816 for &unit in &schedule.units {
20817 match unit {
20818 ScheduleUnit::Cycle(i) => {
20819 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
20820 cycle_count += 1;
20821 }
20822 }
20823 ScheduleUnit::Layer(i) => {
20824 let layer = schedule.unit_layer(i);
20825 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20826 computed_vertices += self.evaluate_layer_parallel(layer)?;
20827 } else {
20828 computed_vertices += self.evaluate_layer_sequential(layer)?;
20829 }
20830 }
20831 }
20832 }
20833 Ok((computed_vertices, cycle_count))
20834 }
20835
20836 fn evaluate_legacy_island_non_finalizing(
20841 &mut self,
20842 vertices: &[VertexId],
20843 mut delta: Option<&mut DeltaCollector>,
20844 ) -> Result<(usize, usize), ExcelError> {
20845 if vertices.is_empty() {
20846 return Ok((0, 0));
20847 }
20848 let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
20849 let mut computed = 0usize;
20850 let mut cycles = 0usize;
20851 for &unit in &schedule.units {
20852 self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
20853 match unit {
20854 ScheduleUnit::Cycle(index) => {
20855 if self.handle_cycle_unit(
20856 schedule.unit_cycle(index),
20857 delta.as_deref_mut(),
20858 None,
20859 None,
20860 )? > 0
20861 {
20862 cycles = cycles.saturating_add(1);
20863 }
20864 }
20865 ScheduleUnit::Layer(index) => {
20866 let layer = schedule.unit_layer(index);
20867 let evaluated = if let Some(delta) = delta.as_deref_mut() {
20868 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20869 self.evaluate_layer_parallel_with_delta(layer, delta)?
20870 } else {
20871 self.evaluate_layer_sequential_with_delta(layer, delta)?
20872 }
20873 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20874 self.evaluate_layer_parallel(layer)?
20875 } else {
20876 self.evaluate_layer_sequential(layer)?
20877 };
20878 computed = computed.saturating_add(evaluated);
20879 }
20880 }
20881 }
20882 self.graph.clear_dirty_flags(vertices);
20883 Ok((computed, cycles))
20884 }
20885
20886 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
20898 self.reset_virtual_dep_telemetry_if_disabled();
20899 #[cfg(feature = "tracing")]
20900 let _span_eval = tracing::info_span!("evaluate_all").entered();
20901 let start = crate::instant::FzInstant::now();
20902 let mut computed_vertices = 0;
20903 let mut cycle_errors = 0;
20904 let mut replan_iterations = 0;
20905 const MAX_REPLAN: usize = 5;
20906 let mut telemetry = self
20907 .config
20908 .enable_virtual_dep_telemetry
20909 .then(|| self.start_virtual_dep_telemetry());
20910
20911 loop {
20912 let to_evaluate = self.graph.get_evaluation_vertices();
20913 if to_evaluate.is_empty() {
20914 if let Some(t) = telemetry.as_mut()
20915 && t.bailout_reason.is_none()
20916 {
20917 t.bailout_reason = Some("no_work");
20918 }
20919 break;
20920 }
20921
20922 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20923 if let Some(t) = telemetry.as_mut() {
20924 Self::accumulate_schedule_meta(t, &meta);
20925 }
20926
20927 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
20928 computed_vertices += pass_computed;
20929 cycle_errors += pass_cycles;
20930
20931 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
20933 if let Some(t) = telemetry.as_mut() {
20934 t.changed_vdeps_total += changed_vertices.len();
20935 }
20936
20937 self.resource_checkpoint(0)?;
20938 self.graph.clear_dirty_flags(&to_evaluate);
20939 for v in &changed_vertices {
20940 self.graph.set_dirty(*v, true);
20941 }
20942
20943 if changed_vertices.is_empty() {
20944 if let Some(t) = telemetry.as_mut() {
20945 t.bailout_reason = Some("converged");
20946 }
20947 break;
20948 }
20949 if replan_iterations >= MAX_REPLAN {
20950 if let Some(mut t) = telemetry.take() {
20951 t.bailout_reason = Some("max_replan");
20952 t.replan_iterations = replan_iterations;
20953 self.last_virtual_dep_telemetry = t;
20954 }
20955 return Err(
20956 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
20957 );
20958 }
20959
20960 replan_iterations += 1;
20961 }
20962
20963 if let Some(mut t) = telemetry {
20964 t.replan_iterations = replan_iterations;
20965 self.last_virtual_dep_telemetry = t;
20966 }
20967
20968 self.redirty_for_next_recalc();
20970
20971 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20973
20974 Ok(EvalResult {
20975 computed_vertices,
20976 cycle_errors,
20977 elapsed: start.elapsed(),
20978 })
20979 }
20980
20981 pub fn evaluate_all_with_target_delta(
20982 &mut self,
20983 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
20984 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20985 engine.observe_function_semantic_epoch()?;
20986 let mut collector = DeltaCollector::new(DeltaMode::Cells);
20987 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20988 Ok((result, collector.finish_target()))
20989 })
20990 }
20991
20992 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
20993 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
20994 }
20995
20996 pub fn evaluate_all_with_delta_policy(
20997 &mut self,
20998 policy: EvalDeltaCompatibilityPolicy,
20999 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
21000 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
21001 engine.observe_function_semantic_epoch()?;
21002 let mut collector = DeltaCollector::new(DeltaMode::Cells);
21003 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
21004 Ok((result, collector.finish_with_policy(policy)?))
21005 })
21006 }
21007
21008 fn evaluate_all_with_delta_collector(
21009 &mut self,
21010 delta: &mut DeltaCollector,
21011 ) -> Result<EvalResult, ExcelError> {
21012 self.begin_evaluation_request();
21013 let _source_cache = self.source_cache_session();
21014 if self.config.defer_graph_building {
21015 self.build_graph_all()?;
21016 }
21017 if self.graph.formula_authority().active_span_count() > 0 {
21018 return self.evaluate_authoritative_formula_plane(None, Some(delta));
21019 }
21020 self.reset_virtual_dep_telemetry_if_disabled();
21021 #[cfg(feature = "tracing")]
21022 let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
21023 let start = crate::instant::FzInstant::now();
21024 let mut computed_vertices = 0;
21025 let mut cycle_errors = 0;
21026
21027 let mut replan_iterations = 0;
21028 const MAX_REPLAN: usize = 5;
21029 let mut telemetry = self
21030 .config
21031 .enable_virtual_dep_telemetry
21032 .then(|| self.start_virtual_dep_telemetry());
21033
21034 loop {
21035 let to_evaluate = self.graph.get_evaluation_vertices();
21036 if to_evaluate.is_empty() {
21037 if let Some(t) = telemetry.as_mut()
21038 && t.bailout_reason.is_none()
21039 {
21040 t.bailout_reason = Some("no_work");
21041 }
21042 break;
21043 }
21044
21045 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21046 if let Some(t) = telemetry.as_mut() {
21047 Self::accumulate_schedule_meta(t, &meta);
21048 }
21049
21050 for &unit in &schedule.units {
21051 match unit {
21052 ScheduleUnit::Cycle(i) => {
21053 if self.handle_cycle_unit(
21054 schedule.unit_cycle(i),
21055 Some(delta),
21056 None,
21057 None,
21058 )? > 0
21059 {
21060 cycle_errors += 1;
21061 }
21062 }
21063 ScheduleUnit::Layer(i) => {
21064 let layer = schedule.unit_layer(i);
21065 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21066 computed_vertices +=
21067 self.evaluate_layer_parallel_with_delta(layer, delta)?;
21068 } else {
21069 computed_vertices +=
21070 self.evaluate_layer_sequential_with_delta(layer, delta)?;
21071 }
21072 }
21073 }
21074 }
21075
21076 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21077 if let Some(t) = telemetry.as_mut() {
21078 t.changed_vdeps_total += changed_vertices.len();
21079 }
21080 self.resource_checkpoint(0)?;
21081 self.graph.clear_dirty_flags(&to_evaluate);
21082 for v in &changed_vertices {
21083 self.graph.set_dirty(*v, true);
21084 }
21085
21086 if changed_vertices.is_empty() {
21087 if let Some(t) = telemetry.as_mut() {
21088 t.bailout_reason = Some("converged");
21089 }
21090 break;
21091 }
21092 if replan_iterations >= MAX_REPLAN {
21093 if let Some(mut t) = telemetry.take() {
21094 t.bailout_reason = Some("max_replan");
21095 t.replan_iterations = replan_iterations;
21096 self.last_virtual_dep_telemetry = t;
21097 }
21098 return Err(
21099 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21100 );
21101 }
21102 replan_iterations += 1;
21103 }
21104
21105 if let Some(mut t) = telemetry {
21106 t.replan_iterations = replan_iterations;
21107 self.last_virtual_dep_telemetry = t;
21108 }
21109
21110 self.redirty_for_next_recalc();
21111 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21112
21113 Ok(EvalResult {
21114 computed_vertices,
21115 cycle_errors,
21116 elapsed: start.elapsed(),
21117 })
21118 }
21119
21120 pub fn evaluate_cell(
21130 &mut self,
21131 sheet: &str,
21132 row: u32,
21133 col: u32,
21134 ) -> Result<Option<LiteralValue>, ExcelError> {
21135 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
21136 engine.evaluate_cell_unobserved(sheet, row, col)
21137 })
21138 }
21139
21140 fn evaluate_cell_unobserved(
21141 &mut self,
21142 sheet: &str,
21143 row: u32,
21144 col: u32,
21145 ) -> Result<Option<LiteralValue>, ExcelError> {
21146 if row == 0 || col == 0 {
21147 return Err(ExcelError::new(ExcelErrorKind::Ref)
21148 .with_message("Row and column must be >= 1".to_string()));
21149 }
21150
21151 let result = self.evaluate_cells(&[(sheet, row, col)])?;
21152
21153 match result.len() {
21154 0 => Ok(None),
21155 1 => {
21156 let v = result.into_iter().next().unwrap();
21157 Ok(v)
21158 }
21159 _ => unreachable!("evaluate_cells returned unexpected length"),
21160 }
21161 }
21162
21163 pub fn evaluate_cells(
21170 &mut self,
21171 targets: &[(&str, u32, u32)],
21172 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21173 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
21174 engine.evaluate_cells_unobserved(targets)
21175 })
21176 }
21177
21178 fn evaluate_cells_unobserved(
21179 &mut self,
21180 targets: &[(&str, u32, u32)],
21181 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21182 self.observe_function_semantic_epoch()?;
21183 debug_assert!(
21184 !self.graph.deferred_dirty_active(),
21185 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
21186 was not balanced by end_deferred_dirty"
21187 );
21188 self.validate_deterministic_mode()?;
21189 if targets.is_empty() {
21190 return Ok(Vec::new());
21191 }
21192 let typed_targets = self.legacy_coordinate_targets(targets);
21193 self.evaluate_mixed_targets(&typed_targets, None)?;
21194 Ok(targets
21195 .iter()
21196 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21197 .collect())
21198 }
21199
21200 pub fn evaluate_cells_cancellable(
21201 &mut self,
21202 targets: &[(&str, u32, u32)],
21203 cancel: crate::engine::CancelToken,
21204 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21205 self.observe_evaluation_resource_request(
21206 EvaluationRequestKind::CellsCancellable,
21207 |engine| {
21208 engine.observe_function_semantic_epoch()?;
21209 engine.active_cancel_flag = Some(cancel.clone());
21210 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
21211 engine.active_cancel_flag = None;
21212 res
21213 },
21214 )
21215 }
21216
21217 fn evaluate_cells_cancellable_impl(
21218 &mut self,
21219 targets: &[(&str, u32, u32)],
21220 cancel_flag: &AtomicBool,
21221 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21222 self.validate_deterministic_mode()?;
21223 if targets.is_empty() {
21224 return Ok(Vec::new());
21225 }
21226 if cancel_flag.load(Ordering::Relaxed) {
21227 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21228 .with_message("Evaluation cancelled before target preparation"));
21229 }
21230 let typed_targets = self.legacy_coordinate_targets(targets);
21231 self.evaluate_mixed_targets(&typed_targets, None)?;
21232 Ok(targets
21233 .iter()
21234 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
21235 .collect())
21236 }
21237
21238 pub fn evaluate_cells_with_target_delta(
21239 &mut self,
21240 targets: &[(&str, u32, u32)],
21241 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
21242 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21243 engine.observe_function_semantic_epoch()?;
21244 engine.validate_deterministic_mode()?;
21245 if targets.is_empty() {
21246 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
21247 }
21248 let mut collector = DeltaCollector::new(DeltaMode::Cells);
21249 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
21250 let values = targets
21251 .iter()
21252 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
21253 .collect();
21254 Ok((values, collector.finish_target()))
21255 })
21256 }
21257
21258 pub fn evaluate_cells_with_delta(
21259 &mut self,
21260 targets: &[(&str, u32, u32)],
21261 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21262 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
21263 }
21264
21265 pub fn evaluate_cells_with_delta_policy(
21266 &mut self,
21267 targets: &[(&str, u32, u32)],
21268 policy: EvalDeltaCompatibilityPolicy,
21269 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21270 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21271 engine.evaluate_cells_with_delta_unobserved(targets, policy)
21272 })
21273 }
21274
21275 fn evaluate_cells_with_delta_unobserved(
21276 &mut self,
21277 targets: &[(&str, u32, u32)],
21278 policy: EvalDeltaCompatibilityPolicy,
21279 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21280 self.observe_function_semantic_epoch()?;
21281 self.validate_deterministic_mode()?;
21282 if targets.is_empty() {
21283 return Ok((Vec::new(), EvalDelta::default()));
21284 }
21285 let mut collector = DeltaCollector::new(DeltaMode::Cells);
21286 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
21287 let values = targets
21288 .iter()
21289 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21290 .collect();
21291 Ok((values, collector.finish_with_policy(policy)?))
21292 }
21293
21294 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
21296 if targets.is_empty() {
21297 return Ok(EvalPlan {
21298 total_vertices_to_evaluate: 0,
21299 layers: Vec::new(),
21300 cycles_detected: 0,
21301 dirty_count: 0,
21302 volatile_count: 0,
21303 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21304 estimated_parallel_layers: 0,
21305 target_cells: Vec::new(),
21306 });
21307 }
21308 if self.config.defer_graph_building && self.has_staged_formulas() {
21309 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
21310 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
21311 ));
21312 }
21313
21314 let addresses: Vec<String> = targets
21316 .iter()
21317 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
21318 .collect();
21319
21320 let mut target_addrs = Vec::new();
21322 for (sheet, row, col) in targets {
21323 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
21324 let coord = Coord::from_excel(*row, *col, true, true);
21325 target_addrs.push(CellRef::new(sheet_id, coord));
21326 }
21327 }
21328
21329 let mut target_vertex_ids = Vec::new();
21331 for addr in &target_addrs {
21332 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
21333 target_vertex_ids.push(*vertex_id);
21334 }
21335 }
21336
21337 if target_vertex_ids.is_empty() {
21338 return Ok(EvalPlan {
21339 total_vertices_to_evaluate: 0,
21340 layers: Vec::new(),
21341 cycles_detected: 0,
21342 dirty_count: 0,
21343 volatile_count: 0,
21344 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21345 estimated_parallel_layers: 0,
21346 target_cells: addresses,
21347 });
21348 }
21349
21350 let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
21352
21353 if precedents_to_eval.is_empty() {
21354 return Ok(EvalPlan {
21355 total_vertices_to_evaluate: 0,
21356 layers: Vec::new(),
21357 cycles_detected: 0,
21358 dirty_count: 0,
21359 volatile_count: 0,
21360 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21361 estimated_parallel_layers: 0,
21362 target_cells: addresses,
21363 });
21364 }
21365
21366 let mut dirty_count = 0;
21368 let mut volatile_count = 0;
21369 for &vertex_id in &precedents_to_eval {
21370 if self.graph.is_dirty(vertex_id) {
21371 dirty_count += 1;
21372 }
21373 if self.graph.is_volatile(vertex_id) {
21374 volatile_count += 1;
21375 }
21376 }
21377
21378 let scheduler = Scheduler::new(&self.graph);
21380 let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
21381
21382 let mut layers = Vec::new();
21384 let mut estimated_parallel_layers = 0;
21385 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
21386
21387 for layer in &schedule.layers {
21388 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
21389 if parallel_eligible {
21390 estimated_parallel_layers += 1;
21391 }
21392
21393 let sample_cells: Vec<String> = layer
21395 .vertices
21396 .iter()
21397 .take(5)
21398 .filter_map(|&vertex_id| {
21399 self.graph
21400 .get_cell_ref_for_vertex(vertex_id)
21401 .map(|cell_ref| {
21402 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
21403 format!(
21404 "{}!{}{}",
21405 sheet_name,
21406 Self::col_to_letters(cell_ref.coord.col()),
21407 cell_ref.coord.row() + 1
21408 )
21409 })
21410 })
21411 .collect();
21412
21413 layers.push(LayerInfo {
21414 vertex_count: layer.vertices.len(),
21415 parallel_eligible,
21416 sample_cells,
21417 });
21418 }
21419
21420 Ok(EvalPlan {
21421 total_vertices_to_evaluate: precedents_to_eval.len(),
21422 layers,
21423 cycles_detected: schedule.cycles.len(),
21424 dirty_count,
21425 volatile_count,
21426 parallel_enabled,
21427 estimated_parallel_layers,
21428 target_cells: addresses,
21429 })
21430 }
21431 fn create_evaluation_schedule(
21433 &mut self,
21434 to_evaluate: &[VertexId],
21435 ) -> Result<ScheduleBuildOutput, ExcelError> {
21436 self.graph.flush_pending_edge_deltas();
21439 if self.can_use_static_schedule_cache(to_evaluate) {
21440 if let Some(cached) = self.cached_static_schedule.as_ref()
21441 && cached.topology_epoch == self.topology_epoch
21442 && cached.candidate_vertices.as_slice() == to_evaluate
21443 {
21444 let meta = ScheduleBuildMeta {
21445 candidate_vertices: to_evaluate.len(),
21446 vdeps_vertices: 0,
21447 vdeps_edges: 0,
21448 builder_elapsed_ms: 0,
21449 used_virtual_schedule: false,
21450 schedule_cache_hit: true,
21451 schedule_cache_eligible: true,
21452 };
21453 return Ok((cached.schedule.clone(), FxHashMap::default(), meta));
21454 }
21455
21456 let (schedule, vdeps, mut meta) =
21457 self.create_evaluation_schedule_uncached(to_evaluate)?;
21458 meta.schedule_cache_hit = false;
21459 meta.schedule_cache_eligible = true;
21460 if vdeps.is_empty() {
21461 self.cached_static_schedule = Some(CachedScheduleEntry {
21462 topology_epoch: self.topology_epoch,
21463 candidate_vertices: to_evaluate.to_vec(),
21464 schedule: schedule.clone(),
21465 });
21466 }
21467 return Ok((schedule, vdeps, meta));
21468 }
21469
21470 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
21471 meta.schedule_cache_hit = false;
21472 meta.schedule_cache_eligible = false;
21473 Ok((schedule, vdeps, meta))
21474 }
21475
21476 fn create_evaluation_schedule_uncached(
21477 &self,
21478 to_evaluate: &[VertexId],
21479 ) -> Result<ScheduleBuildOutput, ExcelError> {
21480 let builder = VirtualDepBuilder::new(self);
21481 let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
21482 if self.config.enable_virtual_dep_telemetry {
21483 let build_started = crate::instant::FzInstant::now();
21484 let (vdeps, augmented) = builder.build(to_evaluate);
21485 let builder_elapsed_ms = build_started.elapsed().as_millis();
21486 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
21487 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
21488 } else {
21489 let (vdeps, augmented) = builder.build(to_evaluate);
21490 (vdeps, augmented, 0, 0)
21491 };
21492
21493 let mut final_evaluate = to_evaluate.to_vec();
21494 if !augmented.is_empty() {
21495 final_evaluate.extend(augmented);
21496 final_evaluate.sort_unstable();
21497 final_evaluate.dedup();
21498 }
21499
21500 let use_virtual = !vdeps.is_empty();
21501
21502 let scheduler = Scheduler::new(&self.graph);
21503 let schedule = if use_virtual {
21504 scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
21505 } else {
21506 scheduler.create_schedule(&final_evaluate)?
21507 };
21508
21509 let meta = ScheduleBuildMeta {
21510 candidate_vertices: to_evaluate.len(),
21511 vdeps_vertices: vdeps.len(),
21512 vdeps_edges,
21513 builder_elapsed_ms,
21514 used_virtual_schedule: use_virtual,
21515 schedule_cache_hit: false,
21516 schedule_cache_eligible: false,
21517 };
21518
21519 Ok((schedule, vdeps, meta))
21520 }
21521
21522 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
21523 !to_evaluate.is_empty()
21524 && to_evaluate.iter().copied().all(|v| {
21525 !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
21526 })
21527 }
21528
21529 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
21530 VirtualDepTelemetry {
21531 fallback_mode_activations: self.virtual_dep_fallback_activations,
21532 ..VirtualDepTelemetry::default()
21533 }
21534 }
21535
21536 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
21537 telemetry.candidate_vertices_total += meta.candidate_vertices;
21538 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
21539 telemetry.vdeps_edges_total += meta.vdeps_edges;
21540 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
21541 if meta.schedule_cache_eligible {
21542 if meta.schedule_cache_hit {
21543 telemetry.schedule_cache_hits += 1;
21544 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
21545 } else {
21546 telemetry.schedule_cache_misses += 1;
21547 }
21548 }
21549 if meta.used_virtual_schedule {
21550 telemetry.schedule_virtual_passes += 1;
21551 } else {
21552 telemetry.schedule_static_passes += 1;
21553 }
21554 }
21555
21556 fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
21557 vertices
21558 .iter()
21559 .copied()
21560 .filter(|vertex| self.graph.is_dynamic(*vertex))
21561 .filter_map(|vertex| {
21562 let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
21563 (!regions.is_empty()).then_some((vertex, regions))
21564 })
21565 .collect()
21566 }
21567
21568 fn extend_target_roots_with_dynamic_regions<'a>(
21569 &self,
21570 roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
21571 regions: impl Iterator<Item = &'a Region>,
21572 ) -> Result<(), ExcelError> {
21573 use crate::engine::target_preparation::TargetProducer;
21574 let request_id = self
21575 .active_evaluation_resource_request
21576 .as_ref()
21577 .map(|request| request.request_id);
21578 let root_count = roots.len();
21579 let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
21580 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
21581 for region in regions {
21582 for vertex in self.graph.formula_vertices() {
21583 let key = crate::formula_plane::region_index::RegionKey {
21584 sheet_id: self.graph.get_vertex_sheet_id(vertex),
21585 row: self.graph.vertex_coord(vertex).row(),
21586 col: self.graph.vertex_coord(vertex).col(),
21587 };
21588 if region.contains_key(key) {
21589 extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
21590 target_root_allocation_error(extended.len() + 1, request_id)
21591 })?;
21592 }
21593 }
21594 let authority = self.graph.formula_authority();
21595 for span_ref in authority.active_span_refs() {
21596 let Some(span) = authority.plane.spans.get(span_ref) else {
21597 continue;
21598 };
21599 let Some(demanded) =
21600 Region::from_domain(span.result_region.domain()).intersection(*region)
21601 else {
21602 continue;
21603 };
21604 extended
21605 .push(TargetProducer::Span { span_ref, demanded })
21606 .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
21607 }
21608 }
21609 *roots = extended.into_vec();
21610 Ok(())
21611 }
21612
21613 fn changed_virtual_dep_vertices(
21614 &mut self,
21615 to_evaluate: &[VertexId],
21616 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
21617 ) -> Vec<VertexId> {
21618 #[cfg(test)]
21619 if self.force_virtual_dep_changes_remaining_for_test > 0
21620 && let Some(vertex) = to_evaluate.first().copied()
21621 {
21622 self.force_virtual_dep_changes_remaining_for_test -= 1;
21623 return vec![vertex];
21624 }
21625 if !to_evaluate
21626 .iter()
21627 .copied()
21628 .any(|v| self.graph.is_dynamic(v))
21629 {
21630 return Vec::new();
21631 }
21632
21633 let builder = VirtualDepBuilder::new(self);
21634 let (new_vdeps, _) = builder.build(to_evaluate);
21635
21636 let mut candidates = FxHashSet::default();
21637 candidates.extend(old_vdeps.keys().copied());
21638 candidates.extend(new_vdeps.keys().copied());
21639
21640 let mut changed = Vec::new();
21641 for v in candidates {
21642 if old_vdeps.get(&v) != new_vdeps.get(&v) {
21643 changed.push(v);
21644 }
21645 }
21646 changed
21647 }
21648
21649 fn build_demand_subgraph(
21652 &self,
21653 target_vertices: &[VertexId],
21654 ) -> (
21655 Vec<VertexId>,
21656 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
21657 ) {
21658 #[cfg(feature = "tracing")]
21659 let _span =
21660 tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
21661 use rustc_hash::{FxHashMap, FxHashSet};
21662
21663 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
21664 let mut visited: FxHashSet<VertexId> = FxHashSet::default();
21665 let mut stack: Vec<VertexId> = Vec::new();
21666 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); for &t in target_vertices {
21669 stack.push(t);
21670 }
21671
21672 while let Some(v) = stack.pop() {
21673 if !visited.insert(v) {
21674 continue;
21675 }
21676 if !self.graph.vertex_exists(v) {
21677 continue;
21678 }
21679 match self.graph.get_vertex_kind(v) {
21687 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
21688 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
21689 to_evaluate.insert(v);
21690 }
21691 }
21692 VertexKind::NamedScalar
21693 | VertexKind::NamedArray
21694 | VertexKind::Range
21695 | VertexKind::InfiniteRange => {
21696 to_evaluate.insert(v);
21697 }
21698 _ => {}
21699 }
21700
21701 if let Some(dependencies) = self.graph.dependencies_slice(v) {
21712 for &dep in dependencies {
21713 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21714 stack.push(dep);
21715 }
21716 }
21717 } else {
21718 for dep in self.graph.get_dependencies(v) {
21719 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21720 stack.push(dep);
21721 }
21722 }
21723 } let builder = VirtualDepBuilder::new(self);
21725 let (vdeps_map, _) = builder.build(&[v]);
21726 if let Some(deps) = vdeps_map.get(&v) {
21727 for &u in deps {
21728 vdeps.entry(v).or_default().push(u);
21729 if !visited.contains(&u) {
21730 stack.push(u);
21731 }
21732 }
21733 }
21734 }
21735
21736 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
21737 result.sort_unstable();
21738 for deps in vdeps.values_mut() {
21740 deps.sort_unstable();
21741 deps.dedup();
21742 }
21743 (result, vdeps)
21744 }
21745
21746 fn col_to_letters(col: u32) -> String {
21748 col_letters_from_1based(col).expect("column index must be >= 1")
21749 }
21750
21751 pub fn evaluate_all_cancellable(
21753 &mut self,
21754 cancel: crate::engine::CancelToken,
21755 ) -> Result<EvalResult, ExcelError> {
21756 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
21757 engine.observe_function_semantic_epoch()?;
21758 engine.active_cancel_flag = Some(cancel.clone());
21759 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
21760 engine.active_cancel_flag = None;
21761 res
21762 })
21763 }
21764
21765 fn evaluate_all_cancellable_impl(
21766 &mut self,
21767 cancel_flag: &AtomicBool,
21768 ) -> Result<EvalResult, ExcelError> {
21769 self.begin_evaluation_request();
21770 let _source_cache = self.source_cache_session();
21771 self.validate_deterministic_mode()?;
21772 if self.config.defer_graph_building {
21773 self.build_graph_all()?;
21774 }
21775 if self.graph.formula_authority().active_span_count() > 0 {
21776 if cancel_flag.load(Ordering::Relaxed) {
21777 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21778 "Evaluation cancelled before FormulaPlane scheduling".to_string(),
21779 ));
21780 }
21781 return self.evaluate_authoritative_formula_plane_all();
21782 }
21783 self.reset_virtual_dep_telemetry_if_disabled();
21784 let start = crate::instant::FzInstant::now();
21785 let mut computed_vertices = 0;
21786 let mut cycle_errors = 0;
21787
21788 let mut replan_iterations = 0;
21789 const MAX_REPLAN: usize = 5;
21790 let mut telemetry = self
21791 .config
21792 .enable_virtual_dep_telemetry
21793 .then(|| self.start_virtual_dep_telemetry());
21794
21795 loop {
21796 if cancel_flag.load(Ordering::Relaxed) {
21797 if let Some(mut t) = telemetry {
21798 t.bailout_reason = Some("cancelled");
21799 t.replan_iterations = replan_iterations;
21800 self.last_virtual_dep_telemetry = t;
21801 }
21802 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21803 .with_message("Evaluation cancelled before scheduling".to_string()));
21804 }
21805
21806 let to_evaluate = self.graph.get_evaluation_vertices();
21807 if to_evaluate.is_empty() {
21808 if let Some(t) = telemetry.as_mut()
21809 && t.bailout_reason.is_none()
21810 {
21811 t.bailout_reason = Some("no_work");
21812 }
21813 break;
21814 }
21815
21816 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21817 if let Some(t) = telemetry.as_mut() {
21818 Self::accumulate_schedule_meta(t, &meta);
21819 }
21820
21821 for &unit in &schedule.units {
21824 match unit {
21825 ScheduleUnit::Cycle(i) => {
21826 if cancel_flag.load(Ordering::Relaxed) {
21828 if let Some(mut t) = telemetry {
21829 t.bailout_reason = Some("cancelled");
21830 t.replan_iterations = replan_iterations;
21831 self.last_virtual_dep_telemetry = t;
21832 }
21833 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21834 "Evaluation cancelled during cycle handling".to_string(),
21835 ));
21836 }
21837
21838 if self.handle_cycle_unit(
21839 schedule.unit_cycle(i),
21840 None,
21841 None,
21842 Some(cancel_flag),
21843 )? > 0
21844 {
21845 cycle_errors += 1;
21846 }
21847 }
21848 ScheduleUnit::Layer(i) => {
21849 let layer = schedule.unit_layer(i);
21850 if cancel_flag.load(Ordering::Relaxed) {
21852 if let Some(mut t) = telemetry {
21853 t.bailout_reason = Some("cancelled");
21854 t.replan_iterations = replan_iterations;
21855 self.last_virtual_dep_telemetry = t;
21856 }
21857 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21858 .with_message("Evaluation cancelled between layers".to_string()));
21859 }
21860
21861 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21863 computed_vertices +=
21864 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
21865 } else {
21866 computed_vertices +=
21867 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
21868 }
21869 }
21870 }
21871 }
21872
21873 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21874 if let Some(t) = telemetry.as_mut() {
21875 t.changed_vdeps_total += changed_vertices.len();
21876 }
21877 self.resource_checkpoint(0)?;
21878 self.graph.clear_dirty_flags(&to_evaluate);
21879 for v in &changed_vertices {
21880 self.graph.set_dirty(*v, true);
21881 }
21882
21883 if changed_vertices.is_empty() {
21884 if let Some(t) = telemetry.as_mut() {
21885 t.bailout_reason = Some("converged");
21886 }
21887 break;
21888 }
21889 if replan_iterations >= MAX_REPLAN {
21890 if let Some(mut t) = telemetry.take() {
21891 t.bailout_reason = Some("max_replan");
21892 t.replan_iterations = replan_iterations;
21893 self.last_virtual_dep_telemetry = t;
21894 }
21895 return Err(
21896 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21897 );
21898 }
21899 replan_iterations += 1;
21900 }
21901
21902 if let Some(mut t) = telemetry {
21903 t.replan_iterations = replan_iterations;
21904 self.last_virtual_dep_telemetry = t;
21905 }
21906
21907 self.redirty_for_next_recalc();
21909 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21910
21911 Ok(EvalResult {
21912 computed_vertices,
21913 cycle_errors,
21914 elapsed: start.elapsed(),
21915 })
21916 }
21917
21918 pub fn evaluate_until_cancellable(
21920 &mut self,
21921 targets: &[&str],
21922 cancel: crate::engine::CancelToken,
21923 ) -> Result<EvalResult, ExcelError> {
21924 self.observe_evaluation_resource_request(
21925 EvaluationRequestKind::TargetedCancellable,
21926 |engine| {
21927 engine.observe_function_semantic_epoch()?;
21928 engine.active_cancel_flag = Some(cancel.clone());
21929 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
21930 engine.active_cancel_flag = None;
21931 res
21932 },
21933 )
21934 }
21935
21936 fn evaluate_until_cancellable_impl(
21937 &mut self,
21938 targets: &[&str],
21939 cancel_flag: &AtomicBool,
21940 ) -> Result<EvalResult, ExcelError> {
21941 if cancel_flag.load(Ordering::Relaxed) {
21942 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21943 .with_message("Evaluation cancelled before target preparation"));
21944 }
21945 let mut typed_targets = Vec::with_capacity(targets.len());
21946 for target in targets {
21947 let (sheet, row, col) = self.parse_a1_notation(target)?;
21948 self.graph.sheet_id_mut(&sheet);
21949 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
21950 }
21951 self.evaluate_mixed_targets(&typed_targets, None)
21952 }
21953
21954 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
21955 let mut quoted = false;
21956 let mut separator = None;
21957 let bytes = address.as_bytes();
21958 let mut index = 0usize;
21959 while index < bytes.len() {
21960 match bytes[index] {
21961 b'\'' => {
21962 if quoted && bytes.get(index + 1) == Some(&b'\'') {
21963 index = index.saturating_add(1);
21964 } else {
21965 quoted = !quoted;
21966 }
21967 }
21968 b'!' if !quoted => separator = Some(index),
21969 _ => {}
21970 }
21971 index = index.saturating_add(1);
21972 }
21973 if quoted {
21974 return Err(ExcelError::new(ExcelErrorKind::Ref)
21975 .with_message(format!("Invalid quoted sheet reference `{address}`")));
21976 }
21977 let (sheet, cell_part) = match separator {
21978 Some(separator) => {
21979 let raw_sheet = &address[..separator];
21980 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
21981 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
21982 } else {
21983 raw_sheet.to_string()
21984 };
21985 (sheet, &address[separator + 1..])
21986 }
21987 None => (self.default_sheet_name().to_string(), address),
21988 };
21989
21990 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
21991 ExcelError::new(ExcelErrorKind::Ref)
21992 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
21993 })?;
21994
21995 Ok((sheet, row, col))
21996 }
21997
21998 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
22000 use formualizer_parse::parser::ASTNodeType;
22001 match &ast.node_type {
22002 ASTNodeType::Function { name, args, .. } => {
22003 if let Some(func) = self
22004 .get_function("", name)
22005 .or_else(|| crate::function_registry::get("", name))
22006 && func.caps().contains(crate::function::FnCaps::VOLATILE)
22007 {
22008 return true;
22009 }
22010 args.iter()
22011 .any(|arg| self.is_ast_volatile_with_provider(arg))
22012 }
22013 ASTNodeType::BinaryOp { left, right, .. } => {
22014 self.is_ast_volatile_with_provider(left)
22015 || self.is_ast_volatile_with_provider(right)
22016 }
22017 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
22018 ASTNodeType::Array(rows) => rows.iter().any(|row| {
22019 row.iter()
22020 .any(|cell| self.is_ast_volatile_with_provider(cell))
22021 }),
22022 _ => false,
22023 }
22024 }
22025
22026 fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
22028 let mut to_evaluate = FxHashSet::default();
22029 let mut visited = FxHashSet::default();
22030 let mut stack = Vec::new();
22031
22032 for &target in target_vertices {
22034 stack.push(target);
22035 }
22036
22037 while let Some(vertex_id) = stack.pop() {
22038 if !visited.insert(vertex_id) {
22039 continue; }
22041
22042 if self.graph.vertex_exists(vertex_id) {
22043 let kind = self.graph.get_vertex_kind(vertex_id);
22045 let needs_eval = match kind {
22046 super::vertex::VertexKind::FormulaScalar
22047 | super::vertex::VertexKind::FormulaArray => {
22048 self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
22049 }
22050 _ => false, };
22052
22053 if needs_eval {
22054 to_evaluate.insert(vertex_id);
22055 }
22056
22057 if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
22059 for &dep_id in dependencies {
22060 if !visited.contains(&dep_id) {
22061 stack.push(dep_id);
22062 }
22063 }
22064 } else {
22065 let dependencies = self.graph.get_dependencies(vertex_id);
22066 for dep_id in dependencies {
22067 if !visited.contains(&dep_id) {
22068 stack.push(dep_id);
22069 }
22070 }
22071 }
22072 }
22073 }
22074
22075 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
22076 result.sort_unstable();
22077 result
22078 }
22079
22080 fn evaluate_layer_sequential(
22082 &mut self,
22083 layer: &super::scheduler::Layer,
22084 ) -> Result<usize, ExcelError> {
22085 self.resource_checkpoint(layer.vertices.len() as u64)?;
22086 self.evaluate_layer_sequential_effects(layer)
22087 }
22088
22089 fn update_vertex_value_with_delta(
22090 &mut self,
22091 vertex_id: VertexId,
22092 new_value: LiteralValue,
22093 delta: &mut DeltaCollector,
22094 ) {
22095 if delta.mode != DeltaMode::Off
22096 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
22097 {
22098 let sheet_name = self.graph.sheet_name(cell.sheet_id);
22099 let old = self
22100 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22101 .unwrap_or(LiteralValue::Empty);
22102 if old != new_value {
22103 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
22104 }
22105 }
22106 self.graph.update_vertex_value(vertex_id, new_value.clone());
22107 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
22108 }
22109
22110 fn evaluate_layer_sequential_with_delta(
22111 &mut self,
22112 layer: &super::scheduler::Layer,
22113 delta: &mut DeltaCollector,
22114 ) -> Result<usize, ExcelError> {
22115 self.resource_checkpoint(layer.vertices.len() as u64)?;
22116 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
22117 }
22118
22119 fn evaluate_layer_sequential_cancellable(
22121 &mut self,
22122 layer: &super::scheduler::Layer,
22123 cancel_flag: &AtomicBool,
22124 ) -> Result<usize, ExcelError> {
22125 self.resource_checkpoint(layer.vertices.len() as u64)?;
22126 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
22127 }
22128
22129 fn evaluate_layer_sequential_cancellable_demand_driven(
22131 &mut self,
22132 layer: &super::scheduler::Layer,
22133 cancel_flag: &AtomicBool,
22134 ) -> Result<usize, ExcelError> {
22135 self.resource_checkpoint(layer.vertices.len() as u64)?;
22136 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
22137 }
22138
22139 fn evaluate_layer_parallel(
22141 &mut self,
22142 layer: &super::scheduler::Layer,
22143 ) -> Result<usize, ExcelError> {
22144 self.resource_checkpoint(layer.vertices.len() as u64)?;
22145 self.evaluate_layer_parallel_effects(layer)
22146 }
22147
22148 fn evaluate_layer_parallel_with_delta(
22149 &mut self,
22150 layer: &super::scheduler::Layer,
22151 delta: &mut DeltaCollector,
22152 ) -> Result<usize, ExcelError> {
22153 self.resource_checkpoint(layer.vertices.len() as u64)?;
22154 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
22155 }
22156
22157 fn evaluate_layer_parallel_cancellable(
22159 &mut self,
22160 layer: &super::scheduler::Layer,
22161 cancel_flag: &AtomicBool,
22162 ) -> Result<usize, ExcelError> {
22163 self.resource_checkpoint(layer.vertices.len() as u64)?;
22164 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
22165 }
22166
22167 fn apply_parallel_vertex_result(
22172 &mut self,
22173 vertex_id: VertexId,
22174 result: LiteralValue,
22175 mut delta: Option<&mut DeltaCollector>,
22176 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22177 ) -> Result<(), ExcelError> {
22178 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
22181 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
22182 && owner != vertex_id
22183 {
22184 return Ok(());
22185 }
22186
22187 let kind = self.graph.get_vertex_kind(vertex_id);
22188
22189 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
22191 if is_formula {
22192 match result {
22193 LiteralValue::Array(rows) => {
22194 self.apply_array_result_from_parallel(
22195 vertex_id,
22196 rows,
22197 delta.as_deref_mut(),
22198 overwritable_formulas,
22199 )?;
22200 }
22201 other => {
22202 self.apply_non_array_result_from_parallel(
22203 vertex_id,
22204 other,
22205 delta.as_deref_mut(),
22206 );
22207 }
22208 }
22209 return Ok(());
22210 }
22211
22212 if let Some(d) = delta {
22214 self.update_vertex_value_with_delta(vertex_id, result, d);
22215 } else {
22216 self.graph.update_vertex_value(vertex_id, result.clone());
22217 self.mirror_vertex_value_to_overlay(vertex_id, &result);
22218 }
22219 Ok(())
22220 }
22221
22222 fn apply_non_array_result_from_parallel(
22223 &mut self,
22224 vertex_id: VertexId,
22225 value: LiteralValue,
22226 delta: Option<&mut DeltaCollector>,
22227 ) {
22228 let spill_cells = self
22231 .graph
22232 .spill_cells_for_anchor(vertex_id)
22233 .map(|cells| cells.to_vec())
22234 .unwrap_or_default();
22235
22236 if let Some(d) = delta
22237 && d.mode != DeltaMode::Off
22238 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
22239 {
22240 if spill_cells.is_empty() {
22241 let old = self
22242 .read_cell_value(
22243 self.graph.sheet_name(anchor.sheet_id),
22244 anchor.coord.row() + 1,
22245 anchor.coord.col() + 1,
22246 )
22247 .unwrap_or(LiteralValue::Empty);
22248 if old != value {
22249 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22250 }
22251 } else {
22252 for cell in spill_cells.iter() {
22253 let sheet_name = self.graph.sheet_name(cell.sheet_id);
22254 let old = self
22255 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22256 .unwrap_or(LiteralValue::Empty);
22257 let new = if cell.sheet_id == anchor.sheet_id
22258 && cell.coord.row() == anchor.coord.row()
22259 && cell.coord.col() == anchor.coord.col()
22260 {
22261 value.clone()
22262 } else {
22263 LiteralValue::Empty
22264 };
22265 Self::record_cell_if_changed(d, cell, &old, &new);
22266 }
22267 }
22268 }
22269
22270 self.graph.clear_spill_region(vertex_id);
22271 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
22272 self.record_formula_plane_structural_change(scope);
22273 }
22274
22275 if self.config.arrow_storage_enabled
22276 && self.config.delta_overlay_enabled
22277 && self.config.write_formula_overlay_enabled
22278 {
22279 let empty = LiteralValue::Empty;
22280 for cell in spill_cells.iter() {
22281 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
22282 self.mirror_value_to_computed_overlay(
22283 &sheet_name,
22284 cell.coord.row() + 1,
22285 cell.coord.col() + 1,
22286 &empty,
22287 );
22288 }
22289 }
22290
22291 self.graph.update_vertex_value(vertex_id, value.clone());
22292 self.mirror_vertex_value_to_overlay(vertex_id, &value);
22293 }
22294
22295 fn apply_array_result_from_parallel(
22296 &mut self,
22297 vertex_id: VertexId,
22298 rows: Vec<Vec<LiteralValue>>,
22299 mut delta: Option<&mut DeltaCollector>,
22300 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22301 ) -> Result<(), ExcelError> {
22302 self.graph
22304 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
22305
22306 let anchor = self
22307 .graph
22308 .get_cell_ref(vertex_id)
22309 .expect("cell ref for vertex");
22310 let sheet_id = anchor.sheet_id;
22311 let h = rows.len() as u32;
22312 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
22313
22314 let spill_cells = (h as u64).saturating_mul(w as u64);
22316 if spill_cells > self.config.spill.max_spill_cells as u64 {
22317 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22318 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22319 .with_message("SpillTooLarge")
22320 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22321 expected_rows: h,
22322 expected_cols: w,
22323 });
22324 let spill_val = LiteralValue::Error(spill_err.clone());
22325 if let Some(d) = delta.as_deref_mut()
22326 && d.mode != DeltaMode::Off
22327 {
22328 let old = self
22329 .read_cell_value(
22330 self.graph.sheet_name(anchor.sheet_id),
22331 anchor.coord.row() + 1,
22332 anchor.coord.col() + 1,
22333 )
22334 .unwrap_or(LiteralValue::Empty);
22335 if old != spill_val {
22336 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22337 }
22338 }
22339 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22340 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22341 return Ok(());
22342 }
22343
22344 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
22348 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
22349 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
22350 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22351 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22352 .with_message("Spill exceeds sheet bounds")
22353 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22354 expected_rows: h,
22355 expected_cols: w,
22356 });
22357 let spill_val = LiteralValue::Error(spill_err.clone());
22358 if let Some(d) = delta.as_deref_mut()
22359 && d.mode != DeltaMode::Off
22360 {
22361 let old = self
22362 .read_cell_value(
22363 self.graph.sheet_name(anchor.sheet_id),
22364 anchor.coord.row() + 1,
22365 anchor.coord.col() + 1,
22366 )
22367 .unwrap_or(LiteralValue::Empty);
22368 if old != spill_val {
22369 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22370 }
22371 }
22372 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22373 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22374 return Ok(());
22375 }
22376
22377 let mut targets = Vec::new();
22378 for r in 0..h {
22379 for c in 0..w {
22380 targets.push(self.graph.make_cell_ref_internal(
22381 sheet_id,
22382 anchor.coord.row() + r,
22383 anchor.coord.col() + c,
22384 ));
22385 }
22386 }
22387
22388 match self.spill_mgr.reserve(
22389 vertex_id,
22390 anchor,
22391 SpillShape { rows: h, cols: w },
22392 SpillMeta {
22393 epoch: self.recalc_epoch,
22394 config: self.config.spill,
22395 },
22396 ) {
22397 Ok(()) => {
22398 if let Err(e) = self.commit_spill_and_mirror(
22399 vertex_id,
22400 &targets,
22401 rows.clone(),
22402 delta.as_deref_mut(),
22403 overwritable_formulas,
22404 ) {
22405 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22406 let err_val = LiteralValue::Error(e.clone());
22407 if let Some(d) = delta.as_deref_mut()
22408 && d.mode != DeltaMode::Off
22409 {
22410 let old = self
22411 .read_cell_value(
22412 self.graph.sheet_name(anchor.sheet_id),
22413 anchor.coord.row() + 1,
22414 anchor.coord.col() + 1,
22415 )
22416 .unwrap_or(LiteralValue::Empty);
22417 if old != err_val {
22418 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22419 }
22420 }
22421 self.graph.update_vertex_value(vertex_id, err_val.clone());
22422 self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
22423 return Ok(());
22424 }
22425
22426 let top_left = rows
22428 .first()
22429 .and_then(|r| r.first())
22430 .cloned()
22431 .unwrap_or(LiteralValue::Empty);
22432 self.graph.update_vertex_value(vertex_id, top_left.clone());
22433 self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
22434 Ok(())
22435 }
22436 Err(e) => {
22437 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22438 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22439 .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
22440 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22441 expected_rows: h,
22442 expected_cols: w,
22443 });
22444 let spill_val = LiteralValue::Error(spill_err.clone());
22445 if let Some(d) = delta
22446 && d.mode != DeltaMode::Off
22447 {
22448 let old = self
22449 .read_cell_value(
22450 self.graph.sheet_name(anchor.sheet_id),
22451 anchor.coord.row() + 1,
22452 anchor.coord.col() + 1,
22453 )
22454 .unwrap_or(LiteralValue::Empty);
22455 if old != spill_val {
22456 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22457 }
22458 }
22459 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22460 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22461 Ok(())
22462 }
22463 }
22464 }
22465
22466 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
22468 if !self.graph.vertex_exists(vertex_id) {
22470 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
22471 .with_message(format!("Vertex not found: {vertex_id:?}")));
22472 }
22473
22474 let kind = self.graph.get_vertex_kind(vertex_id);
22476 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
22477
22478 let ast_id = match kind {
22479 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
22480 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
22481 ast_id
22482 } else {
22483 return Ok(LiteralValue::Number(0.0));
22484 }
22485 }
22486 VertexKind::Empty | VertexKind::Cell => {
22487 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
22488 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22489 let row = cell_ref.coord.row() + 1;
22490 let col = cell_ref.coord.col() + 1;
22491 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
22492 return Ok(v);
22493 }
22494 }
22495 return Ok(LiteralValue::Number(0.0));
22496 }
22497 VertexKind::NamedScalar => {
22498 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22499 ExcelError::new(ExcelErrorKind::Name)
22500 .with_message("Named range metadata missing".to_string())
22501 })?;
22502
22503 return match &named_range.definition {
22504 NamedDefinition::Cell(cell_ref) => {
22505 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22506 Ok(self
22507 .get_cell_value(
22508 sheet_name,
22509 cell_ref.coord.row() + 1,
22510 cell_ref.coord.col() + 1,
22511 )
22512 .unwrap_or(LiteralValue::Empty))
22513 }
22514 NamedDefinition::Literal(v) => Ok(v.clone()),
22515 NamedDefinition::Formula { ast, .. } => {
22516 let context_sheet = match named_range.scope {
22517 NameScope::Sheet(id) => id,
22518 NameScope::Workbook => sheet_id,
22519 };
22520 let sheet_name = self.graph.sheet_name(context_sheet);
22521 let cell_ref = self
22522 .graph
22523 .get_cell_ref(vertex_id)
22524 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22525 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22526 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
22527 }
22528 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
22529 .with_message("Range-valued name evaluated as scalar".to_string())),
22530 };
22531 }
22532 VertexKind::NamedArray => {
22533 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22534 ExcelError::new(ExcelErrorKind::Name)
22535 .with_message("Named range metadata missing".to_string())
22536 })?;
22537
22538 return match &named_range.definition {
22539 NamedDefinition::Range(range_ref) => {
22540 if range_ref.start.sheet_id != range_ref.end.sheet_id {
22541 return Err(ExcelError::new(ExcelErrorKind::Ref)
22542 .with_message("Named range cannot span sheets".to_string()));
22543 }
22544 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
22545 let sr0 = range_ref.start.coord.row();
22546 let sc0 = range_ref.start.coord.col();
22547 let er0 = range_ref.end.coord.row();
22548 let ec0 = range_ref.end.coord.col();
22549 if sr0 > er0 || sc0 > ec0 {
22550 return Err(ExcelError::new(ExcelErrorKind::Ref)
22551 .with_message("Invalid named range bounds".to_string()));
22552 }
22553
22554 let h = (er0 - sr0 + 1) as usize;
22555 let w = (ec0 - sc0 + 1) as usize;
22556 let cell_count = (h as u64).saturating_mul(w as u64);
22557 if cell_count > self.config.spill.max_spill_cells as u64 {
22558 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
22559 "Named range too large to materialize as an array".to_string(),
22560 ));
22561 }
22562
22563 let mut rows = Vec::with_capacity(h);
22564 for r0 in sr0..=er0 {
22565 let mut row = Vec::with_capacity(w);
22566 for c0 in sc0..=ec0 {
22567 let v = self
22568 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
22569 .unwrap_or(LiteralValue::Empty);
22570 row.push(v);
22571 }
22572 rows.push(row);
22573 }
22574 Ok(LiteralValue::Array(rows))
22575 }
22576 NamedDefinition::Cell(cell_ref) => {
22577 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22578 let row = cell_ref.coord.row() + 1;
22579 let col = cell_ref.coord.col() + 1;
22580 let v = self
22581 .get_cell_value(sheet_name, row, col)
22582 .unwrap_or(LiteralValue::Empty);
22583 Ok(LiteralValue::Array(vec![vec![v]]))
22584 }
22585 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
22586 NamedDefinition::Formula { ast, .. } => {
22587 let context_sheet = match named_range.scope {
22588 NameScope::Sheet(id) => id,
22589 NameScope::Workbook => sheet_id,
22590 };
22591 let sheet_name = self.graph.sheet_name(context_sheet);
22592 let cell_ref = self
22593 .graph
22594 .get_cell_ref(vertex_id)
22595 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22596 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22597 match interpreter.evaluate_ast(ast) {
22598 Ok(cv) => {
22599 let v = cv.into_literal();
22600 match v {
22601 LiteralValue::Array(_) => Ok(v),
22602 other => Ok(LiteralValue::Array(vec![vec![other]])),
22603 }
22604 }
22605 Err(err) => Ok(LiteralValue::Error(err)),
22606 }
22607 }
22608 };
22609 }
22610 VertexKind::InfiniteRange
22611 | VertexKind::Range
22612 | VertexKind::External
22613 | VertexKind::Table => {
22614 return Ok(LiteralValue::Number(0.0));
22616 }
22617 };
22618
22619 let sheet_name = self.graph.sheet_name(sheet_id);
22621 let cell_ref = self
22622 .graph
22623 .get_cell_ref(vertex_id)
22624 .expect("cell ref for vertex");
22625 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22626
22627 interpreter
22628 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
22629 .map(|cv| {
22630 crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
22631 })
22632 }
22633
22634 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
22636 self.thread_pool.as_ref()
22637 }
22638}
22639
22640#[derive(Default)]
22641struct RowBoundsCache {
22642 snapshot: u64,
22643 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
22645}
22646
22647impl RowBoundsCache {
22648 fn new(snapshot: u64) -> Self {
22649 Self {
22650 snapshot,
22651 map: Default::default(),
22652 }
22653 }
22654 fn get_row_bounds(
22655 &self,
22656 sheet_id: SheetId,
22657 col_idx: usize,
22658 snapshot: u64,
22659 ) -> Option<(Option<u32>, Option<u32>)> {
22660 if self.snapshot != snapshot {
22661 return None;
22662 }
22663 self.map.get(&(sheet_id as u32, col_idx)).copied()
22664 }
22665 fn put_row_bounds(
22666 &mut self,
22667 sheet_id: SheetId,
22668 col_idx: usize,
22669 snapshot: u64,
22670 bounds: (Option<u32>, Option<u32>),
22671 ) {
22672 if self.snapshot != snapshot {
22673 self.snapshot = snapshot;
22674 self.map.clear();
22675 }
22676 self.map.insert((sheet_id as u32, col_idx), bounds);
22677 }
22678}
22679
22680struct UsedAxisBoundsCache {
22681 snapshot: u64,
22682 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22683 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22684 #[cfg(test)]
22685 row_hits: std::sync::atomic::AtomicUsize,
22686 #[cfg(test)]
22687 row_misses: std::sync::atomic::AtomicUsize,
22688 #[cfg(test)]
22689 col_hits: std::sync::atomic::AtomicUsize,
22690 #[cfg(test)]
22691 col_misses: std::sync::atomic::AtomicUsize,
22692}
22693
22694impl UsedAxisBoundsCache {
22695 fn new(snapshot: u64) -> Self {
22696 Self {
22697 snapshot,
22698 row_bounds_by_col_span: Default::default(),
22699 col_bounds_by_row_span: Default::default(),
22700 #[cfg(test)]
22701 row_hits: std::sync::atomic::AtomicUsize::new(0),
22702 #[cfg(test)]
22703 row_misses: std::sync::atomic::AtomicUsize::new(0),
22704 #[cfg(test)]
22705 col_hits: std::sync::atomic::AtomicUsize::new(0),
22706 #[cfg(test)]
22707 col_misses: std::sync::atomic::AtomicUsize::new(0),
22708 }
22709 }
22710
22711 fn reset_for_snapshot(&mut self, snapshot: u64) {
22712 if self.snapshot != snapshot {
22713 self.snapshot = snapshot;
22714 self.row_bounds_by_col_span.clear();
22715 self.col_bounds_by_row_span.clear();
22716 }
22717 }
22718
22719 fn get_row_bounds(
22720 &self,
22721 sheet_id: SheetId,
22722 start_col: u32,
22723 end_col: u32,
22724 snapshot: u64,
22725 ) -> Option<Option<(u32, u32)>> {
22726 if self.snapshot != snapshot {
22727 return None;
22728 }
22729 let cached = self
22730 .row_bounds_by_col_span
22731 .get(&(sheet_id, start_col, end_col))
22732 .copied();
22733 #[cfg(test)]
22734 if cached.is_some() {
22735 self.row_hits.fetch_add(1, Ordering::Relaxed);
22736 }
22737 cached
22738 }
22739
22740 fn put_row_bounds(
22741 &mut self,
22742 sheet_id: SheetId,
22743 start_col: u32,
22744 end_col: u32,
22745 snapshot: u64,
22746 bounds: Option<(u32, u32)>,
22747 ) {
22748 self.reset_for_snapshot(snapshot);
22749 self.row_bounds_by_col_span
22750 .insert((sheet_id, start_col, end_col), bounds);
22751 #[cfg(test)]
22752 self.row_misses.fetch_add(1, Ordering::Relaxed);
22753 }
22754
22755 fn get_col_bounds(
22756 &self,
22757 sheet_id: SheetId,
22758 start_row: u32,
22759 end_row: u32,
22760 snapshot: u64,
22761 ) -> Option<Option<(u32, u32)>> {
22762 if self.snapshot != snapshot {
22763 return None;
22764 }
22765 let cached = self
22766 .col_bounds_by_row_span
22767 .get(&(sheet_id, start_row, end_row))
22768 .copied();
22769 #[cfg(test)]
22770 if cached.is_some() {
22771 self.col_hits.fetch_add(1, Ordering::Relaxed);
22772 }
22773 cached
22774 }
22775
22776 fn put_col_bounds(
22777 &mut self,
22778 sheet_id: SheetId,
22779 start_row: u32,
22780 end_row: u32,
22781 snapshot: u64,
22782 bounds: Option<(u32, u32)>,
22783 ) {
22784 self.reset_for_snapshot(snapshot);
22785 self.col_bounds_by_row_span
22786 .insert((sheet_id, start_row, end_row), bounds);
22787 #[cfg(test)]
22788 self.col_misses.fetch_add(1, Ordering::Relaxed);
22789 }
22790}
22791
22792#[derive(Default)]
22794pub struct ShimSpillManager {
22795 region_locks: RegionLockManager,
22796 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
22797}
22798
22799impl ShimSpillManager {
22800 pub(crate) fn reserve(
22801 &mut self,
22802 owner: VertexId,
22803 anchor_cell: CellRef,
22804 shape: SpillShape,
22805 _meta: SpillMeta,
22806 ) -> Result<(), ExcelError> {
22807 let region = crate::engine::spill::Region {
22809 sheet_id: anchor_cell.sheet_id as u32,
22810 row_start: anchor_cell.coord.row(),
22811 row_end: anchor_cell
22812 .coord
22813 .row()
22814 .saturating_add(shape.rows)
22815 .saturating_sub(1),
22816 col_start: anchor_cell.coord.col(),
22817 col_end: anchor_cell
22818 .coord
22819 .col()
22820 .saturating_add(shape.cols)
22821 .saturating_sub(1),
22822 };
22823 match self.region_locks.reserve(region, owner) {
22824 Ok(id) => {
22825 if id != 0 {
22826 self.active_locks.insert(owner, id);
22827 }
22828 Ok(())
22829 }
22830 Err(e) => Err(e),
22831 }
22832 }
22833
22834 pub(crate) fn release_owner(&mut self, owner: VertexId) {
22840 if let Some(id) = self.active_locks.remove(&owner) {
22841 self.region_locks.release(id);
22842 }
22843 }
22844
22845 pub(crate) fn commit_array_with_value_probe<F>(
22846 &mut self,
22847 graph: &mut DependencyGraph,
22848 anchor_vertex: VertexId,
22849 targets: &[CellRef],
22850 rows: Vec<Vec<LiteralValue>>,
22851 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22852 mut value_probe: F,
22853 ) -> Result<(), ExcelError>
22854 where
22855 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
22856 {
22857 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
22858
22859 let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
22863 anchor_vertex,
22864 targets,
22865 overwritable_formulas,
22866 );
22867 if let Err(e) = plan_res {
22868 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22869 self.region_locks.release(id);
22870 }
22871 return Err(e);
22872 }
22873
22874 if !graph.value_cache_enabled() {
22875 let (expected_rows, expected_cols) = if targets.is_empty() {
22877 (0u32, 0u32)
22878 } else {
22879 let mut min_r = u32::MAX;
22880 let mut max_r = 0u32;
22881 let mut min_c = u32::MAX;
22882 let mut max_c = 0u32;
22883 for cell in targets {
22884 let r = cell.coord.row();
22885 let c = cell.coord.col();
22886 min_r = min_r.min(r);
22887 max_r = max_r.max(r);
22888 min_c = min_c.min(c);
22889 max_c = max_c.max(c);
22890 }
22891 (
22892 max_r.saturating_sub(min_r).saturating_add(1),
22893 max_c.saturating_sub(min_c).saturating_add(1),
22894 )
22895 };
22896
22897 let anchor_cell = graph
22898 .get_cell_ref(anchor_vertex)
22899 .expect("anchor cell ref for spill commit");
22900
22901 for cell in targets {
22902 if *cell == anchor_cell {
22904 continue;
22905 }
22906 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
22908 continue;
22909 }
22910 if let Some(&vid) = graph.get_vertex_id_for_address(cell)
22912 && vid != anchor_vertex
22913 {
22914 match graph.get_vertex_kind(vid) {
22915 crate::engine::vertex::VertexKind::FormulaScalar
22916 | crate::engine::vertex::VertexKind::FormulaArray => {
22917 continue;
22919 }
22920 _ => {}
22921 }
22922 }
22923
22924 if let Some(v) = value_probe(graph, cell)
22925 && !matches!(v, LiteralValue::Empty)
22926 {
22927 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22928 self.region_locks.release(id);
22929 }
22930 return Err(ExcelError::new(ExcelErrorKind::Spill)
22931 .with_message("BlockedByValue")
22932 .with_extra(ExcelErrorExtra::Spill {
22933 expected_rows,
22934 expected_cols,
22935 }));
22936 }
22937 }
22938 }
22939
22940 let commit_res = graph.commit_spill_region_atomic_with_fault(
22941 anchor_vertex,
22942 targets.to_vec(),
22943 rows,
22944 None,
22945 );
22946 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22947 self.region_locks.release(id);
22948 }
22949 commit_res.map(|_| ())
22950 }
22951
22952 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
22954 &mut self,
22955 engine: &mut Engine<R>,
22956 anchor_vertex: VertexId,
22957 targets: &[CellRef],
22958 rows: Vec<Vec<LiteralValue>>,
22959 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22960 ) -> Result<(), ExcelError> {
22961 let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
22963 anchor_vertex,
22964 targets,
22965 overwritable_formulas,
22966 );
22967 if let Err(e) = plan_res {
22968 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22969 self.region_locks.release(id);
22970 }
22971 return Err(e);
22972 }
22973
22974 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
22975 anchor_vertex,
22976 targets.to_vec(),
22977 rows.clone(),
22978 None,
22979 );
22980 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22981 self.region_locks.release(id);
22982 }
22983 commit_res.map(|_| ())?;
22984
22985 if engine.config.arrow_storage_enabled
22987 && engine.config.delta_overlay_enabled
22988 && engine.config.write_formula_overlay_enabled
22989 {
22990 for (idx, cell) in targets.iter().enumerate() {
22992 let (r_off, c_off) = {
22993 if rows.is_empty() || rows[0].is_empty() {
22994 (0usize, 0usize)
22995 } else {
22996 let width = rows[0].len();
22997 (idx / width, idx % width)
22998 }
22999 };
23000 let v = rows
23001 .get(r_off)
23002 .and_then(|r| r.get(c_off))
23003 .cloned()
23004 .unwrap_or(LiteralValue::Empty);
23005 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
23006 engine.mirror_value_to_computed_overlay(
23007 &sheet_name,
23008 cell.coord.row() + 1,
23009 cell.coord.col() + 1,
23010 &v,
23011 );
23012 }
23013 }
23014 Ok(())
23015 }
23016}
23017
23018impl<R> Engine<R>
23019where
23020 R: EvaluationContext,
23021{
23022 fn resolve_shared_ref(
23023 &self,
23024 reference: &ReferenceType,
23025 current_sheet: &str,
23026 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
23027 use formualizer_common::{
23028 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
23029 SheetRef as SharedRef,
23030 };
23031
23032 let sr = match reference {
23034 ReferenceType::Cell {
23035 sheet,
23036 row,
23037 col,
23038 row_abs,
23039 col_abs,
23040 } => {
23041 let row0 = row
23042 .checked_sub(1)
23043 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23044 let col0 = col
23045 .checked_sub(1)
23046 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23047 let sheet_loc = match sheet.as_deref() {
23048 Some(name) => SheetLocator::from_name(name),
23049 None => SheetLocator::Current,
23050 };
23051 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
23052 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
23053 }
23054 ReferenceType::Range {
23055 sheet,
23056 start_row,
23057 start_col,
23058 end_row,
23059 end_col,
23060 start_row_abs,
23061 start_col_abs,
23062 end_row_abs,
23063 end_col_abs,
23064 } => {
23065 let sheet_loc = match sheet.as_deref() {
23066 Some(name) => SheetLocator::from_name(name),
23067 None => SheetLocator::Current,
23068 };
23069 let sr = start_row
23070 .map(|r| {
23071 r.checked_sub(1)
23072 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23073 })
23074 .transpose()?;
23075 let sc = start_col
23076 .map(|c| {
23077 c.checked_sub(1)
23078 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23079 })
23080 .transpose()?;
23081 let er = end_row
23082 .map(|r| {
23083 r.checked_sub(1)
23084 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23085 })
23086 .transpose()?;
23087 let ec = end_col
23088 .map(|c| {
23089 c.checked_sub(1)
23090 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23091 })
23092 .transpose()?;
23093 let range = SharedRangeRef::from_parts(
23094 sheet_loc,
23095 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
23096 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
23097 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
23098 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
23099 )
23100 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
23101 SharedRef::Range(range)
23102 }
23103 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23104 };
23105
23106 let current_id = self
23107 .graph
23108 .sheet_id(current_sheet)
23109 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23110
23111 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
23112 match loc {
23113 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
23114 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
23115 SheetLocator::Name(name) => {
23116 let n = name.as_ref();
23117 self.graph
23118 .sheet_id(n)
23119 .map(SheetLocator::Id)
23120 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23121 }
23122 }
23123 };
23124
23125 match sr {
23126 SharedRef::Cell(cell) => {
23127 let owned = cell.into_owned();
23128 let sheet = resolve_loc(owned.sheet)?;
23129 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
23130 }
23131 SharedRef::Range(range) => {
23132 let owned = range.into_owned();
23133 let sheet = resolve_loc(owned.sheet)?;
23134 Ok(SharedRef::Range(SharedRangeRef {
23135 sheet,
23136 start_row: owned.start_row,
23137 start_col: owned.start_col,
23138 end_row: owned.end_row,
23139 end_col: owned.end_col,
23140 }))
23141 }
23142 }
23143 }
23144}
23145
23146impl<R> crate::traits::ReferenceResolver for Engine<R>
23149where
23150 R: EvaluationContext,
23151{
23152 fn resolve_cell_reference(
23153 &self,
23154 sheet: Option<&str>,
23155 row: u32,
23156 col: u32,
23157 ) -> Result<LiteralValue, ExcelError> {
23158 let Some(sheet_name) = sheet else {
23169 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23170 "Unqualified cell reference resolved without sheet context".to_string(),
23171 ));
23172 };
23173 if let Some(v) = self.get_cell_value(sheet_name, row, col) {
23176 Ok(v)
23177 } else {
23178 Ok(LiteralValue::Number(0.0))
23180 }
23181 }
23182}
23183
23184impl<R> crate::traits::RangeResolver for Engine<R>
23185where
23186 R: EvaluationContext,
23187{
23188 fn resolve_range_reference(
23189 &self,
23190 sheet: Option<&str>,
23191 sr: Option<u32>,
23192 sc: Option<u32>,
23193 er: Option<u32>,
23194 ec: Option<u32>,
23195 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
23196 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
23199 }
23200}
23201
23202impl<R> crate::traits::NamedRangeResolver for Engine<R>
23203where
23204 R: EvaluationContext,
23205{
23206 fn resolve_named_range_reference(
23207 &self,
23208 name: &str,
23209 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
23210 self.resolver.resolve_named_range_reference(name)
23211 }
23212}
23213
23214impl<R> crate::traits::TableResolver for Engine<R>
23215where
23216 R: EvaluationContext,
23217{
23218 fn resolve_table_reference(
23219 &self,
23220 tref: &formualizer_parse::parser::TableReference,
23221 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23222 self.resolver.resolve_table_reference(tref)
23223 }
23224}
23225
23226impl<R> crate::traits::SourceResolver for Engine<R>
23227where
23228 R: EvaluationContext,
23229{
23230 fn source_scalar_version(&self, name: &str) -> Option<u64> {
23231 self.resolver.source_scalar_version(name)
23232 }
23233
23234 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
23235 self.resolver.resolve_source_scalar(name)
23236 }
23237
23238 fn source_table_version(&self, name: &str) -> Option<u64> {
23239 self.resolver.source_table_version(name)
23240 }
23241
23242 fn resolve_source_table(
23243 &self,
23244 name: &str,
23245 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23246 self.resolver.resolve_source_table(name)
23247 }
23248}
23249
23250impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
23252
23253impl<R> crate::traits::FunctionProvider for Engine<R>
23255where
23256 R: EvaluationContext,
23257{
23258 fn planning_semantic_revision(&self) -> Option<u64> {
23259 self.resolver.planning_semantic_revision()
23260 }
23261
23262 fn get_function(
23263 &self,
23264 prefix: &str,
23265 name: &str,
23266 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23267 self.resolver.get_function(prefix, name)
23268 }
23269
23270 fn get_function_for_planning(
23271 &self,
23272 prefix: &str,
23273 name: &str,
23274 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23275 self.resolver.get_function_for_planning(prefix, name)
23276 }
23277}
23278
23279impl<R> Engine<R>
23280where
23281 R: EvaluationContext,
23282{
23283 pub(crate) fn semantic_used_rows_for_columns(
23291 &self,
23292 sheet: &str,
23293 start_col: u32,
23294 end_col: u32,
23295 ) -> Option<(u32, u32)> {
23296 let arrow_bounds = self
23297 .sheet_store()
23298 .sheet(sheet)
23299 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23300 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23301 Self::union_used_bounds(arrow_bounds, formula_bounds)
23302 }
23303
23304 pub(crate) fn semantic_used_cols_for_rows(
23305 &self,
23306 sheet: &str,
23307 start_row: u32,
23308 end_row: u32,
23309 ) -> Option<(u32, u32)> {
23310 let arrow_bounds = self
23311 .sheet_store()
23312 .sheet(sheet)
23313 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23314 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23315 Self::union_used_bounds(arrow_bounds, formula_bounds)
23316 }
23317}
23318
23319impl<R> crate::traits::EvaluationContext for Engine<R>
23321where
23322 R: EvaluationContext,
23323{
23324 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
23325 &self.clock
23326 }
23327
23328 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
23329 self.thread_pool.as_ref()
23330 }
23331
23332 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
23333 self.active_cancel_flag.clone()
23334 }
23335
23336 fn chunk_hint(&self) -> Option<usize> {
23337 let hint =
23339 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
23340 Some(hint.clamp(1024, 1 << 20)) }
23342
23343 fn volatile_level(&self) -> crate::traits::VolatileLevel {
23344 self.config.volatile_level
23345 }
23346
23347 fn workbook_seed(&self) -> u64 {
23348 self.config.workbook_seed
23349 }
23350
23351 fn recalc_epoch(&self) -> u64 {
23352 self.recalc_epoch
23353 }
23354
23355 fn workbook_sheet_count(&self) -> Option<usize> {
23356 Some(self.graph.sheet_reg().active_len())
23357 }
23358
23359 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
23360 self.graph.sheet_reg().active_position(sheet)
23361 }
23362
23363 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
23364 self.sheet_index_by_name(current_sheet)
23365 }
23366
23367 fn inspect_reference(
23368 &self,
23369 reference: &ReferenceType,
23370 current_sheet: &str,
23371 ) -> Result<Option<ReferenceInfo>, ExcelError> {
23372 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
23373 let sheet_id = self
23374 .graph
23375 .sheet_id(sheet_name)
23376 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23377 let sheet_index = self
23378 .graph
23379 .sheet_reg()
23380 .active_position_by_id(sheet_id)
23381 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23382 Ok((sheet_id, sheet_index))
23383 };
23384
23385 let cell_info =
23386 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
23387 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23388 let row0 = row
23389 .checked_sub(1)
23390 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23391 let col0 = col
23392 .checked_sub(1)
23393 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23394 Ok(ReferenceInfo {
23395 first_sheet_index: Some(sheet_index),
23396 sheet_count: Some(1),
23397 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23398 })
23399 };
23400
23401 let range_info = |sheet_name: &str,
23402 start_row: Option<u32>,
23403 start_col: Option<u32>|
23404 -> Result<ReferenceInfo, ExcelError> {
23405 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23406 let row = start_row.unwrap_or(1);
23407 let col = start_col.unwrap_or(1);
23408 let row0 = row
23409 .checked_sub(1)
23410 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23411 let col0 = col
23412 .checked_sub(1)
23413 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23414 Ok(ReferenceInfo {
23415 first_sheet_index: Some(sheet_index),
23416 sheet_count: Some(1),
23417 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23418 })
23419 };
23420
23421 let info = match reference {
23422 ReferenceType::Cell {
23423 sheet, row, col, ..
23424 } => {
23425 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23426 cell_info(sheet_name, *row, *col)?
23427 }
23428 ReferenceType::Range {
23429 sheet,
23430 start_row,
23431 start_col,
23432 ..
23433 } => {
23434 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23435 range_info(sheet_name, *start_row, *start_col)?
23436 }
23437 ReferenceType::Cell3D {
23438 sheet_first,
23439 sheet_last,
23440 row,
23441 col,
23442 ..
23443 } => {
23444 let first = cell_info(sheet_first, *row, *col)?;
23445 ReferenceInfo {
23446 first_sheet_index: first.first_sheet_index,
23447 sheet_count: self
23448 .graph
23449 .sheet_reg()
23450 .active_span_len(sheet_first, sheet_last),
23451 first_cell: first.first_cell,
23452 }
23453 }
23454 ReferenceType::Range3D {
23455 sheet_first,
23456 sheet_last,
23457 start_row,
23458 start_col,
23459 ..
23460 } => {
23461 let first = range_info(sheet_first, *start_row, *start_col)?;
23462 ReferenceInfo {
23463 first_sheet_index: first.first_sheet_index,
23464 sheet_count: self
23465 .graph
23466 .sheet_reg()
23467 .active_span_len(sheet_first, sheet_last),
23468 first_cell: first.first_cell,
23469 }
23470 }
23471 ReferenceType::NamedRange(name) => {
23472 let current_id = self
23473 .graph
23474 .sheet_id(current_sheet)
23475 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23476 let named = self
23477 .graph
23478 .resolve_name_entry(name, current_id)
23479 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23480 match &named.definition {
23481 NamedDefinition::Cell(cell) => ReferenceInfo {
23482 first_sheet_index: self
23483 .graph
23484 .sheet_reg()
23485 .active_position_by_id(cell.sheet_id),
23486 sheet_count: Some(1),
23487 first_cell: Some(*cell),
23488 },
23489 NamedDefinition::Range(range) => ReferenceInfo {
23490 first_sheet_index: self
23491 .graph
23492 .sheet_reg()
23493 .active_position_by_id(range.start.sheet_id),
23494 sheet_count: Some(1),
23495 first_cell: Some(range.start),
23496 },
23497 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
23498 ReferenceInfo {
23499 first_sheet_index: None,
23500 sheet_count: None,
23501 first_cell: None,
23502 }
23503 }
23504 }
23505 }
23506 ReferenceType::Table(tref) => {
23507 let table = self
23508 .graph
23509 .resolve_table_entry(&tref.name)
23510 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23511 ReferenceInfo {
23512 first_sheet_index: self
23513 .graph
23514 .sheet_reg()
23515 .active_position_by_id(table.range.start.sheet_id),
23516 sheet_count: Some(1),
23517 first_cell: Some(table.range.start),
23518 }
23519 }
23520 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23521 };
23522
23523 Ok(Some(info))
23524 }
23525
23526 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
23527 let sheet_name = self.graph.sheet_name(cell.sheet_id);
23528 if sheet_name.is_empty() {
23529 return Err(ExcelError::new(ExcelErrorKind::Ref));
23530 }
23531 let row = cell.coord.row() + 1;
23532 let col = cell.coord.col() + 1;
23533
23534 if let Some(entries) = self.staged_formulas.get(sheet_name)
23535 && let Some(text) = entries.get(row, col)
23536 {
23537 return Ok(Some(if text.starts_with('=') {
23538 text.to_owned()
23539 } else {
23540 format!("={text}")
23541 }));
23542 }
23543
23544 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
23545 return Ok(None);
23546 };
23547 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
23548 }
23549
23550 fn used_rows_for_columns(
23551 &self,
23552 sheet: &str,
23553 start_col: u32,
23554 end_col: u32,
23555 ) -> Option<(u32, u32)> {
23556 let sheet_id = self.graph.sheet_id(sheet)?;
23558 let snap = self.data_snapshot_id();
23559 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23560 guard
23561 .as_ref()
23562 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
23563 }) {
23564 return cached;
23565 }
23566
23567 let arrow_bounds = self
23568 .sheet_store()
23569 .sheet(sheet)
23570 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23571 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23572 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23573 Some(bounds)
23574 } else {
23575 let sc0 = start_col.saturating_sub(1);
23576 let ec0 = end_col.saturating_sub(1);
23577 self.graph
23578 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
23579 .map(|(a0, b0)| (a0 + 1, b0 + 1))
23580 };
23581
23582 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23583 guard
23584 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23585 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
23586 }
23587
23588 computed
23589 }
23590
23591 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
23592 let sheet_id = self.graph.sheet_id(sheet)?;
23594 let snap = self.data_snapshot_id();
23595 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23596 guard
23597 .as_ref()
23598 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
23599 }) {
23600 return cached;
23601 }
23602
23603 let arrow_bounds = self
23604 .sheet_store()
23605 .sheet(sheet)
23606 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23607 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23608 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23609 Some(bounds)
23610 } else {
23611 let sr0 = start_row.saturating_sub(1);
23612 let er0 = end_row.saturating_sub(1);
23613 self.graph
23614 .used_col_bounds_for_rows(sheet_id, sr0, er0)
23615 .map(|(a0, b0)| (a0 + 1, b0 + 1))
23616 };
23617
23618 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23619 guard
23620 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23621 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
23622 }
23623
23624 computed
23625 }
23626
23627 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
23628 let _ = self.graph.sheet_id(sheet)?;
23629 Some((1_048_576, 16_384)) }
23633
23634 fn data_snapshot_id(&self) -> u64 {
23635 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
23636 }
23637
23638 fn backend_caps(&self) -> crate::traits::BackendCaps {
23639 crate::traits::BackendCaps {
23640 streaming: true,
23641 used_region: true,
23642 write: false,
23643 tables: false,
23644 async_stream: false,
23645 }
23646 }
23647
23648 fn build_lookup_index(
23649 &self,
23650 view: &RangeView<'_>,
23651 axis: LookupAxis,
23652 ) -> Option<Arc<LookupIndex>> {
23653 self.build_lookup_index_impl(view, axis)
23654 }
23655
23656 fn date_system(&self) -> crate::engine::DateSystem {
23659 self.config.date_system
23660 }
23661 fn resolve_range_view<'c>(
23663 &'c self,
23664 reference: &ReferenceType,
23665 current_sheet: &str,
23666 ) -> Result<RangeView<'c>, ExcelError> {
23667 match reference {
23668 ReferenceType::External(ext) => {
23669 let name = ext.raw.as_str();
23670 match ext.kind {
23671 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
23672 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
23673 return Err(ExcelError::new(ExcelErrorKind::Name)
23674 .with_message(format!("Undefined name: {name}")));
23675 };
23676 let version = source
23677 .version
23678 .or_else(|| self.resolver.source_scalar_version(name));
23679 let v = self.resolve_source_scalar_cached(name, version)?;
23680 Ok(RangeView::from_owned_rows(
23681 vec![vec![v]],
23682 self.config.date_system,
23683 ))
23684 }
23685 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
23686 let Some(source) = self.graph.resolve_source_table_entry(name) else {
23687 return Err(ExcelError::new(ExcelErrorKind::Name)
23688 .with_message(format!("Undefined table: {name}")));
23689 };
23690 let version = source
23691 .version
23692 .or_else(|| self.resolver.source_table_version(name));
23693 let table = self.resolve_source_table_cached(name, version)?;
23694 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
23695 self.source_table_to_range_view(table.as_ref(), &spec)
23696 }
23697 }
23698 }
23699 ReferenceType::Range { .. } => {
23700 let shared = self.resolve_shared_ref(reference, current_sheet)?;
23701 let formualizer_common::SheetRef::Range(range) = shared else {
23702 return Err(ExcelError::new(ExcelErrorKind::Ref));
23703 };
23704 let sheet_id = match range.sheet {
23707 formualizer_common::SheetLocator::Id(id) => id,
23708 formualizer_common::SheetLocator::Current
23709 | formualizer_common::SheetLocator::Name(_) => {
23710 return Err(ExcelError::new(ExcelErrorKind::Ref));
23711 }
23712 };
23713 let sheet_name = self.graph.sheet_name(sheet_id);
23714
23715 let bounded_range = if range.start_row.is_some()
23716 && range.start_col.is_some()
23717 && range.end_row.is_some()
23718 && range.end_col.is_some()
23719 {
23720 Some(RangeRef::try_from_shared(range.as_ref())?)
23721 } else {
23722 None
23723 };
23724
23725 let sr = bounded_range
23726 .as_ref()
23727 .map(|r| r.start.coord.row() + 1)
23728 .or_else(|| range.start_row.map(|b| b.index + 1));
23729 let sc = bounded_range
23730 .as_ref()
23731 .map(|r| r.start.coord.col() + 1)
23732 .or_else(|| range.start_col.map(|b| b.index + 1));
23733 let er = bounded_range
23734 .as_ref()
23735 .map(|r| r.end.coord.row() + 1)
23736 .or_else(|| range.end_row.map(|b| b.index + 1));
23737 let ec = bounded_range
23738 .as_ref()
23739 .map(|r| r.end.coord.col() + 1)
23740 .or_else(|| range.end_col.map(|b| b.index + 1));
23741
23742 let extent = resolve_used_extent_with_fallback(
23743 OpenRangeBounds {
23744 start_row: sr,
23745 start_column: sc,
23746 end_row: er,
23747 end_column: ec,
23748 },
23749 ExtentPolicy::EvaluationCompat {
23750 fallback_row: None,
23751 fallback_column: None,
23752 },
23753 || {
23754 self.sheet_bounds(sheet_name)
23755 .map(|_| self.config.max_open_ended_rows)
23756 },
23757 || {
23758 self.sheet_bounds(sheet_name)
23759 .map(|_| self.config.max_open_ended_cols)
23760 },
23761 |first, last| self.used_rows_for_columns(sheet_name, first, last),
23762 |first, last| self.used_cols_for_rows(sheet_name, first, last),
23763 );
23764 let (sr, sc, er, ec) = extent
23765 .map(|extent| {
23766 (
23767 extent.start_row,
23768 extent.start_column,
23769 extent.end_row,
23770 extent.end_column,
23771 )
23772 })
23773 .unwrap_or((1, 1, 0, 0));
23774
23775 if self.force_materialize_range_views {
23776 if er < sr || ec < sc {
23777 return Ok(RangeView::from_owned_rows(
23778 Vec::new(),
23779 self.config.date_system,
23780 ));
23781 }
23782 let h = (er - sr + 1) as u64;
23783 let w = (ec - sc + 1) as u64;
23784 let cell_count = h.saturating_mul(w);
23785 if cell_count <= self.config.spill.max_spill_cells as u64 {
23786 let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
23787 for r in sr..=er {
23788 let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
23789 for c in sc..=ec {
23790 rowv.push(
23791 self.get_cell_value(sheet_name, r, c)
23792 .unwrap_or(LiteralValue::Empty),
23793 );
23794 }
23795 rows.push(rowv);
23796 }
23797 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
23798 }
23799 }
23800
23801 let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
23802 return Ok(RangeView::from_owned_rows(
23803 Vec::new(),
23804 self.config.date_system,
23805 ));
23806 };
23807
23808 let rv = if er < sr || ec < sc {
23809 asheet.range_view(1, 1, 0, 0)
23810 } else {
23811 let sr0 = sr.saturating_sub(1) as usize;
23812 let sc0 = sc.saturating_sub(1) as usize;
23813 let er0 = er.saturating_sub(1) as usize;
23814 let ec0 = ec.saturating_sub(1) as usize;
23815 asheet.range_view(sr0, sc0, er0, ec0)
23816 };
23817
23818 Ok(rv)
23819 }
23820 ReferenceType::Cell { .. } => {
23821 let shared = self.resolve_shared_ref(reference, current_sheet)?;
23822 let formualizer_common::SheetRef::Cell(cell) = shared else {
23823 return Err(ExcelError::new(ExcelErrorKind::Ref));
23824 };
23825 let addr = CellRef::try_from_shared(cell)?;
23826 let sheet_id = addr.sheet_id;
23827 let sheet_name = self.graph.sheet_name(sheet_id);
23828 let row = addr.coord.row() + 1;
23829 let col = addr.coord.col() + 1;
23830
23831 if self.force_materialize_range_views {
23832 let v = self
23833 .get_cell_value(sheet_name, row, col)
23834 .unwrap_or(LiteralValue::Empty);
23835 return Ok(RangeView::from_owned_rows(
23836 vec![vec![v]],
23837 self.config.date_system,
23838 ));
23839 }
23840
23841 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
23842 let r0 = row.saturating_sub(1) as usize;
23843 let c0 = col.saturating_sub(1) as usize;
23844 let rv = asheet.range_view(r0, c0, r0, c0);
23845 Ok(rv)
23846 } else {
23847 let v = self
23848 .get_cell_value(sheet_name, row, col)
23849 .unwrap_or(LiteralValue::Empty);
23850 Ok(RangeView::from_owned_rows(
23851 vec![vec![v]],
23852 self.config.date_system,
23853 ))
23854 }
23855 }
23856 ReferenceType::NamedRange(name) => {
23857 if let Some(current_id) = self.graph.sheet_id(current_sheet)
23858 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
23859 {
23860 match &named.definition {
23861 NamedDefinition::Cell(cell_ref) => {
23862 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23863 if self.force_materialize_range_views {
23864 let v = self
23865 .get_cell_value(
23866 sheet_name,
23867 cell_ref.coord.row() + 1,
23868 cell_ref.coord.col() + 1,
23869 )
23870 .unwrap_or(LiteralValue::Empty);
23871 return Ok(RangeView::from_owned_rows(
23872 vec![vec![v]],
23873 self.config.date_system,
23874 ));
23875 } else {
23876 let asheet = self
23877 .sheet_store()
23878 .sheet(sheet_name)
23879 .expect("Arrow sheet missing for named cell");
23880 let r0 = cell_ref.coord.row() as usize;
23881 let c0 = cell_ref.coord.col() as usize;
23882 let rv = asheet.range_view(r0, c0, r0, c0);
23883 return Ok(rv);
23884 }
23885 }
23886 NamedDefinition::Range(range_ref) => {
23887 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
23888 let sr = range_ref.start.coord.row() + 1;
23889 let sc = range_ref.start.coord.col() + 1;
23890 let er = range_ref.end.coord.row() + 1;
23891 let ec = range_ref.end.coord.col() + 1;
23892 if self.force_materialize_range_views {
23893 let h = (er.saturating_sub(sr) + 1) as u64;
23894 let w = (ec.saturating_sub(sc) + 1) as u64;
23895 let cell_count = h.saturating_mul(w);
23896 if cell_count <= self.config.spill.max_spill_cells as u64 {
23897 let mut rows: Vec<Vec<LiteralValue>> =
23898 Vec::with_capacity(h as usize);
23899 for r in sr..=er {
23900 let mut rowv: Vec<LiteralValue> =
23901 Vec::with_capacity(w as usize);
23902 for c in sc..=ec {
23903 rowv.push(
23904 self.get_cell_value(sheet_name, r, c)
23905 .unwrap_or(LiteralValue::Empty),
23906 );
23907 }
23908 rows.push(rowv);
23909 }
23910 return Ok(RangeView::from_owned_rows(
23911 rows,
23912 self.config.date_system,
23913 ));
23914 }
23915 }
23916 let asheet = self
23917 .sheet_store()
23918 .sheet(sheet_name)
23919 .expect("Arrow sheet missing for named range");
23920 let sr0 = range_ref.start.coord.row() as usize;
23921 let sc0 = range_ref.start.coord.col() as usize;
23922 let er0 = range_ref.end.coord.row() as usize;
23923 let ec0 = range_ref.end.coord.col() as usize;
23924 let rv = asheet.range_view(sr0, sc0, er0, ec0);
23925 return Ok(rv);
23926 }
23927 NamedDefinition::Literal(v) => {
23928 return Ok(RangeView::from_owned_rows(
23929 vec![vec![v.clone()]],
23930 self.config.date_system,
23931 ));
23932 }
23933 NamedDefinition::Formula { .. } => {
23934 if let Some(value) = self.graph.get_value(named.vertex) {
23935 return Ok(RangeView::from_owned_rows(
23936 vec![vec![value]],
23937 self.config.date_system,
23938 ));
23939 }
23940 }
23941 }
23942 }
23943
23944 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
23945 let version = source
23946 .version
23947 .or_else(|| self.resolver.source_scalar_version(name));
23948 let v = self.resolve_source_scalar_cached(name, version)?;
23949 return Ok(RangeView::from_owned_rows(
23950 vec![vec![v]],
23951 self.config.date_system,
23952 ));
23953 }
23954
23955 let data = self.resolver.resolve_named_range_reference(name)?;
23956 Ok(RangeView::from_owned_rows(data, self.config.date_system))
23957 }
23958 ReferenceType::Table(tref) => {
23959 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
23960 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
23961 let asheet = self
23962 .sheet_store()
23963 .sheet(sheet_name)
23964 .expect("Arrow sheet missing for table reference");
23965
23966 let sr0 = table.range.start.coord.row() as usize;
23967 let sc0 = table.range.start.coord.col() as usize;
23968 let er0 = table.range.end.coord.row() as usize;
23969 let ec0 = table.range.end.coord.col() as usize;
23970
23971 let has_totals = table.totals_row;
23972 let has_headers = table.header_row;
23973 let data_sr = if has_headers {
23974 sr0.saturating_add(1)
23975 } else {
23976 sr0
23977 };
23978 let data_er = if has_totals {
23979 er0.saturating_sub(1)
23980 } else {
23981 er0
23982 };
23983
23984 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
23985 if sr > er || sc > ec {
23986 asheet.range_view(1, 1, 0, 0)
23987 } else {
23988 asheet.range_view(sr, sc, er, ec)
23989 }
23990 };
23991
23992 let av = match &tref.specifier {
23993 None => {
23994 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23995 "Table reference without specifier is unsupported".to_string(),
23996 ));
23997 }
23998 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
23999 let Some(idx) = table.col_index(col) else {
24000 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24001 "Column refers to unknown table column".to_string(),
24002 ));
24003 };
24004 let c0 = sc0 + idx;
24005 select(data_sr, c0, data_er, c0)
24006 }
24007 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
24008 start,
24009 end,
24010 )) => {
24011 let Some(si) = table.col_index(start) else {
24012 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24013 "Column range refers to unknown column(s)".to_string(),
24014 ));
24015 };
24016 let Some(ei) = table.col_index(end) else {
24017 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24018 "Column range refers to unknown column(s)".to_string(),
24019 ));
24020 };
24021 let (mut a, mut b) = (si, ei);
24022 if a > b {
24023 std::mem::swap(&mut a, &mut b);
24024 }
24025 let c_start = sc0 + a;
24026 let c_end = sc0 + b;
24027 select(data_sr, c_start, data_er, c_end)
24028 }
24029 Some(formualizer_parse::parser::TableSpecifier::All)
24030 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24031 formualizer_parse::parser::SpecialItem::All,
24032 )) => select(sr0, sc0, er0, ec0),
24033 Some(formualizer_parse::parser::TableSpecifier::Data)
24034 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24035 formualizer_parse::parser::SpecialItem::Data,
24036 )) => select(data_sr, sc0, data_er, ec0),
24037 Some(formualizer_parse::parser::TableSpecifier::Headers)
24038 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24039 formualizer_parse::parser::SpecialItem::Headers,
24040 )) => {
24041 if !has_headers {
24042 asheet.range_view(1, 1, 0, 0)
24043 } else {
24044 select(sr0, sc0, sr0, ec0)
24045 }
24046 }
24047 Some(formualizer_parse::parser::TableSpecifier::Totals)
24048 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24049 formualizer_parse::parser::SpecialItem::Totals,
24050 )) => {
24051 if !has_totals {
24052 asheet.range_view(1, 1, 0, 0)
24053 } else {
24054 select(er0, sc0, er0, ec0)
24055 }
24056 }
24057 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24058 formualizer_parse::parser::SpecialItem::ThisRow,
24059 )) => {
24060 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24061 "@ (This Row) requires table-aware context; not yet supported"
24062 .to_string(),
24063 ));
24064 }
24065 Some(formualizer_parse::parser::TableSpecifier::Row(_))
24066 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
24067 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24068 "Complex structured references not yet supported".to_string(),
24069 ));
24070 }
24071 };
24072
24073 return Ok(av);
24074 }
24075
24076 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
24077 let version = source
24078 .version
24079 .or_else(|| self.resolver.source_table_version(&tref.name));
24080 let table = self.resolve_source_table_cached(&tref.name, version)?;
24081 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
24082 }
24083
24084 let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
24086 let owned = boxed.materialise().into_owned();
24087 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
24088 }
24089 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
24090 Err(ExcelError::new(ExcelErrorKind::NImpl)
24091 .with_message("3D references are not yet supported".to_string()))
24092 }
24093 }
24094 }
24095
24096 fn resolve_cell_reference_value(
24097 &self,
24098 sheet: Option<&str>,
24099 row: u32,
24100 col: u32,
24101 current_sheet: &str,
24102 ) -> Result<LiteralValue, ExcelError> {
24103 let sheet_name = sheet.unwrap_or(current_sheet);
24104 if self.graph.sheet_id(sheet_name).is_none() {
24105 return Err(ExcelError::new(ExcelErrorKind::Ref));
24106 }
24107 Ok(self
24108 .get_cell_value(sheet_name, row, col)
24109 .unwrap_or(LiteralValue::Empty))
24110 }
24111
24112 fn build_criteria_mask(
24113 &self,
24114 view: &RangeView<'_>,
24115 col_in_view: usize,
24116 pred: &crate::args::CriteriaPredicate,
24117 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24118 if view.dims().1 == 0 {
24119 return None;
24120 }
24121 let sheet_rows = view.sheet().nrows as usize;
24124 if sheet_rows == 0 || view.start_row() >= sheet_rows {
24125 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
24126 }
24127 compute_criteria_mask(view, col_in_view, pred)
24128 }
24129
24130 fn build_row_visibility_mask(
24131 &self,
24132 view: &RangeView<'_>,
24133 mode: VisibilityMaskMode,
24134 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24135 self.build_row_visibility_mask_for_view(view, mode)
24136 }
24137}
24138
24139impl<R> Engine<R>
24140where
24141 R: EvaluationContext,
24142{
24143 fn clear_spill_projection_and_mirror(
24144 &mut self,
24145 anchor_vertex: VertexId,
24146 delta: Option<&mut DeltaCollector>,
24147 ) {
24148 let spill_cells = self
24149 .graph
24150 .spill_cells_for_anchor(anchor_vertex)
24151 .map(|cells| cells.to_vec())
24152 .unwrap_or_default();
24153 if spill_cells.is_empty() {
24154 return;
24155 }
24156
24157 if let Some(delta) = delta
24158 && delta.mode != DeltaMode::Off
24159 {
24160 let empty = LiteralValue::Empty;
24161 for cell in spill_cells.iter() {
24162 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24163 let old = self
24164 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24165 .unwrap_or(LiteralValue::Empty);
24166 if old != empty {
24167 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24168 }
24169 }
24170 }
24171
24172 self.graph.clear_spill_region(anchor_vertex);
24173 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24174 self.record_formula_plane_structural_change(scope);
24175 }
24176
24177 if self.config.arrow_storage_enabled
24178 && self.config.delta_overlay_enabled
24179 && self.config.write_formula_overlay_enabled
24180 {
24181 let empty = LiteralValue::Empty;
24182 for cell in spill_cells.iter() {
24183 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24184 self.mirror_value_to_computed_overlay(
24185 &sheet_name,
24186 cell.coord.row() + 1,
24187 cell.coord.col() + 1,
24188 &empty,
24189 );
24190 }
24191 }
24192 }
24193
24194 fn apply_cycle_outcome(
24208 &mut self,
24209 cycle: &[VertexId],
24210 mut delta: Option<&mut DeltaCollector>,
24211 dirty_filter: Option<&FxHashSet<VertexId>>,
24212 ) -> usize {
24213 let circ_error = LiteralValue::Error(
24214 ExcelError::new(ExcelErrorKind::Circ)
24215 .with_message("Circular dependency detected".to_string()),
24216 );
24217 let mut stamped = 0usize;
24218 for &vertex_id in cycle {
24219 if let Some(filter) = dirty_filter
24220 && !filter.contains(&vertex_id)
24221 {
24222 continue;
24223 }
24224 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
24225 stamped += 1;
24226 }
24227 stamped
24228 }
24229
24230 fn stamp_cycle_error(
24245 &mut self,
24246 vertex_id: VertexId,
24247 circ_error: &LiteralValue,
24248 mut delta: Option<&mut DeltaCollector>,
24249 ) {
24250 if self.graph.spill_registry_has_anchor(vertex_id) {
24252 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24253 }
24254 self.spill_mgr.release_owner(vertex_id);
24257
24258 if let Some(d) = delta
24260 && d.mode != DeltaMode::Off
24261 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24262 {
24263 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24264 let old = self
24265 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24266 .unwrap_or(LiteralValue::Empty);
24267 if old != *circ_error {
24268 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24269 }
24270 }
24271
24272 self.graph
24273 .update_vertex_value(vertex_id, circ_error.clone());
24274 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
24275 }
24276
24277 fn handle_cycle_unit(
24290 &mut self,
24291 cycle: &[VertexId],
24292 mut delta: Option<&mut DeltaCollector>,
24293 dirty_filter: Option<&FxHashSet<VertexId>>,
24294 cancel_flag: Option<&AtomicBool>,
24295 ) -> Result<usize, ExcelError> {
24296 self.resource_checkpoint(cycle.len() as u64)?;
24297 match self.config.cycle.detection {
24298 CycleDetection::Static => {
24299 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
24300 }
24301 CycleDetection::Runtime => {
24302 if let Some(filter) = dirty_filter
24303 && !cycle.iter().any(|v| filter.contains(v))
24304 {
24305 return Ok(0);
24306 }
24307 self.evaluate_scc_unit(cycle, delta, cancel_flag)
24311 }
24312 }
24313 }
24314
24315 pub(crate) fn evaluate_scc_unit(
24335 &mut self,
24336 cycle: &[VertexId],
24337 mut delta: Option<&mut DeltaCollector>,
24338 cancel_flag: Option<&AtomicBool>,
24339 ) -> Result<usize, ExcelError> {
24340 struct SccMember {
24341 vertex: VertexId,
24342 cell: Option<CellRef>,
24343 }
24344
24345 let task_start = crate::instant::FzInstant::now();
24346
24347 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
24352 let mut name_members: Vec<(VertexId, String)> = Vec::new();
24353 let mut other_members: Vec<VertexId> = Vec::new();
24354 for &v in cycle {
24355 match self.graph.get_vertex_kind(v) {
24356 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24357 match self.graph.get_cell_ref(v) {
24358 Some(cell) => cell_members.push((v, cell)),
24359 None => other_members.push(v),
24360 }
24361 }
24362 VertexKind::NamedScalar | VertexKind::NamedArray => {
24363 match self.graph.name_key_for_vertex(v) {
24364 Some(key) => name_members.push((v, key)),
24365 None => other_members.push(v),
24366 }
24367 }
24368 _ => other_members.push(v),
24369 }
24370 }
24371 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
24372 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
24373 other_members.sort_unstable();
24374
24375 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
24376 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
24377 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
24378 for (v, c) in &cell_members {
24379 members.push(SccMember {
24380 vertex: *v,
24381 cell: Some(*c),
24382 });
24383 }
24384 for (v, _) in &name_members {
24385 members.push(SccMember {
24386 vertex: *v,
24387 cell: None,
24388 });
24389 }
24390 for v in &other_members {
24391 members.push(SccMember {
24392 vertex: *v,
24393 cell: None,
24394 });
24395 }
24396 let n = members.len();
24397 let recordable = cell_refs.len() + name_keys.len();
24400
24401 let circ_error = LiteralValue::Error(
24402 ExcelError::new(ExcelErrorKind::Circ)
24403 .with_message("Circular dependency detected".to_string()),
24404 );
24405
24406 if !self.iterative_state_values.is_empty() {
24417 let restore: Vec<(VertexId, LiteralValue)> = members
24418 .iter()
24419 .filter_map(|m| {
24420 let cell = m.cell?;
24421 let persisted = self.iterative_state_values.get(&m.vertex)?;
24422 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24423 let overlay = self
24424 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24425 .unwrap_or(LiteralValue::Empty);
24426 if matches!(overlay, LiteralValue::Empty) {
24427 Some((m.vertex, persisted.clone()))
24428 } else {
24429 None
24430 }
24431 })
24432 .collect();
24433 for (vertex, value) in restore {
24434 self.mirror_vertex_value_to_overlay(vertex, &value);
24435 }
24436 }
24437
24438 let snapshot: Vec<LiteralValue> = members
24441 .iter()
24442 .map(|m| match m.cell {
24443 Some(cell) => {
24444 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24445 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24446 .unwrap_or(LiteralValue::Empty)
24447 }
24448 None => self
24449 .graph
24450 .get_value(m.vertex)
24451 .unwrap_or(LiteralValue::Empty),
24452 })
24453 .collect();
24454
24455 let mut excluded = vec![false; n];
24460 let mut last_value = snapshot.clone();
24461 let mut stamped = 0usize;
24462 for (i, m) in members.iter().enumerate() {
24463 match self.graph.get_vertex_kind(m.vertex) {
24464 VertexKind::FormulaArray => {
24465 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
24469 excluded[i] = true;
24470 last_value[i] = circ_error.clone();
24471 stamped += 1;
24472 }
24473 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
24474 _ => excluded[i] = true,
24475 }
24476 }
24477
24478 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
24479
24480 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
24482 let mut pos: Vec<i64> = vec![-1; n];
24484 let mut changed = vec![false; n];
24486
24487 macro_rules! run_member {
24490 ($i:expr) => {{
24491 let i: usize = $i;
24492 let m = &members[i];
24493 if i < recordable {
24494 collector.set_current(i as u32);
24495 }
24496 let value = {
24497 let ctx = RecordingContext::new(&*self, &collector);
24498 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
24499 Ok(v) => v,
24500 Err(e) => LiteralValue::Error(e),
24501 }
24502 };
24503 let is_cell_formula = m.cell.is_some();
24504 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
24505 self.stamp_cycle_error(m.vertex, &circ_error, None);
24510 excluded[i] = true;
24511 stamped += 1;
24512 changed[i] = last_value[i] != circ_error;
24513 last_value[i] = circ_error.clone();
24514 } else {
24515 self.graph.update_vertex_value(m.vertex, value.clone());
24516 self.mirror_vertex_value_to_overlay(m.vertex, &value);
24517 #[cfg(debug_assertions)]
24521 if let Some(cell) = m.cell {
24522 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24523 debug_assert!(
24524 self.read_delta_overlay_cell(
24525 &sheet_name,
24526 cell.coord.row() + 1,
24527 cell.coord.col() + 1
24528 )
24529 .is_none(),
24530 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
24531 cell.coord.row() + 1,
24532 cell.coord.col() + 1
24533 );
24534 }
24535 changed[i] = last_value[i] != value;
24536 last_value[i] = value;
24537 }
24538 }};
24539 }
24540
24541 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
24542 if let Some(flag) = flag
24543 && flag.load(Ordering::Relaxed)
24544 {
24545 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24546 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
24547 }
24548 Ok(())
24549 };
24550
24551 check_cancel(cancel_flag)?;
24553 let mut passes = 1usize;
24554 {
24555 let mut p = 0i64;
24556 for i in 0..n {
24557 if excluded[i] {
24558 continue;
24559 }
24560 run_member!(i);
24561 pos[i] = p;
24562 p += 1;
24563 }
24564 }
24565
24566 let policy = self.config.cycle.policy;
24582 let cap = n + 2;
24583 let mut witnessed_cycles = 0usize;
24584 let mut capped = false;
24585 let mut iterating = false;
24587 let mut converged = false;
24588 let mut prev_pass: Option<Vec<LiteralValue>> = None;
24592 let mut iter_max_delta = 0f64;
24595 let mut iter_nan_converged = 0usize;
24596 let mut settle_passes = 0usize;
24600 loop {
24601 let drained = collector.take_edges();
24604 for i in 0..n {
24605 if pos[i] >= 0 {
24606 out_edges[i].clear();
24607 }
24608 }
24609 for (from, to) in drained {
24610 debug_assert!(
24611 pos[from as usize] >= 0,
24612 "edge from a member that did not run"
24613 );
24614 out_edges[from as usize].push(to);
24615 }
24616 let mut edges: Vec<(u32, u32)> = Vec::new();
24617 for (i, outs) in out_edges.iter().enumerate() {
24618 if excluded[i] {
24619 continue;
24620 }
24621 for &t in outs {
24622 edges.push((i as u32, t));
24623 }
24624 }
24625 edges.sort_unstable();
24626 edges.dedup();
24627
24628 let analysis = analyze_live_graph(n, &edges);
24629
24630 if analysis.cycle_count > 0 {
24631 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
24635 match policy {
24636 CyclePolicy::Error => {
24637 for i in 0..n {
24643 if analysis.in_cycle[i] && !excluded[i] {
24644 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
24645 excluded[i] = true;
24646 last_value[i] = circ_error.clone();
24647 stamped += 1;
24648 }
24649 }
24650 check_cancel(cancel_flag)?;
24651 let order: Vec<usize> = analysis
24652 .topo
24653 .iter()
24654 .map(|&i| i as usize)
24655 .filter(|&i| !excluded[i])
24656 .collect();
24657 if !order.is_empty() {
24658 passes += 1;
24659 for i in order {
24660 run_member!(i);
24661 }
24662 }
24663 break;
24664 }
24665 CyclePolicy::Iterate {
24666 max_iterations,
24667 max_change,
24668 } => {
24669 iterating = true;
24671
24672 if let Some(prev) = &prev_pass {
24678 let mut round_max_delta = 0f64;
24679 let mut round_nan = 0usize;
24680 let mut all_converged = true;
24681 for i in 0..n {
24682 if excluded[i] {
24683 continue;
24687 }
24688 let out = crate::engine::convergence::values_converged(
24689 &prev[i],
24690 &last_value[i],
24691 max_change,
24692 self.config.date_system,
24693 );
24694 if out.nan_converged {
24695 round_nan += 1;
24696 }
24697 if let Some(d) = out.abs_delta {
24698 round_max_delta = round_max_delta.max(d);
24699 }
24700 if !out.converged {
24701 all_converged = false;
24702 }
24703 }
24704 iter_max_delta = round_max_delta;
24707 iter_nan_converged = round_nan;
24708 if all_converged {
24709 converged = true;
24710 break;
24711 }
24712 }
24713
24714 if passes >= max_iterations as usize {
24726 capped = true;
24727 break;
24728 }
24729
24730 check_cancel(cancel_flag)?;
24731 prev_pass = Some(last_value.clone());
24739 for x in pos.iter_mut() {
24740 *x = -1;
24741 }
24742 changed.fill(false);
24743 passes += 1;
24744 let mut p = 0i64;
24745 for i in 0..n {
24746 if excluded[i] {
24747 continue;
24748 }
24749 run_member!(i);
24750 pos[i] = p;
24751 p += 1;
24752 }
24753 continue;
24754 }
24755 }
24756 }
24757
24758 let mut stale: Vec<usize> = Vec::new();
24761 for i in 0..n {
24762 if excluded[i] {
24763 continue;
24764 }
24765 let is_stale = out_edges[i].iter().any(|&t| {
24766 let t = t as usize;
24767 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
24768 });
24769 if is_stale {
24770 stale.push(i);
24771 }
24772 }
24773 if stale.is_empty() {
24774 break; }
24776 if 1 + settle_passes >= cap {
24777 capped = true;
24779 for (i, m) in members.iter().enumerate() {
24780 if !excluded[i] {
24781 self.stamp_cycle_error(m.vertex, &circ_error, None);
24782 excluded[i] = true;
24783 last_value[i] = circ_error.clone();
24784 stamped += 1;
24785 }
24786 }
24787 break;
24788 }
24789
24790 check_cancel(cancel_flag)?;
24791 prev_pass = None;
24798 let topo_pos = analysis.topo_positions();
24799 stale.sort_unstable_by_key(|&i| topo_pos[i]);
24800 for x in pos.iter_mut() {
24801 *x = -1;
24802 }
24803 changed.fill(false);
24804 passes += 1;
24805 settle_passes += 1;
24806 for (p, i) in stale.into_iter().enumerate() {
24807 run_member!(i);
24808 pos[i] = p as i64;
24809 }
24810 }
24811
24812 if iterating && !converged && !capped {
24817 converged = true;
24818 }
24819
24820 collector.clear_current();
24823 if let Some(d) = delta
24824 && d.mode != DeltaMode::Off
24825 {
24826 for (i, m) in members.iter().enumerate() {
24827 if let Some(cell) = m.cell
24828 && last_value[i] != snapshot[i]
24829 {
24830 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24831 }
24832 }
24833 }
24834
24835 if iterating {
24842 self.pending_iterative_redirty
24843 .extend(members.iter().map(|m| m.vertex));
24844 }
24845
24846 {
24847 let t = &mut self.last_cycle_telemetry;
24848 t.static_sccs += 1;
24849 if witnessed_cycles == 0 && stamped == 0 && !capped {
24850 t.phantom_sccs += 1;
24851 }
24852 t.live_cycles_witnessed += witnessed_cycles;
24853 t.circ_cells_stamped += stamped;
24854 t.settle_passes_total += passes;
24855 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
24856 if iterating {
24857 t.iterated_sccs += 1;
24858 if converged {
24859 t.converged_sccs += 1;
24860 }
24861 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
24862 t.nan_converged += iter_nan_converged;
24863 }
24864 if capped {
24865 t.capped_sccs += 1;
24866 }
24867 t.elapsed_ms += task_start.elapsed().as_millis();
24868 }
24869
24870 Ok(stamped)
24871 }
24872
24873 fn evaluate_vertex_recorded(
24881 &self,
24882 vertex_id: VertexId,
24883 ctx: &RecordingContext<'_, R>,
24884 collector: &LiveEdgeCollector,
24885 ) -> Result<LiteralValue, ExcelError> {
24886 if !self.graph.vertex_exists(vertex_id) {
24887 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24888 .with_message(format!("Vertex not found: {vertex_id:?}")));
24889 }
24890
24891 let kind = self.graph.get_vertex_kind(vertex_id);
24892 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24893
24894 match kind {
24895 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24896 let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
24897 return Ok(LiteralValue::Number(0.0)); };
24899 let sheet_name = self.graph.sheet_name(sheet_id);
24900 let cell_ref = self
24901 .graph
24902 .get_cell_ref(vertex_id)
24903 .expect("cell ref for vertex");
24904 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24905 interpreter
24906 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24907 .map(|cv| {
24908 crate::engine::result_finalization::finalize_formula_result(
24909 cv.into_literal(),
24910 )
24911 })
24912 }
24913 VertexKind::NamedScalar | VertexKind::NamedArray => {
24914 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24915 ExcelError::new(ExcelErrorKind::Name)
24916 .with_message("Named range metadata missing".to_string())
24917 })?;
24918
24919 match &named_range.definition {
24920 NamedDefinition::Formula { ast, .. } => {
24921 let context_sheet = match named_range.scope {
24922 NameScope::Sheet(id) => id,
24923 NameScope::Workbook => sheet_id,
24924 };
24925 let sheet_name = self.graph.sheet_name(context_sheet);
24926 let cell_ref = self
24927 .graph
24928 .get_cell_ref(vertex_id)
24929 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24930 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24931 if kind == VertexKind::NamedScalar {
24932 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24933 } else {
24934 match interpreter.evaluate_ast(ast) {
24935 Ok(cv) => match cv.into_literal() {
24936 v @ LiteralValue::Array(_) => Ok(v),
24937 other => Ok(LiteralValue::Array(vec![vec![other]])),
24938 },
24939 Err(err) => Ok(LiteralValue::Error(err)),
24940 }
24941 }
24942 }
24943 NamedDefinition::Cell(cell_ref) => {
24944 collector.record_scalar(
24948 cell_ref.sheet_id,
24949 cell_ref.coord.row(),
24950 cell_ref.coord.col(),
24951 );
24952 self.evaluate_vertex_immutable(vertex_id)
24953 }
24954 NamedDefinition::Range(range_ref) => {
24955 if range_ref.start.sheet_id == range_ref.end.sheet_id {
24956 collector.record_rect(
24957 range_ref.start.sheet_id,
24958 range_ref.start.coord.row(),
24959 range_ref.start.coord.col(),
24960 range_ref.end.coord.row(),
24961 range_ref.end.coord.col(),
24962 );
24963 }
24964 self.evaluate_vertex_immutable(vertex_id)
24965 }
24966 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
24967 }
24968 }
24969 _ => self.evaluate_vertex_immutable(vertex_id),
24970 }
24971 }
24972
24973 fn commit_spill_and_mirror(
24975 &mut self,
24976 anchor_vertex: VertexId,
24977 targets: &[CellRef],
24978 rows: Vec<Vec<LiteralValue>>,
24979 delta: Option<&mut DeltaCollector>,
24980 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24981 ) -> Result<(), ExcelError> {
24982 let prev_spill_cells = self
24983 .graph
24984 .spill_cells_for_anchor(anchor_vertex)
24985 .map(|cells| cells.to_vec())
24986 .unwrap_or_default();
24987
24988 if let Some(delta) = delta
24989 && delta.mode != DeltaMode::Off
24990 {
24991 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
24992 targets.iter().copied().collect();
24993 let empty = LiteralValue::Empty;
24994
24995 for cell in prev_spill_cells.iter() {
24997 if target_set.contains(cell) {
24998 continue;
24999 }
25000 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25001 let old = self
25002 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25003 .unwrap_or(LiteralValue::Empty);
25004 if old != empty {
25005 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25006 }
25007 }
25008
25009 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
25011 let width = rows[0].len();
25012 for (idx, cell) in targets.iter().enumerate() {
25013 let r_off = idx / width;
25014 let c_off = idx % width;
25015 let new = rows
25016 .get(r_off)
25017 .and_then(|r| r.get(c_off))
25018 .cloned()
25019 .unwrap_or(LiteralValue::Empty);
25020 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25021 let old = self
25022 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25023 .unwrap_or(LiteralValue::Empty);
25024 if old != new {
25025 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25026 }
25027 }
25028 } else {
25029 for cell in targets.iter() {
25031 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25032 let old = self
25033 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25034 .unwrap_or(LiteralValue::Empty);
25035 if !matches!(old, LiteralValue::Empty) {
25036 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25037 }
25038 }
25039 }
25040 }
25041
25042 let arrow_sheets = &self.arrow_sheets;
25045 self.spill_mgr.commit_array_with_value_probe(
25046 &mut self.graph,
25047 anchor_vertex,
25048 targets,
25049 rows.clone(),
25050 overwritable_formulas,
25051 |g, cell| {
25052 let sheet_name = g.sheet_name(cell.sheet_id);
25053 let asheet = arrow_sheets.sheet(sheet_name)?;
25054 let r0 = cell.coord.row() as usize;
25055 let c0 = cell.coord.col() as usize;
25056 let v = asheet.get_cell_value(r0, c0);
25057 if matches!(v, LiteralValue::Empty) {
25058 None
25059 } else {
25060 Some(v)
25061 }
25062 },
25063 )?;
25064
25065 if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
25066 self.record_formula_plane_structural_change(scope);
25067 }
25068 if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
25069 self.record_formula_plane_structural_change(scope);
25070 }
25071
25072 if self.config.arrow_storage_enabled
25073 && self.config.delta_overlay_enabled
25074 && self.config.write_formula_overlay_enabled
25075 {
25076 if !prev_spill_cells.is_empty() {
25077 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
25078 targets.iter().copied().collect();
25079 let empty = LiteralValue::Empty;
25080 for cell in prev_spill_cells.iter() {
25081 if !target_set.contains(cell) {
25082 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25083 self.mirror_value_to_computed_overlay(
25084 &sheet_name,
25085 cell.coord.row() + 1,
25086 cell.coord.col() + 1,
25087 &empty,
25088 );
25089 }
25090 }
25091 }
25092
25093 for (idx, cell) in targets.iter().enumerate() {
25094 if rows.is_empty() || rows[0].is_empty() {
25095 break;
25096 }
25097 let width = rows[0].len();
25098 let r_off = idx / width;
25099 let c_off = idx % width;
25100 let v = rows[r_off][c_off].clone();
25101 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25102 self.mirror_value_to_computed_overlay(
25103 &sheet_name,
25104 cell.coord.row() + 1,
25105 cell.coord.col() + 1,
25106 &v,
25107 );
25108 }
25109 }
25110 Ok(())
25111 }
25112}
25113
25114use crate::engine::effects::Effect;
25119use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
25120
25121impl<R> Engine<R>
25122where
25123 R: EvaluationContext,
25124{
25125 pub(crate) fn plan_vertex_effects(
25132 &mut self,
25133 vertex_id: VertexId,
25134 computed_value: LiteralValue,
25135 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25136 ) -> Result<Vec<Effect>, ExcelError> {
25137 let kind = self.graph.get_vertex_kind(vertex_id);
25138 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
25139
25140 if !is_formula {
25144 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
25145 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
25146 && owner != vertex_id
25147 {
25148 return Ok(Vec::new());
25149 }
25150 return Ok(vec![Effect::WriteCell {
25152 vertex_id,
25153 value: computed_value,
25154 }]);
25155 }
25156
25157 match computed_value {
25158 LiteralValue::Array(rows) => {
25159 self.plan_array_effects(vertex_id, rows, overwritable_formulas)
25160 }
25161 other => self.plan_scalar_effects(vertex_id, other),
25162 }
25163 }
25164
25165 fn plan_scalar_effects(
25167 &self,
25168 vertex_id: VertexId,
25169 value: LiteralValue,
25170 ) -> Result<Vec<Effect>, ExcelError> {
25171 let has_spill = self
25172 .graph
25173 .spill_cells_for_anchor(vertex_id)
25174 .is_some_and(|c| !c.is_empty());
25175
25176 let mut effects = Vec::new();
25177 if has_spill {
25178 effects.push(Effect::SpillClear {
25179 anchor_vertex: vertex_id,
25180 });
25181 }
25182 effects.push(Effect::WriteCell { vertex_id, value });
25183 Ok(effects)
25184 }
25185
25186 fn plan_array_effects(
25188 &mut self,
25189 vertex_id: VertexId,
25190 rows: Vec<Vec<LiteralValue>>,
25191 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25192 ) -> Result<Vec<Effect>, ExcelError> {
25193 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
25195
25196 let anchor = self
25197 .graph
25198 .get_cell_ref(vertex_id)
25199 .expect("cell ref for vertex");
25200 let sheet_id = anchor.sheet_id;
25201 let h = rows.len() as u32;
25202 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
25203
25204 let spill_cells = (h as u64).saturating_mul(w as u64);
25206 if spill_cells > self.config.spill.max_spill_cells as u64 {
25207 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
25208 }
25209
25210 const PACKED_MAX_ROW: u32 = 1_048_575;
25212 const PACKED_MAX_COL: u32 = 16_383;
25213 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
25214 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
25215 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
25216 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
25217 }
25218
25219 let mut targets = Vec::new();
25220 for r in 0..h {
25221 for c in 0..w {
25222 targets.push(self.graph.make_cell_ref_internal(
25223 sheet_id,
25224 anchor.coord.row() + r,
25225 anchor.coord.col() + c,
25226 ));
25227 }
25228 }
25229
25230 match self.spill_mgr.reserve(
25232 vertex_id,
25233 anchor,
25234 SpillShape { rows: h, cols: w },
25235 SpillMeta {
25236 epoch: self.recalc_epoch,
25237 config: self.config.spill,
25238 },
25239 ) {
25240 Ok(()) => {
25241 if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
25243 vertex_id,
25244 &targets,
25245 overwritable_formulas,
25246 ) {
25247 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
25248 }
25249
25250 if !self.graph.value_cache_enabled() {
25254 let sheet_name = self.graph.sheet_name(sheet_id);
25255 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25256 for cell in targets.iter() {
25257 if *cell == anchor {
25259 continue;
25260 }
25261 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
25263 continue;
25264 }
25265 if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
25267 && vid != vertex_id
25268 {
25269 match self.graph.get_vertex_kind(vid) {
25270 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
25271 continue;
25272 }
25273 _ => {}
25274 }
25275 }
25276
25277 let v = asheet.get_cell_value(
25278 cell.coord.row() as usize,
25279 cell.coord.col() as usize,
25280 );
25281 if !matches!(v, LiteralValue::Empty) {
25282 return self.plan_spill_error_effects(
25283 vertex_id,
25284 "BlockedByValue",
25285 h,
25286 w,
25287 );
25288 }
25289 }
25290 }
25291 }
25292
25293 let top_left = rows
25294 .first()
25295 .and_then(|r| r.first())
25296 .cloned()
25297 .unwrap_or(LiteralValue::Empty);
25298
25299 let mut effects = Vec::new();
25300 let has_prev = self
25302 .graph
25303 .spill_cells_for_anchor(vertex_id)
25304 .is_some_and(|c| !c.is_empty());
25305 if has_prev {
25306 effects.push(Effect::SpillClear {
25307 anchor_vertex: vertex_id,
25308 });
25309 }
25310 effects.push(Effect::SpillCommit {
25311 anchor_vertex: vertex_id,
25312 anchor_cell: anchor,
25313 target_cells: targets,
25314 values: rows,
25315 });
25316 effects.push(Effect::WriteCell {
25317 vertex_id,
25318 value: top_left,
25319 });
25320 Ok(effects)
25321 }
25322 Err(e) => {
25323 let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
25324 self.plan_spill_error_effects(vertex_id, &msg, h, w)
25325 }
25326 }
25327 }
25328
25329 fn plan_spill_error_effects(
25331 &self,
25332 vertex_id: VertexId,
25333 message: &str,
25334 expected_rows: u32,
25335 expected_cols: u32,
25336 ) -> Result<Vec<Effect>, ExcelError> {
25337 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
25338 .with_message(message)
25339 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
25340 expected_rows,
25341 expected_cols,
25342 });
25343 let spill_val = LiteralValue::Error(spill_err);
25344
25345 let effects = vec![
25346 Effect::SpillClear {
25347 anchor_vertex: vertex_id,
25348 },
25349 Effect::WriteCell {
25350 vertex_id,
25351 value: spill_val,
25352 },
25353 ];
25354 Ok(effects)
25355 }
25356
25357 pub(crate) fn apply_effect(
25359 &mut self,
25360 effect: &Effect,
25361 delta: Option<&mut DeltaCollector>,
25362 log: Option<&mut ChangeLog>,
25363 ) -> Result<(), ExcelError> {
25364 self.apply_effect_with_computed_writes(effect, delta, log, None)
25365 }
25366
25367 fn apply_effect_with_computed_writes(
25368 &mut self,
25369 effect: &Effect,
25370 delta: Option<&mut DeltaCollector>,
25371 log: Option<&mut ChangeLog>,
25372 computed_writes: Option<&mut ComputedWriteBuffer>,
25373 ) -> Result<(), ExcelError> {
25374 match effect {
25375 Effect::WriteCell { vertex_id, value } => {
25376 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
25377 }
25378 Effect::SpillClear { anchor_vertex } => {
25379 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
25380 }
25381 Effect::SpillCommit {
25382 anchor_vertex,
25383 anchor_cell: _,
25384 target_cells,
25385 values,
25386 } => {
25387 self.apply_spill_commit(
25388 *anchor_vertex,
25389 target_cells,
25390 values.clone(),
25391 delta,
25392 log,
25393 computed_writes,
25394 )?;
25395 }
25396 }
25397 Ok(())
25398 }
25399
25400 fn apply_write_cell(
25402 &mut self,
25403 vertex_id: VertexId,
25404 value: &LiteralValue,
25405 delta: Option<&mut DeltaCollector>,
25406 mut computed_writes: Option<&mut ComputedWriteBuffer>,
25407 ) -> Result<(), ExcelError> {
25408 if let Some(d) = delta
25409 && d.mode != DeltaMode::Off
25410 {
25411 if let Some(buffer) = computed_writes.as_deref_mut() {
25412 self.flush_computed_write_buffer(buffer)?;
25413 }
25414 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
25415 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25416 let old = self
25417 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25418 .unwrap_or(LiteralValue::Empty);
25419 if old != *value {
25420 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25421 }
25422 }
25423 }
25424 self.graph.update_vertex_value(vertex_id, value.clone());
25425 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
25426 Ok(())
25427 }
25428
25429 fn apply_spill_clear(
25431 &mut self,
25432 anchor_vertex: VertexId,
25433 delta: Option<&mut DeltaCollector>,
25434 log: Option<&mut ChangeLog>,
25435 computed_writes: Option<&mut ComputedWriteBuffer>,
25436 ) -> Result<(), ExcelError> {
25437 if let Some(buffer) = computed_writes {
25438 self.flush_computed_write_buffer(buffer)?;
25439 }
25440
25441 let spill_cells = self
25442 .graph
25443 .spill_cells_for_anchor(anchor_vertex)
25444 .map(|cells| cells.to_vec())
25445 .unwrap_or_default();
25446 if spill_cells.is_empty() {
25447 return Ok(());
25448 }
25449
25450 let snapshot = if log.is_some() {
25452 self.snapshot_spill_for_anchor(anchor_vertex)
25453 } else {
25454 None
25455 };
25456
25457 if let Some(d) = delta
25459 && d.mode != DeltaMode::Off
25460 {
25461 let empty = LiteralValue::Empty;
25462 for cell in spill_cells.iter() {
25463 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25464 let old = self
25465 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25466 .unwrap_or(LiteralValue::Empty);
25467 if old != empty {
25468 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25469 }
25470 }
25471 }
25472
25473 self.graph.clear_spill_region(anchor_vertex);
25474 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
25475 self.record_formula_plane_structural_change(scope);
25476 }
25477
25478 if self.config.arrow_storage_enabled
25480 && self.config.delta_overlay_enabled
25481 && self.config.write_formula_overlay_enabled
25482 {
25483 let empty = LiteralValue::Empty;
25484 for cell in spill_cells.iter() {
25485 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25486 self.mirror_value_to_computed_overlay(
25487 &sheet_name,
25488 cell.coord.row() + 1,
25489 cell.coord.col() + 1,
25490 &empty,
25491 );
25492 }
25493 }
25494
25495 if let Some(log) = log
25497 && let Some(old) = snapshot
25498 {
25499 log.record(ChangeEvent::SpillCleared {
25500 anchor: anchor_vertex,
25501 old,
25502 });
25503 }
25504 Ok(())
25505 }
25506
25507 fn apply_spill_commit(
25509 &mut self,
25510 anchor_vertex: VertexId,
25511 target_cells: &[CellRef],
25512 values: Vec<Vec<LiteralValue>>,
25513 delta: Option<&mut DeltaCollector>,
25514 log: Option<&mut ChangeLog>,
25515 computed_writes: Option<&mut ComputedWriteBuffer>,
25516 ) -> Result<(), ExcelError> {
25517 if let Some(buffer) = computed_writes {
25518 self.flush_computed_write_buffer(buffer)?;
25519 }
25520
25521 let old_snapshot = if log.is_some() {
25523 self.snapshot_spill_for_anchor(anchor_vertex)
25524 } else {
25525 None
25526 };
25527
25528 self.commit_spill_and_mirror(
25530 anchor_vertex,
25531 target_cells,
25532 values.clone(),
25533 delta,
25534 None, )?;
25536
25537 if let Some(log) = log {
25539 log.record(ChangeEvent::SpillCommitted {
25540 anchor: anchor_vertex,
25541 old: old_snapshot,
25542 new: SpillSnapshot {
25543 target_cells: target_cells.to_vec(),
25544 values,
25545 },
25546 });
25547 }
25548 Ok(())
25549 }
25550
25551 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
25556 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
25557 if cells.is_empty() {
25558 return None;
25559 }
25560
25561 let max = self.config.spill.max_spill_cells as usize;
25562 let mut cells = cells;
25563 if cells.len() > max {
25564 cells.truncate(max);
25565 }
25566
25567 let first = *cells.first().expect("non-empty spill cells");
25568 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
25569 let row0 = first.coord.row();
25570 let col0 = first.coord.col();
25571
25572 let mut max_row = row0;
25573 let mut max_col = col0;
25574 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
25575 for cell in &cells {
25576 max_row = max_row.max(cell.coord.row());
25577 max_col = max_col.max(cell.coord.col());
25578 let v = self
25579 .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25580 .unwrap_or(LiteralValue::Empty);
25581 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
25582 }
25583
25584 let rows = (max_row - row0 + 1) as usize;
25585 let cols = (max_col - col0 + 1) as usize;
25586 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
25587 for r in 0..rows {
25588 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
25589 for c in 0..cols {
25590 row.push(
25591 by_coord
25592 .get(&(row0 + r as u32, col0 + c as u32))
25593 .cloned()
25594 .unwrap_or(LiteralValue::Empty),
25595 );
25596 }
25597 values.push(row);
25598 }
25599
25600 Some(SpillSnapshot {
25601 target_cells: cells,
25602 values,
25603 })
25604 }
25605
25606 fn flush_before_range_dependent_vertex(
25607 &mut self,
25608 vertex_id: VertexId,
25609 computed_writes: &mut ComputedWriteBuffer,
25610 ) -> Result<(), ExcelError> {
25611 if self.graph.get_range_dependencies(vertex_id).is_some() {
25612 self.flush_computed_write_buffer(computed_writes)?;
25613 }
25614 Ok(())
25615 }
25616
25617 fn plan_vertex_effects_with_computed_flush(
25618 &mut self,
25619 vertex_id: VertexId,
25620 computed_value: LiteralValue,
25621 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25622 computed_writes: &mut ComputedWriteBuffer,
25623 ) -> Result<Vec<Effect>, ExcelError> {
25624 if matches!(&computed_value, LiteralValue::Array(_)) {
25625 self.flush_computed_write_buffer(computed_writes)?;
25626 }
25627 self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
25628 }
25629
25630 fn evaluate_small_layer_direct_effects(
25633 &mut self,
25634 layer: &super::scheduler::Layer,
25635 mut delta: Option<&mut DeltaCollector>,
25636 mut log: Option<&mut ChangeLog>,
25637 cancel_flag: Option<&AtomicBool>,
25638 cancel_check_every: usize,
25639 cancel_message: &'static str,
25640 ) -> Result<usize, ExcelError> {
25641 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25642 if cancel_check_every > 0
25643 && i % cancel_check_every == 0
25644 && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
25645 {
25646 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25647 .with_message(cancel_message.to_string()));
25648 }
25649 let value = match self.evaluate_vertex_immutable(vertex_id) {
25650 Ok(v) => v,
25651 Err(e) => LiteralValue::Error(e),
25652 };
25653 let effects = self.plan_vertex_effects(vertex_id, value, None)?;
25654 for effect in &effects {
25655 self.apply_effect_with_computed_writes(
25656 effect,
25657 delta.as_deref_mut(),
25658 log.as_deref_mut(),
25659 None,
25660 )?;
25661 }
25662 }
25663 Ok(layer.vertices.len())
25664 }
25665
25666 fn evaluate_layer_sequential_effects(
25668 &mut self,
25669 layer: &super::scheduler::Layer,
25670 ) -> Result<usize, ExcelError> {
25671 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25672 return self.evaluate_small_layer_direct_effects(
25673 layer,
25674 None,
25675 None,
25676 None,
25677 0,
25678 "Evaluation cancelled within layer",
25679 );
25680 }
25681
25682 let mut computed_writes = ComputedWriteBuffer::default();
25683 for &vertex_id in &layer.vertices {
25684 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25685 let value = match self.evaluate_vertex_immutable(vertex_id) {
25686 Ok(v) => v,
25687 Err(e) => LiteralValue::Error(e),
25688 };
25689 let effects = match self.plan_vertex_effects_with_computed_flush(
25690 vertex_id,
25691 value,
25692 None,
25693 &mut computed_writes,
25694 ) {
25695 Ok(effects) => effects,
25696 Err(e) => {
25697 self.flush_computed_write_buffer(&mut computed_writes)?;
25698 return Err(e);
25699 }
25700 };
25701 for effect in &effects {
25702 if let Err(e) = self.apply_effect_with_computed_writes(
25703 effect,
25704 None,
25705 None,
25706 Some(&mut computed_writes),
25707 ) {
25708 self.flush_computed_write_buffer(&mut computed_writes)?;
25709 return Err(e);
25710 }
25711 }
25712 }
25713 self.flush_computed_write_buffer(&mut computed_writes)?;
25714 Ok(layer.vertices.len())
25715 }
25716
25717 fn evaluate_layer_sequential_with_delta_effects(
25719 &mut self,
25720 layer: &super::scheduler::Layer,
25721 delta: &mut DeltaCollector,
25722 ) -> Result<usize, ExcelError> {
25723 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25724 return self.evaluate_small_layer_direct_effects(
25725 layer,
25726 Some(delta),
25727 None,
25728 None,
25729 0,
25730 "Evaluation cancelled within layer",
25731 );
25732 }
25733
25734 let mut computed_writes = ComputedWriteBuffer::default();
25735 for &vertex_id in &layer.vertices {
25736 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25737 let value = match self.evaluate_vertex_immutable(vertex_id) {
25738 Ok(v) => v,
25739 Err(e) => LiteralValue::Error(e),
25740 };
25741 let effects = match self.plan_vertex_effects_with_computed_flush(
25742 vertex_id,
25743 value,
25744 None,
25745 &mut computed_writes,
25746 ) {
25747 Ok(effects) => effects,
25748 Err(e) => {
25749 self.flush_computed_write_buffer(&mut computed_writes)?;
25750 return Err(e);
25751 }
25752 };
25753 for effect in &effects {
25754 if let Err(e) = self.apply_effect_with_computed_writes(
25755 effect,
25756 Some(delta),
25757 None,
25758 Some(&mut computed_writes),
25759 ) {
25760 self.flush_computed_write_buffer(&mut computed_writes)?;
25761 return Err(e);
25762 }
25763 }
25764 }
25765 self.flush_computed_write_buffer(&mut computed_writes)?;
25766 Ok(layer.vertices.len())
25767 }
25768
25769 fn evaluate_layer_sequential_cancellable_effects(
25771 &mut self,
25772 layer: &super::scheduler::Layer,
25773 cancel_flag: &AtomicBool,
25774 ) -> Result<usize, ExcelError> {
25775 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25776 return self.evaluate_small_layer_direct_effects(
25777 layer,
25778 None,
25779 None,
25780 Some(cancel_flag),
25781 256,
25782 "Evaluation cancelled within layer",
25783 );
25784 }
25785
25786 let mut computed_writes = ComputedWriteBuffer::default();
25787 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25788 if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
25789 self.flush_computed_write_buffer(&mut computed_writes)?;
25790 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25791 .with_message("Evaluation cancelled within layer".to_string()));
25792 }
25793 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25794 let value = match self.evaluate_vertex_immutable(vertex_id) {
25795 Ok(v) => v,
25796 Err(e) => LiteralValue::Error(e),
25797 };
25798 let effects = match self.plan_vertex_effects_with_computed_flush(
25799 vertex_id,
25800 value,
25801 None,
25802 &mut computed_writes,
25803 ) {
25804 Ok(effects) => effects,
25805 Err(e) => {
25806 self.flush_computed_write_buffer(&mut computed_writes)?;
25807 return Err(e);
25808 }
25809 };
25810 for effect in &effects {
25811 if let Err(e) = self.apply_effect_with_computed_writes(
25812 effect,
25813 None,
25814 None,
25815 Some(&mut computed_writes),
25816 ) {
25817 self.flush_computed_write_buffer(&mut computed_writes)?;
25818 return Err(e);
25819 }
25820 }
25821 }
25822 self.flush_computed_write_buffer(&mut computed_writes)?;
25823 Ok(layer.vertices.len())
25824 }
25825
25826 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
25828 &mut self,
25829 layer: &super::scheduler::Layer,
25830 cancel_flag: &AtomicBool,
25831 ) -> Result<usize, ExcelError> {
25832 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25833 return self.evaluate_small_layer_direct_effects(
25834 layer,
25835 None,
25836 None,
25837 Some(cancel_flag),
25838 128,
25839 "Demand-driven evaluation cancelled within layer",
25840 );
25841 }
25842
25843 let mut computed_writes = ComputedWriteBuffer::default();
25844 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25845 if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
25846 self.flush_computed_write_buffer(&mut computed_writes)?;
25847 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25848 .with_message("Demand-driven evaluation cancelled within layer".to_string()));
25849 }
25850 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25851 let value = match self.evaluate_vertex_immutable(vertex_id) {
25852 Ok(v) => v,
25853 Err(e) => LiteralValue::Error(e),
25854 };
25855 let effects = match self.plan_vertex_effects_with_computed_flush(
25856 vertex_id,
25857 value,
25858 None,
25859 &mut computed_writes,
25860 ) {
25861 Ok(effects) => effects,
25862 Err(e) => {
25863 self.flush_computed_write_buffer(&mut computed_writes)?;
25864 return Err(e);
25865 }
25866 };
25867 for effect in &effects {
25868 if let Err(e) = self.apply_effect_with_computed_writes(
25869 effect,
25870 None,
25871 None,
25872 Some(&mut computed_writes),
25873 ) {
25874 self.flush_computed_write_buffer(&mut computed_writes)?;
25875 return Err(e);
25876 }
25877 }
25878 }
25879 self.flush_computed_write_buffer(&mut computed_writes)?;
25880 Ok(layer.vertices.len())
25881 }
25882
25883 fn evaluate_layer_parallel_effects(
25885 &mut self,
25886 layer: &super::scheduler::Layer,
25887 ) -> Result<usize, ExcelError> {
25888 use rayon::prelude::*;
25889
25890 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25891
25892 let mut phase1: Vec<VertexId> = Vec::new();
25893 let mut phase2: Vec<VertexId> = Vec::new();
25894 for &vid in &layer.vertices {
25895 if self.graph.get_range_dependencies(vid).is_some() {
25896 phase2.push(vid);
25897 } else {
25898 phase1.push(vid);
25899 }
25900 }
25901
25902 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25903 let mut applied = 0usize;
25904
25905 for group in [&phase1[..], &phase2[..]] {
25906 if group.is_empty() {
25907 continue;
25908 }
25909 let mut computed_writes = ComputedWriteBuffer::default();
25910
25911 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25912 thread_pool.install(|| {
25913 group
25914 .par_iter()
25915 .map(
25916 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25917 Ok(v) => Ok((vertex_id, v)),
25918 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25919 },
25920 )
25921 .collect()
25922 });
25923
25924 match results {
25925 Ok(vertex_results) => {
25926 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25929 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25930 for (vertex_id, result) in vertex_results {
25931 if matches!(result, LiteralValue::Array(_)) {
25932 arrays.push((vertex_id, result));
25933 } else {
25934 others.push((vertex_id, result));
25935 }
25936 }
25937 for (vertex_id, result) in arrays {
25938 let effects = match self.plan_vertex_effects_with_computed_flush(
25939 vertex_id,
25940 result,
25941 Some(&inflight),
25942 &mut computed_writes,
25943 ) {
25944 Ok(effects) => effects,
25945 Err(e) => {
25946 self.flush_computed_write_buffer(&mut computed_writes)?;
25947 return Err(e);
25948 }
25949 };
25950 for effect in &effects {
25951 if let Err(e) = self.apply_effect_with_computed_writes(
25952 effect,
25953 None,
25954 None,
25955 Some(&mut computed_writes),
25956 ) {
25957 self.flush_computed_write_buffer(&mut computed_writes)?;
25958 return Err(e);
25959 }
25960 }
25961 applied = applied.saturating_add(1);
25962 }
25963 self.flush_computed_write_buffer(&mut computed_writes)?;
25965 for (vertex_id, result) in others {
25966 let effects = match self.plan_vertex_effects_with_computed_flush(
25967 vertex_id,
25968 result,
25969 Some(&inflight),
25970 &mut computed_writes,
25971 ) {
25972 Ok(effects) => effects,
25973 Err(e) => {
25974 self.flush_computed_write_buffer(&mut computed_writes)?;
25975 return Err(e);
25976 }
25977 };
25978 for effect in &effects {
25979 if let Err(e) = self.apply_effect_with_computed_writes(
25980 effect,
25981 None,
25982 None,
25983 Some(&mut computed_writes),
25984 ) {
25985 self.flush_computed_write_buffer(&mut computed_writes)?;
25986 return Err(e);
25987 }
25988 }
25989 applied = applied.saturating_add(1);
25990 }
25991 self.flush_computed_write_buffer(&mut computed_writes)?;
25993 }
25994 Err(e) => {
25995 self.flush_computed_write_buffer(&mut computed_writes)?;
25996 return Err(e);
25997 }
25998 }
25999 }
26000
26001 Ok(applied)
26002 }
26003
26004 fn evaluate_layer_parallel_with_delta_effects(
26006 &mut self,
26007 layer: &super::scheduler::Layer,
26008 delta: &mut DeltaCollector,
26009 ) -> Result<usize, ExcelError> {
26010 use rayon::prelude::*;
26011
26012 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26013
26014 let mut phase1: Vec<VertexId> = Vec::new();
26015 let mut phase2: Vec<VertexId> = Vec::new();
26016 for &vid in &layer.vertices {
26017 if self.graph.get_range_dependencies(vid).is_some() {
26018 phase2.push(vid);
26019 } else {
26020 phase1.push(vid);
26021 }
26022 }
26023
26024 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26025 let mut applied = 0usize;
26026
26027 for group in [&phase1[..], &phase2[..]] {
26028 if group.is_empty() {
26029 continue;
26030 }
26031 let mut computed_writes = ComputedWriteBuffer::default();
26032 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26033 thread_pool.install(|| {
26034 group
26035 .par_iter()
26036 .map(
26037 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
26038 Ok(v) => Ok((vertex_id, v)),
26039 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26040 },
26041 )
26042 .collect()
26043 });
26044
26045 match results {
26046 Ok(vertex_results) => {
26047 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26048 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26049 for (vertex_id, result) in vertex_results {
26050 if matches!(result, LiteralValue::Array(_)) {
26051 arrays.push((vertex_id, result));
26052 } else {
26053 others.push((vertex_id, result));
26054 }
26055 }
26056 for (vertex_id, result) in arrays {
26057 let effects = match self.plan_vertex_effects_with_computed_flush(
26058 vertex_id,
26059 result,
26060 Some(&inflight),
26061 &mut computed_writes,
26062 ) {
26063 Ok(effects) => effects,
26064 Err(e) => {
26065 self.flush_computed_write_buffer(&mut computed_writes)?;
26066 return Err(e);
26067 }
26068 };
26069 for effect in &effects {
26070 if let Err(e) = self.apply_effect_with_computed_writes(
26071 effect,
26072 Some(delta),
26073 None,
26074 Some(&mut computed_writes),
26075 ) {
26076 self.flush_computed_write_buffer(&mut computed_writes)?;
26077 return Err(e);
26078 }
26079 }
26080 applied = applied.saturating_add(1);
26081 }
26082 self.flush_computed_write_buffer(&mut computed_writes)?;
26083 for (vertex_id, result) in others {
26084 let effects = match self.plan_vertex_effects_with_computed_flush(
26085 vertex_id,
26086 result,
26087 Some(&inflight),
26088 &mut computed_writes,
26089 ) {
26090 Ok(effects) => effects,
26091 Err(e) => {
26092 self.flush_computed_write_buffer(&mut computed_writes)?;
26093 return Err(e);
26094 }
26095 };
26096 for effect in &effects {
26097 if let Err(e) = self.apply_effect_with_computed_writes(
26098 effect,
26099 Some(delta),
26100 None,
26101 Some(&mut computed_writes),
26102 ) {
26103 self.flush_computed_write_buffer(&mut computed_writes)?;
26104 return Err(e);
26105 }
26106 }
26107 applied = applied.saturating_add(1);
26108 }
26109 self.flush_computed_write_buffer(&mut computed_writes)?;
26110 }
26111 Err(e) => {
26112 self.flush_computed_write_buffer(&mut computed_writes)?;
26113 return Err(e);
26114 }
26115 }
26116 }
26117
26118 Ok(applied)
26119 }
26120
26121 fn evaluate_layer_parallel_cancellable_effects(
26123 &mut self,
26124 layer: &super::scheduler::Layer,
26125 cancel_flag: &AtomicBool,
26126 ) -> Result<usize, ExcelError> {
26127 use rayon::prelude::*;
26128
26129 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26130
26131 if cancel_flag.load(Ordering::Relaxed) {
26132 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26133 .with_message("Parallel evaluation cancelled before starting".to_string()));
26134 }
26135
26136 let mut phase1: Vec<VertexId> = Vec::new();
26137 let mut phase2: Vec<VertexId> = Vec::new();
26138 for &vid in &layer.vertices {
26139 if self.graph.get_range_dependencies(vid).is_some() {
26140 phase2.push(vid);
26141 } else {
26142 phase1.push(vid);
26143 }
26144 }
26145
26146 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26147 let mut applied = 0usize;
26148
26149 for group in [&phase1[..], &phase2[..]] {
26150 if group.is_empty() {
26151 continue;
26152 }
26153 let mut computed_writes = ComputedWriteBuffer::default();
26154
26155 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26156 thread_pool.install(|| {
26157 group
26158 .par_iter()
26159 .map(|&vertex_id| {
26160 if cancel_flag.load(Ordering::Relaxed) {
26161 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26162 .with_message(
26163 "Parallel evaluation cancelled during execution"
26164 .to_string(),
26165 ));
26166 }
26167 match self.evaluate_vertex_immutable(vertex_id) {
26168 Ok(v) => Ok((vertex_id, v)),
26169 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26170 }
26171 })
26172 .collect()
26173 });
26174
26175 match results {
26176 Ok(vertex_results) => {
26177 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26178 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26179 for (vertex_id, result) in vertex_results {
26180 if matches!(result, LiteralValue::Array(_)) {
26181 arrays.push((vertex_id, result));
26182 } else {
26183 others.push((vertex_id, result));
26184 }
26185 }
26186 for (vertex_id, result) in arrays {
26187 let effects = match self.plan_vertex_effects_with_computed_flush(
26188 vertex_id,
26189 result,
26190 Some(&inflight),
26191 &mut computed_writes,
26192 ) {
26193 Ok(effects) => effects,
26194 Err(e) => {
26195 self.flush_computed_write_buffer(&mut computed_writes)?;
26196 return Err(e);
26197 }
26198 };
26199 for effect in &effects {
26200 if let Err(e) = self.apply_effect_with_computed_writes(
26201 effect,
26202 None,
26203 None,
26204 Some(&mut computed_writes),
26205 ) {
26206 self.flush_computed_write_buffer(&mut computed_writes)?;
26207 return Err(e);
26208 }
26209 }
26210 applied = applied.saturating_add(1);
26211 }
26212 self.flush_computed_write_buffer(&mut computed_writes)?;
26213 for (vertex_id, result) in others {
26214 let effects = match self.plan_vertex_effects_with_computed_flush(
26215 vertex_id,
26216 result,
26217 Some(&inflight),
26218 &mut computed_writes,
26219 ) {
26220 Ok(effects) => effects,
26221 Err(e) => {
26222 self.flush_computed_write_buffer(&mut computed_writes)?;
26223 return Err(e);
26224 }
26225 };
26226 for effect in &effects {
26227 if let Err(e) = self.apply_effect_with_computed_writes(
26228 effect,
26229 None,
26230 None,
26231 Some(&mut computed_writes),
26232 ) {
26233 self.flush_computed_write_buffer(&mut computed_writes)?;
26234 return Err(e);
26235 }
26236 }
26237 applied = applied.saturating_add(1);
26238 }
26239 self.flush_computed_write_buffer(&mut computed_writes)?;
26240 }
26241 Err(e) => {
26242 self.flush_computed_write_buffer(&mut computed_writes)?;
26243 return Err(e);
26244 }
26245 }
26246 }
26247
26248 Ok(applied)
26249 }
26250
26251 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
26258 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
26259 engine.evaluate_all_logged_unobserved(log)
26260 })
26261 }
26262
26263 fn evaluate_all_logged_unobserved(
26264 &mut self,
26265 log: &mut ChangeLog,
26266 ) -> Result<EvalResult, ExcelError> {
26267 self.observe_function_semantic_epoch()?;
26268 self.begin_evaluation_request();
26269 let _source_cache = self.source_cache_session();
26270 self.validate_deterministic_mode()?;
26271 if self.config.defer_graph_building {
26272 self.build_graph_all()?;
26273 }
26274 if self.graph.formula_authority().active_span_count() > 0 {
26275 return self.evaluate_authoritative_formula_plane_all();
26276 }
26277 self.reset_virtual_dep_telemetry_if_disabled();
26278 let start = crate::instant::FzInstant::now();
26279 let mut computed_vertices = 0;
26280 let mut cycle_errors = 0;
26281
26282 let mut replan_iterations = 0;
26283 const MAX_REPLAN: usize = 5;
26284 let mut telemetry = self
26285 .config
26286 .enable_virtual_dep_telemetry
26287 .then(|| self.start_virtual_dep_telemetry());
26288
26289 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
26290
26291 let result = (|| -> Result<EvalResult, ExcelError> {
26292 loop {
26293 let to_evaluate = self.graph.get_evaluation_vertices();
26294 if to_evaluate.is_empty() {
26295 if let Some(t) = telemetry.as_mut()
26296 && t.bailout_reason.is_none()
26297 {
26298 t.bailout_reason = Some("no_work");
26299 }
26300 break;
26301 }
26302
26303 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
26304 if let Some(t) = telemetry.as_mut() {
26305 Self::accumulate_schedule_meta(t, &meta);
26306 }
26307
26308 for &unit in &schedule.units {
26311 match unit {
26312 ScheduleUnit::Cycle(i) => {
26313 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
26324 {
26325 cycle_errors += 1;
26326 }
26327 }
26328 ScheduleUnit::Layer(i) => {
26329 computed_vertices +=
26330 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
26331 }
26332 }
26333 }
26334
26335 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
26336 if let Some(t) = telemetry.as_mut() {
26337 t.changed_vdeps_total += changed_vertices.len();
26338 }
26339 self.resource_checkpoint(0)?;
26340 self.graph.clear_dirty_flags(&to_evaluate);
26341 for v in &changed_vertices {
26342 self.graph.set_dirty(*v, true);
26343 }
26344
26345 if changed_vertices.is_empty() {
26346 if let Some(t) = telemetry.as_mut() {
26347 t.bailout_reason = Some("converged");
26348 }
26349 break;
26350 }
26351 if replan_iterations >= MAX_REPLAN {
26352 if let Some(mut t) = telemetry.take() {
26353 t.bailout_reason = Some("max_replan");
26354 t.replan_iterations = replan_iterations;
26355 self.last_virtual_dep_telemetry = t;
26356 }
26357 return Err(
26358 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
26359 );
26360 }
26361 replan_iterations += 1;
26362 }
26363
26364 if let Some(mut t) = telemetry {
26365 t.replan_iterations = replan_iterations;
26366 self.last_virtual_dep_telemetry = t;
26367 }
26368
26369 self.redirty_for_next_recalc();
26370 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
26371
26372 Ok(EvalResult {
26373 computed_vertices,
26374 cycle_errors,
26375 elapsed: start.elapsed(),
26376 })
26377 })();
26378 log.end_compound();
26379 result
26380 }
26381
26382 fn evaluate_layer_logged(
26384 &mut self,
26385 layer: &super::scheduler::Layer,
26386 log: &mut ChangeLog,
26387 ) -> Result<usize, ExcelError> {
26388 self.resource_checkpoint(layer.vertices.len() as u64)?;
26389 let mut computed_writes = ComputedWriteBuffer::default();
26390 for &vertex_id in &layer.vertices {
26391 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
26392 let value = match self.evaluate_vertex_immutable(vertex_id) {
26393 Ok(v) => v,
26394 Err(e) => LiteralValue::Error(e),
26395 };
26396 let effects = match self.plan_vertex_effects_with_computed_flush(
26397 vertex_id,
26398 value,
26399 None,
26400 &mut computed_writes,
26401 ) {
26402 Ok(effects) => effects,
26403 Err(e) => {
26404 self.flush_computed_write_buffer(&mut computed_writes)?;
26405 return Err(e);
26406 }
26407 };
26408 for effect in &effects {
26409 if let Err(e) = self.apply_effect_with_computed_writes(
26410 effect,
26411 None,
26412 Some(log),
26413 Some(&mut computed_writes),
26414 ) {
26415 self.flush_computed_write_buffer(&mut computed_writes)?;
26416 return Err(e);
26417 }
26418 }
26419 }
26420 self.flush_computed_write_buffer(&mut computed_writes)?;
26421 Ok(layer.vertices.len())
26422 }
26423}