1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::prepared_legacy_graph::{
8 PreparedLegacyGraphError, PreparedLegacyGraphPlan,
9};
10use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
11use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
12use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
13use crate::engine::live_graph::analyze_live_graph;
14use crate::engine::lookup_index_cache::{
15 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
16 LookupIndexKey, estimate_bytes,
17};
18use crate::engine::named_range::{NameScope, NamedDefinition};
19use crate::engine::range_view::RangeView;
20use crate::engine::row_visibility::RowVisibilityState;
21use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
22use crate::engine::target_preparation::{
23 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
24};
25use crate::engine::used_extent::{
26 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
27};
28use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
29use crate::engine::{
30 CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
31 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
32 EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
33 FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
34 FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
35 FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
36 FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
37 VertexId, VertexKind, VisibilityMaskMode,
38};
39use crate::formula_plane::placement::prepare_anchor_once_fragment;
40use crate::formula_plane::placement::{
41 CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
42 place_candidate_family_with_analyses, prepare_anchor_once_family,
43 split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
44 validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
45};
46use crate::formula_plane::producer::{
47 DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
48 FormulaProducerWork, ProducerDirtyDomain, SpanReadSummary,
49};
50use crate::formula_plane::region_index::{DirtyDomain, Region};
51use crate::formula_plane::runtime::{
52 FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
53};
54use crate::formula_plane::scheduler::{
55 MixedSchedule, MixedScheduleFallbackReason, MixedTopology, MixedTopologyCompileResult,
56 MixedTopologyCompileStats, MixedTopologyConfig, build_demand_closure_cached,
57 build_demand_closure_in_memory_runs, build_demand_closure_paged,
58 build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
59 schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
60 schedule_dirty_work_repeated_passes,
61};
62#[cfg(not(target_arch = "wasm32"))]
63use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
64#[cfg(not(target_arch = "wasm32"))]
65use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
66#[cfg(test)]
67use crate::formula_plane::span_eval::SpanEvalReport;
68use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
69use crate::formula_plane::structural::relocate_ast_for_template_placement;
70use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
71use crate::function::FnCaps;
72use crate::interpreter::Interpreter;
73use crate::reference::{CellRef, Coord, RangeRef};
74use crate::traits::FunctionProvider;
75use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
76use formualizer_common::{
77 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
78};
79use formualizer_parse::parser::ReferenceType;
80use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
81use rayon::ThreadPoolBuilder;
82use rustc_hash::{FxHashMap, FxHashSet};
83use std::collections::{BTreeMap, BTreeSet, VecDeque};
84use std::sync::Arc;
85use std::sync::atomic::{AtomicBool, Ordering};
86use std::time::Instant;
87
88type StagedFormulaEntry = (u32, u32, String);
89type StagedSheetParts = (
90 Vec<StagedFormulaEntry>,
91 Option<crate::engine::DeferredFormulaPackage>,
92);
93
94#[derive(Default)]
104pub(crate) struct StagedSheet {
105 entries: Vec<StagedFormulaEntry>,
106 index: FxHashMap<(u32, u32), usize>,
107 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
108}
109
110impl StagedSheet {
111 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
112 package
113 .invalidated
114 .extend(package.families.iter().map(|family| family.source_id));
115 package.invalidated.extend(
116 package
117 .partitioned_families
118 .iter()
119 .map(|family| family.source_id),
120 );
121 }
122
123 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
124 let Some(package) = self.deferred_package.as_mut() else {
125 return;
126 };
127 let family = package
128 .replay
129 .lock()
130 .ok()
131 .and_then(|mut replay| replay.formula_at(row, col).ok())
132 .flatten()
133 .and_then(|record| record.partition_owner.or(record.family));
134 if let Some(family) = family {
135 package.invalidated.insert(family);
136 } else {
137 Self::invalidate_all_deferred_families(package);
141 }
142 package.suppressed.insert((row, col));
143 }
144
145 fn reconcile_attached_deferred_package(&mut self) {
146 let Some(package) = self.deferred_package.as_mut() else {
147 return;
148 };
149 if self.entries.is_empty() {
150 return;
151 }
152
153 let replayed = (|| {
154 let mut disposition = crate::engine::FormulaReplayDisposition::default();
155 for partition in &package.partitioned_families {
156 disposition.register_partition(partition, false)?;
157 }
158 package
159 .replay
160 .lock()
161 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
162 .replay_partitioned(&disposition, &package.partitioned_families)
163 })();
164
165 let Ok(records) = replayed else {
166 Self::invalidate_all_deferred_families(package);
167 package
168 .suppressed
169 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
170 return;
171 };
172 let mut owners = FxHashMap::default();
173 for record in &records {
174 owners
175 .entry((record.row, record.col))
176 .or_insert(record.partition_owner.or(record.family));
177 }
178 for (row, col, _) in &self.entries {
179 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
180 package.invalidated.insert(family);
181 } else {
182 Self::invalidate_all_deferred_families(package);
183 }
184 package.suppressed.insert((*row, *col));
185 }
186 package.reconciliation_replay = Some(records);
187 }
188
189 fn stage(&mut self, row: u32, col: u32, text: String) {
190 self.invalidate_deferred_at(row, col);
191 match self.index.entry((row, col)) {
192 std::collections::hash_map::Entry::Occupied(slot) => {
193 self.entries[*slot.get()].2 = text;
194 }
195 std::collections::hash_map::Entry::Vacant(slot) => {
196 slot.insert(self.entries.len());
197 self.entries.push((row, col, text));
198 }
199 }
200 }
201
202 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
203 let deferred_text = self.get(row, col);
204 self.invalidate_deferred_at(row, col);
205 let Some(idx) = self.index.remove(&(row, col)) else {
206 return deferred_text;
207 };
208 let (_, _, text) = self.entries.remove(idx);
209 for slot in self.index.values_mut() {
211 if *slot > idx {
212 *slot -= 1;
213 }
214 }
215 Some(text)
216 }
217
218 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
219 self.index
220 .get(&(row, col))
221 .map(|&i| self.entries[i].2.as_str())
222 }
223
224 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
225 let idx = self.index.remove(&(row, col))?;
226 let (_, _, text) = self.entries.remove(idx);
227 for slot in self.index.values_mut() {
228 if *slot > idx {
229 *slot -= 1;
230 }
231 }
232 Some(text)
233 }
234
235 fn get(&self, row: u32, col: u32) -> Option<String> {
236 if let Some(text) = self.get_ordinary(row, col) {
237 return Some(text.to_string());
238 }
239 let package = self.deferred_package.as_ref()?;
240 if package.suppressed.contains(&(row, col)) {
241 return None;
242 }
243 package
244 .replay
245 .lock()
246 .ok()?
247 .formula_at(row, col)
248 .ok()?
249 .map(|record| record.text)
250 }
251
252 fn len(&self) -> usize {
253 self.deferred_package
254 .as_ref()
255 .map_or(self.entries.len(), |package| {
256 usize::try_from(package.report.source_formula_records_spooled)
257 .unwrap_or(usize::MAX)
258 .saturating_sub(package.suppressed.len())
259 .saturating_add(self.entries.len())
260 })
261 }
262
263 fn is_empty(&self) -> bool {
264 self.entries.is_empty() && self.deferred_package.is_none()
265 }
266
267 fn into_parts(self) -> StagedSheetParts {
268 (self.entries, self.deferred_package)
269 }
270}
271
272type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
273
274#[derive(Clone, Debug, PartialEq, Eq, Hash)]
275struct PreparationRegion {
276 sheet: String,
277 sheet_id: SheetId,
278 start_row: u32,
279 start_col: u32,
280 end_row: u32,
281 end_col: u32,
282}
283
284struct OrderedTargetProducers {
287 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
288 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
289}
290
291impl OrderedTargetProducers {
292 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
293 let mut ordered = Vec::new();
294 ordered.try_reserve(capacity)?;
295 let mut seen = FxHashSet::default();
296 seen.try_reserve(capacity)?;
297 Ok(Self { ordered, seen })
298 }
299
300 fn from_ordered(
301 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
302 ) -> Result<Self, std::collections::TryReserveError> {
303 let mut seen = FxHashSet::default();
304 seen.try_reserve(ordered.len())?;
305 seen.extend(ordered.iter().copied());
306 Ok(Self { ordered, seen })
307 }
308
309 fn push(
310 &mut self,
311 producer: crate::engine::target_preparation::TargetProducer,
312 ) -> Result<bool, std::collections::TryReserveError> {
313 #[cfg(test)]
314 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
315 if self.seen.contains(&producer) {
316 return Ok(false);
317 }
318 self.seen.try_reserve(1)?;
319 self.ordered.try_reserve(1)?;
320 self.seen.insert(producer);
321 self.ordered.push(producer);
322 Ok(true)
323 }
324
325 fn len(&self) -> usize {
326 self.ordered.len()
327 }
328
329 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
330 self.ordered
331 }
332}
333
334#[cfg(test)]
335thread_local! {
336 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
337}
338
339fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
340 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
341 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
342 limit: u64::MAX,
343 observed: observed as u64,
344 request_id,
345 })
346 .into_excel_error()
347}
348
349#[derive(Debug)]
350struct PreparedOrdinaryStagedFormula {
351 sheet: String,
352 sheet_id: SheetId,
353 lease: StagedFormulaLease,
354 ast_id: Option<AstNodeId>,
355 plan: Option<DependencyPlanRow>,
356}
357
358struct PreparedTargetSourcePackage {
359 sheet: String,
360 sheet_id: SheetId,
361 lease: StagedPackageLease,
362 source_report: crate::engine::FormulaCompressedSourceReport,
363 replay_records: Vec<crate::engine::DeferredReplayFormula>,
364 disposition: crate::engine::FormulaReplayDisposition,
365 placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
366 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
367 direct_families: usize,
368 direct_cells: u64,
369 direct_fragments: u64,
370 direct_complete_families: u64,
371 direct_complete_cells: u64,
372 direct_partition_families: u64,
373 direct_partition_cells: u64,
374 anchor_parses: u64,
375 anchor_asts: u64,
376 anchor_analyses: u64,
377}
378
379impl PreparedTargetSourcePackage {
380 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
381 self.replay_records.iter().filter(|record| {
382 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
383 return true;
384 };
385 let coord = crate::engine::SourceCoord { row, col };
386 let disposition = match record.family {
387 Some(family) => self.disposition.shared_disposition(family, coord),
388 None => self.disposition.ordinary_disposition(coord).0,
389 };
390 !matches!(
391 disposition,
392 crate::engine::FormulaReplayCoordinateDisposition::Direct
393 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
394 )
395 })
396 }
397}
398
399fn producer_dirty_to_span_dirty(
400 dirty: ProducerDirtyDomain,
401 span_ref: FormulaSpanRef,
402) -> DirtyDomain {
403 match dirty {
404 ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
405 ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
406 ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
407 }
408}
409type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
410type StagedFormulaBatches = Vec<(String, StagedSheet)>;
411#[derive(Debug)]
412pub(crate) struct SourceFamilyPreparationError {
413 reason: String,
414 parse_attempted: bool,
415 ast_created: bool,
416 analysis_created: bool,
417}
418
419impl SourceFamilyPreparationError {
420 fn contract(reason: &'static str) -> Self {
421 Self {
422 reason: reason.to_string(),
423 parse_attempted: false,
424 ast_created: false,
425 analysis_created: false,
426 }
427 }
428
429 fn parse(reason: &'static str) -> Self {
430 Self {
431 reason: reason.to_string(),
432 parse_attempted: true,
433 ast_created: false,
434 analysis_created: false,
435 }
436 }
437
438 fn ast(reason: impl Into<String>) -> Self {
439 Self {
440 reason: reason.into(),
441 parse_attempted: true,
442 ast_created: true,
443 analysis_created: false,
444 }
445 }
446
447 fn analysis(reason: impl Into<String>) -> Self {
448 Self {
449 reason: reason.into(),
450 parse_attempted: true,
451 ast_created: true,
452 analysis_created: true,
453 }
454 }
455}
456
457struct PreparedPartitionShadow {
458 fragment_count: u64,
459 direct_cells: u64,
460 fallback_cells: u64,
461 function_semantics_used: bool,
462}
463
464pub(crate) fn classify_mixed_topology_incomplete(
465 observed: &MixedTopologyCompileStats,
466) -> crate::engine::EvaluationIncompleteReason {
467 if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
468 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
469 } else if observed.memory_overflow_count > 0 {
470 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
471 } else if observed.edge_overflow_count > 0 {
472 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
473 } else if observed.candidate_overflow_count > 0 {
474 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
475 } else {
476 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
477 }
478}
479
480#[derive(Clone, Debug, Default)]
481struct LegacyIslandPlan {
482 membership: Vec<VertexId>,
483 dirty_vertices: Vec<VertexId>,
484 sheet_ids: Vec<SheetId>,
485 island_id: u64,
486 retained_bytes: usize,
487 boundary_relationships: usize,
488 omitted_relationships: usize,
489 omitted_direct_relationships: usize,
490}
491
492impl LegacyIslandPlan {
493 fn is_empty(&self) -> bool {
494 self.membership.is_empty()
495 }
496}
497
498type FormulaPlaneMixedScheduleBuild = (
499 MixedSchedule,
500 BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
501 u64,
502 Vec<VertexId>,
503 Vec<usize>,
504 LegacyIslandPlan,
505);
506
507#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub(crate) enum FormulaSpanDemotionFault {
509 #[default]
510 None,
511 AstPreparation,
512 LegacyGraphPreparation,
513 FinalLegacyGraphValidation,
514 FinalAuthorityValidation,
515 AllocationReservation,
516 BeforeFirstMutation,
517}
518
519#[derive(Debug)]
520pub(crate) enum FormulaSpanDemotionError {
521 StaleAuthority,
522 InvalidSpan,
523 CountOverflow,
524 Resource(ExcelError),
525 AstPreparation(ExcelError),
526 LegacyGraph(PreparedLegacyGraphError),
527 Injected(FormulaSpanDemotionFault),
528}
529
530impl std::fmt::Display for FormulaSpanDemotionError {
531 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
532 match self {
533 Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
534 Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
535 Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
536 Self::Resource(error) => write!(f, "{error}"),
537 Self::AstPreparation(error) => {
538 write!(f, "FormulaPlane demotion analysis failed: {error}")
539 }
540 Self::LegacyGraph(error) => {
541 write!(f, "FormulaPlane demotion graph plan failed: {error}")
542 }
543 Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
544 }
545 }
546}
547
548impl std::error::Error for FormulaSpanDemotionError {}
549
550pub(crate) struct PreparedFormulaSpanDemotion {
551 span_refs: Vec<FormulaSpanRef>,
552 expected_plane_epoch: u64,
553 expected_indexes_epoch: u64,
554 expected_indexed_plane_epoch: u64,
555 placement_count: usize,
556 legacy_graph: PreparedLegacyGraphPlan,
557 fault: FormulaSpanDemotionFault,
558}
559
560#[derive(Clone, Copy, Debug, PartialEq, Eq)]
561pub(crate) struct FormulaSpanDemotionReport {
562 pub(crate) spans_demoted: usize,
563 pub(crate) placements_materialized: usize,
564}
565
566type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
567type CompressedReplayBatch = (
568 FormulaIngestBatch,
569 crate::engine::FormulaCompressedSourceBatch,
570);
571type PreparedSourceBatch = (
572 FormulaIngestBatch,
573 crate::engine::FormulaCompressedSourceReport,
574 crate::engine::FormulaCompressedPreparation,
575);
576type PreparedStagedFormulaBatches = (
577 PreparedFormulaBatches,
578 Vec<CompressedReplayBatch>,
579 Vec<PreparedSourceBatch>,
580);
581
582#[doc(hidden)]
585pub struct SourceFormulaIngress<'a, R> {
586 engine: &'a mut Engine<R>,
587}
588
589impl<R> SourceFormulaIngress<'_, R>
590where
591 R: EvaluationContext,
592{
593 pub fn prepare_families(
594 &mut self,
595 sheet_name: &str,
596 families: &[crate::engine::SourceFormulaFamily],
597 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
598 self.engine.observe_function_semantic_epoch()?;
599 Ok(self
600 .engine
601 .prepare_source_formula_families(sheet_name, families))
602 }
603
604 pub fn prepare_eager_proposals(
605 &mut self,
606 sheet_name: &str,
607 families: &[crate::engine::SourceFormulaFamily],
608 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
609 formula_record_count: u64,
610 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
611 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
612 self.engine.observe_function_semantic_epoch()?;
613 let replay = Arc::new(std::sync::Mutex::new(replay));
614 self.engine.prepare_source_formula_proposals(
615 sheet_name,
616 families,
617 partitions,
618 partitions,
619 formula_record_count,
620 replay,
621 &BTreeSet::new(),
622 )
623 }
624
625 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
626 self.engine.stage_deferred_formula_package(package);
627 }
628
629 pub fn ingest_replay_batches(
630 &mut self,
631 batches: Vec<(
632 FormulaIngestBatch,
633 crate::engine::FormulaCompressedSourceBatch,
634 )>,
635 ) -> Result<FormulaIngestReport, ExcelError> {
636 self.engine
637 .ingest_compressed_formula_source_batches(batches)
638 }
639
640 pub fn finish_prepared(
641 &mut self,
642 batches: Vec<(
643 FormulaIngestBatch,
644 crate::engine::FormulaCompressedSourceReport,
645 crate::engine::FormulaCompressedPreparation,
646 )>,
647 ) -> Result<FormulaIngestReport, ExcelError> {
648 self.engine.finish_compressed_formula_sources(batches)
649 }
650}
651
652const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
656
657#[derive(Debug, Clone, PartialEq)]
658pub(crate) enum ComputedWrite {
659 Cell {
660 seq: u64,
661 sheet_id: SheetId,
662 row0: u32,
663 col0: u32,
664 value: OverlayValue,
665 },
666 Rect {
667 seq: u64,
668 sheet_id: SheetId,
669 sr0: u32,
670 sc0: u32,
671 values: Vec<Vec<OverlayValue>>,
672 },
673}
674
675impl ComputedWrite {
676 #[inline]
677 pub(crate) fn seq(&self) -> u64 {
678 match self {
679 ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
680 }
681 }
682}
683
684#[derive(Debug, Default)]
685pub(crate) struct ComputedWriteBuffer {
686 writes: Vec<ComputedWrite>,
687 next_seq: u64,
688 estimated_bytes: usize,
689}
690
691impl ComputedWriteBuffer {
692 const ENTRY_BASE_BYTES: usize = 32;
693
694 #[inline]
695 pub(crate) fn is_empty(&self) -> bool {
696 self.writes.is_empty()
697 }
698
699 #[inline]
700 pub(crate) fn len(&self) -> usize {
701 self.writes.len()
702 }
703
704 #[inline]
705 pub(crate) fn estimated_bytes(&self) -> usize {
706 self.estimated_bytes
707 }
708
709 #[inline]
710 pub(crate) fn writes(&self) -> &[ComputedWrite] {
711 &self.writes
712 }
713
714 pub(crate) fn push_cell(
715 &mut self,
716 sheet_id: SheetId,
717 row0: u32,
718 col0: u32,
719 value: OverlayValue,
720 ) {
721 let seq = self.next_sequence();
722 self.estimated_bytes = self
723 .estimated_bytes
724 .saturating_add(Self::estimate_value_bytes(&value));
725 self.writes.push(ComputedWrite::Cell {
726 seq,
727 sheet_id,
728 row0,
729 col0,
730 value,
731 });
732 }
733
734 pub(crate) fn push_rect(
735 &mut self,
736 sheet_id: SheetId,
737 sr0: u32,
738 sc0: u32,
739 values: Vec<Vec<OverlayValue>>,
740 ) {
741 let seq = self.next_sequence();
742 let added = values
743 .iter()
744 .flat_map(|row| row.iter())
745 .map(Self::estimate_value_bytes)
746 .fold(0usize, usize::saturating_add);
747 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
748 self.writes.push(ComputedWrite::Rect {
749 seq,
750 sheet_id,
751 sr0,
752 sc0,
753 values,
754 });
755 }
756
757 pub(crate) fn clear(&mut self) {
758 self.writes.clear();
759 self.estimated_bytes = 0;
760 }
761
762 fn take_writes(&mut self) -> Vec<ComputedWrite> {
763 self.estimated_bytes = 0;
764 std::mem::take(&mut self.writes)
765 }
766
767 fn next_sequence(&mut self) -> u64 {
768 let seq = self.next_seq;
769 self.next_seq = self.next_seq.wrapping_add(1);
770 seq
771 }
772
773 #[inline]
774 fn estimate_value_bytes(value: &OverlayValue) -> usize {
775 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
776 }
777}
778
779#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
780struct ComputedWriteChunkKey {
781 sheet_id: SheetId,
782 col0: u32,
783 chunk_idx: usize,
784 chunk_start_row0: u32,
785}
786
787#[derive(Debug, Clone, PartialEq)]
788pub(crate) struct ComputedWriteChunkEntryPlan {
789 pub(crate) row_in_chunk: usize,
790 pub(crate) seq: u64,
791 pub(crate) value: OverlayValue,
792}
793
794#[derive(Debug, Clone, PartialEq, Eq)]
795pub(crate) enum ComputedWriteChunkPlanShape {
796 Point,
797 SparseOffsets {
798 entries: usize,
799 span_len: usize,
800 },
801 DenseRange {
802 start: usize,
803 len: usize,
804 },
805 RunRange {
806 start: usize,
807 len: usize,
808 runs: usize,
809 },
810}
811
812#[derive(Debug, Clone, PartialEq)]
813pub(crate) struct ComputedWriteChunkPlan {
814 pub(crate) sheet_id: SheetId,
815 pub(crate) col0: u32,
816 pub(crate) chunk_idx: usize,
817 pub(crate) chunk_start_row0: u32,
818 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
819 pub(crate) shape: ComputedWriteChunkPlanShape,
820}
821
822#[derive(Debug, Clone, Default, PartialEq)]
823pub(crate) struct ComputedWriteCoalescingPlan {
824 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
825 pub(crate) input_cells: usize,
826 pub(crate) coalesced_cells: usize,
827 pub(crate) overwritten_cells: usize,
828}
829
830impl ComputedWriteCoalescingPlan {
831 #[inline]
832 pub(crate) fn is_empty(&self) -> bool {
833 self.chunks.is_empty()
834 }
835}
836
837impl ComputedWriteChunkPlan {
838 fn from_group(
839 key: ComputedWriteChunkKey,
840 mut entries: Vec<ComputedWriteChunkEntryPlan>,
841 ) -> (Self, usize) {
842 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
843 let input_len = entries.len();
844 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
845 for entry in entries {
846 if let Some(prev) = coalesced.last_mut()
847 && prev.row_in_chunk == entry.row_in_chunk
848 {
849 *prev = entry;
850 continue;
851 }
852 coalesced.push(entry);
853 }
854 let overwritten = input_len.saturating_sub(coalesced.len());
855 let shape = Self::classify_shape(&coalesced);
856 (
857 Self {
858 sheet_id: key.sheet_id,
859 col0: key.col0,
860 chunk_idx: key.chunk_idx,
861 chunk_start_row0: key.chunk_start_row0,
862 entries: coalesced,
863 shape,
864 },
865 overwritten,
866 )
867 }
868
869 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
870 debug_assert!(!entries.is_empty());
871 if entries.len() == 1 {
872 return ComputedWriteChunkPlanShape::Point;
873 }
874
875 let start = entries[0].row_in_chunk;
876 let end = entries[entries.len() - 1].row_in_chunk;
877 let span_len = end.saturating_sub(start).saturating_add(1);
878 if span_len != entries.len() {
879 return ComputedWriteChunkPlanShape::SparseOffsets {
880 entries: entries.len(),
881 span_len,
882 };
883 }
884
885 let runs = Self::run_count(entries);
886 if runs < entries.len() {
887 ComputedWriteChunkPlanShape::RunRange {
888 start,
889 len: entries.len(),
890 runs,
891 }
892 } else {
893 ComputedWriteChunkPlanShape::DenseRange {
894 start,
895 len: entries.len(),
896 }
897 }
898 }
899
900 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
901 let mut runs = 0usize;
902 let mut prev: Option<&OverlayValue> = None;
903 for entry in entries {
904 if prev != Some(&entry.value) {
905 runs = runs.saturating_add(1);
906 prev = Some(&entry.value);
907 }
908 }
909 runs
910 }
911}
912
913pub struct Engine<R> {
914 pub(crate) graph: DependencyGraph,
915 resolver: R,
916 pub config: EvalConfig,
917 workbook_load_limits: crate::engine::WorkbookLoadLimits,
918 clock: crate::timezone::SnapshotClock,
923 thread_pool: Option<Arc<rayon::ThreadPool>>,
924 pub recalc_epoch: u64,
925 snapshot_id: std::sync::atomic::AtomicU64,
926 topology_epoch: u64,
927 legacy_island_structural_summaries_trusted: bool,
930 cached_static_schedule: Option<CachedScheduleEntry>,
931 cached_mixed_topology: Option<CachedMixedTopology>,
932 mixed_topology_cache_builds: u64,
933 mixed_topology_cache_hits: u64,
934 mixed_topology_cache_overflows: u64,
935 mixed_topology_cache_skip_streak: u64,
936 spill_mgr: ShimSpillManager,
937 arrow_sheets: SheetStore,
939 has_edited: bool,
941 overlay_compactions: u64,
943
944 computed_overlay_bytes_estimate: usize,
946 computed_overlay_mirroring_disabled: bool,
947 pub(crate) force_materialize_range_views: bool,
950 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
952 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
954 lookup_index_cache: LookupIndexCache,
955 source_cache: Arc<std::sync::RwLock<SourceCache>>,
956 source_formula_token: Arc<()>,
958 recalc_plan_token: Arc<()>,
960 staged_formulas: StagedFormulaMap,
962 staged_formula_index: StagedFormulaIndex,
964 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
966 row_visibility_mask_cache: std::sync::RwLock<
968 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
969 >,
970 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
972 last_formula_ingest_report: Option<FormulaIngestReport>,
974 formula_ingest_report_total: FormulaIngestReport,
976 formula_plane_cycle_member_span_demotions: u64,
986 formula_plane_capacity_bailouts: u64,
991 formula_plane_structural_span_candidates: u64,
993 active_cancel_flag: Option<crate::engine::CancelToken>,
995 active_evaluation_deadline: Option<Instant>,
997
998 action_depth: u32,
1003
1004 last_virtual_dep_telemetry: VirtualDepTelemetry,
1006 virtual_dep_fallback_activations: u64,
1007
1008 last_cycle_telemetry: CycleTelemetry,
1010
1011 next_evaluation_resource_request_id: u64,
1013 evaluation_resource_request_depth: usize,
1014 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1015 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1016 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1017 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1018 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1019 evaluation_resource_config_diagnostic:
1020 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1021 active_resource_ledger: Option<ResourceLedger>,
1022
1023 pending_iterative_redirty: Vec<VertexId>,
1037
1038 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1048
1049 function_semantic_epoch_seen: u64,
1052 function_provider_revision_seen: Option<u64>,
1055
1056 #[cfg(test)]
1057 last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1058 #[cfg(test)]
1059 evaluation_request_begin_count_for_test: u64,
1060 #[cfg(any(test, feature = "test-support"))]
1061 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1062 #[cfg(test)]
1063 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1064 #[cfg(test)]
1065 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1066 #[cfg(test)]
1067 inject_target_semantic_stale_once_for_test: bool,
1068 #[cfg(test)]
1069 force_virtual_dep_changes_remaining_for_test: usize,
1070 #[cfg(test)]
1071 fail_evaluation_commit_preflight_once_for_test: bool,
1072 #[cfg(test)]
1073 force_source_family_fallback: bool,
1074 #[cfg(test)]
1075 rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1076 #[cfg(test)]
1077 fragmented_commit_fault_for_test:
1078 Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1079 #[cfg(test)]
1080 formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1081 #[cfg(test)]
1082 target_preparation_fault_for_test:
1083 Option<crate::engine::target_preparation::TargetPreparationFault>,
1084 #[cfg(test)]
1085 force_non_cycle_schedule_fallback_for_test: bool,
1086 #[cfg(test)]
1087 mixed_topology_index_builds_for_test: u64,
1088 #[cfg(test)]
1089 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1090 #[cfg(test)]
1091 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1092}
1093
1094fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1097 fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1098 let node_type = match &ast.node_type {
1099 ASTNodeType::Literal(_) if !in_array => {
1100 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1101 *next = next.saturating_add(1);
1102 ASTNodeType::Literal(value)
1103 }
1104 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1105 ASTNodeType::Omitted => ASTNodeType::Omitted,
1106 ASTNodeType::Reference {
1107 original,
1108 reference,
1109 } => ASTNodeType::Reference {
1110 original: original.clone(),
1111 reference: reference.clone(),
1112 },
1113 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1114 op: op.clone(),
1115 expr: Box::new(visit(expr, binding, next, in_array)),
1116 },
1117 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1118 op: op.clone(),
1119 left: Box::new(visit(left, binding, next, in_array)),
1120 right: Box::new(visit(right, binding, next, in_array)),
1121 },
1122 ASTNodeType::Function { name, args } => ASTNodeType::Function {
1123 name: name.clone(),
1124 args: args
1125 .iter()
1126 .map(|arg| visit(arg, binding, next, in_array))
1127 .collect(),
1128 },
1129 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1130 callee: Box::new(visit(callee, binding, next, in_array)),
1131 args: args
1132 .iter()
1133 .map(|arg| visit(arg, binding, next, in_array))
1134 .collect(),
1135 },
1136 ASTNodeType::Array(rows) => ASTNodeType::Array(
1137 rows.iter()
1138 .map(|row| {
1139 row.iter()
1140 .map(|cell| visit(cell, binding, next, true))
1141 .collect()
1142 })
1143 .collect(),
1144 ),
1145 };
1146 ASTNode::new(node_type, ast.source_token.clone())
1147 }
1148
1149 let mut next = 0;
1150 visit(ast, binding, &mut next, false)
1151}
1152
1153impl<R: EvaluationContext> Engine<R> {
1156 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1157 self.graph.ingest_pipeline(&self.resolver)
1158 }
1159}
1160
1161pub struct EngineAction<'a, R>
1162where
1163 R: EvaluationContext,
1164{
1165 engine: &'a mut Engine<R>,
1166 name: String,
1167 log: Option<*mut crate::engine::ChangeLog>,
1170 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1173 atomic_policy: bool,
1175}
1176
1177impl<'a, R> EngineAction<'a, R>
1178where
1179 R: EvaluationContext,
1180{
1181 #[inline]
1182 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1183 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1184 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1185 crate::reference::CellRef::new(sheet_id, coord)
1186 }
1187
1188 #[inline]
1189 pub fn name(&self) -> &str {
1190 &self.name
1191 }
1192
1193 #[inline]
1194 pub fn set_cell_value(
1195 &mut self,
1196 sheet: &str,
1197 row: u32,
1198 col: u32,
1199 value: LiteralValue,
1200 ) -> Result<(), crate::engine::EditorError> {
1201 if self.log.is_some() {
1202 let old_value = self.engine.read_cell_value(sheet, row, col);
1203 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1204 let addr = self.addr_for(sheet, row, col);
1205 let Some(log_ptr) = self.log else {
1206 return Err(crate::engine::EditorError::TransactionFailed {
1207 reason: "action_with_logger: missing ChangeLog".to_string(),
1208 });
1209 };
1210
1211 let old_comp = if self.arrow_undo.is_some() {
1214 self.engine.read_computed_overlay_cell(sheet, row, col)
1215 } else {
1216 None
1217 };
1218
1219 if self.engine.graph_admission_enabled() {
1220 let admission =
1221 self.engine
1222 .graph
1223 .preview_value_mutation(addr.sheet_id, row, col)?;
1224 self.engine.preflight_graph_admission(admission)?;
1225 }
1226 self.engine.demote_span_containing_cell_for_write(
1227 addr.sheet_id,
1228 addr.coord.row(),
1229 addr.coord.col(),
1230 )?;
1231 if old_formula.is_none() {
1232 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1233 }
1234
1235 let delta_old_sem = if old_formula.is_some() {
1236 None
1237 } else {
1238 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1239 };
1240
1241 let start_len = unsafe { (&*log_ptr).len() };
1242
1243 let log = unsafe { &mut *log_ptr };
1245 self.engine.edit_with_logger(log, |editor| {
1246 editor.set_cell_value_with_old_state(
1247 addr,
1248 value.clone(),
1249 old_value.clone(),
1250 old_formula.clone(),
1251 );
1252 })?;
1253 self.engine
1254 .record_formula_plane_structural_change(StructuralScope::Cell {
1255 sheet: addr.sheet_id,
1256 row: addr.coord.row(),
1257 col: addr.coord.col(),
1258 });
1259
1260 if let Some(undo_ptr) = self.arrow_undo {
1261 let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1263 let undo = unsafe { &mut *undo_ptr };
1264 self.engine
1265 .record_spill_ops_into_arrow_undo(undo, new_events);
1266
1267 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1269 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1270 let row0 = row.saturating_sub(1);
1271 let col0 = col.saturating_sub(1);
1272 let delta_new_sem = Some(value.clone());
1273 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1274 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1275 }
1276 Ok(())
1277 } else {
1278 self.engine
1279 .set_cell_value(sheet, row, col, value)
1280 .map_err(crate::engine::EditorError::from)
1281 }
1282 }
1283
1284 #[inline]
1285 pub fn set_cell_formula(
1286 &mut self,
1287 sheet: &str,
1288 row: u32,
1289 col: u32,
1290 ast: ASTNode,
1291 ) -> Result<(), crate::engine::EditorError> {
1292 if self.log.is_some() {
1293 let old_value = self.engine.read_cell_value(sheet, row, col);
1294 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1295 let addr = self.addr_for(sheet, row, col);
1296 let Some(log_ptr) = self.log else {
1297 return Err(crate::engine::EditorError::TransactionFailed {
1298 reason: "action_with_logger: missing ChangeLog".to_string(),
1299 });
1300 };
1301
1302 let admitted_formula = if self.engine.graph_admission_enabled() {
1303 let placement =
1304 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1305 let ingested = self.engine.ingest_pipeline().ingest_formula(
1306 FormulaAstInput::Tree(ast.clone()),
1307 placement,
1308 None,
1309 )?;
1310 let admission = self.engine.graph.preview_formula_mutations(&[(
1311 addr.sheet_id,
1312 row,
1313 col,
1314 ingested.dep_plan.clone(),
1315 )])?;
1316 self.engine.preflight_graph_admission(admission)?;
1317 Some((ingested.ast_id, ingested.dep_plan))
1318 } else {
1319 None
1320 };
1321 self.engine.demote_span_containing_cell_for_write(
1322 addr.sheet_id,
1323 addr.coord.row(),
1324 addr.coord.col(),
1325 )?;
1326 if old_formula.is_none() {
1327 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328 }
1329 let delta_old = if self.arrow_undo.is_some() {
1330 if old_formula.is_some() {
1331 None
1332 } else {
1333 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334 }
1335 } else {
1336 None
1337 };
1338 let start_len = unsafe { (&*log_ptr).len() };
1339
1340 let log = unsafe { &mut *log_ptr };
1342 self.engine.edit_with_logger(log, |editor| {
1343 if let Some((ast_id, plan)) = admitted_formula {
1344 editor.set_cell_formula_with_prepared_plan(
1345 addr,
1346 ast.clone(),
1347 old_value,
1348 old_formula,
1349 ast_id,
1350 plan,
1351 );
1352 } else {
1353 editor.set_cell_formula_with_old_state(
1354 addr,
1355 ast.clone(),
1356 old_value,
1357 old_formula,
1358 );
1359 }
1360 })?;
1361 self.engine
1362 .record_formula_plane_structural_change(StructuralScope::Cell {
1363 sheet: addr.sheet_id,
1364 row: addr.coord.row(),
1365 col: addr.coord.col(),
1366 });
1367
1368 if let Some(undo_ptr) = self.arrow_undo {
1369 let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1370 let undo = unsafe { &mut *undo_ptr };
1371 self.engine
1372 .record_spill_ops_into_arrow_undo(undo, new_events);
1373 let delta_new: Option<LiteralValue> = None;
1374 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1375 let row0 = row.saturating_sub(1);
1376 let col0 = col.saturating_sub(1);
1377 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1378 }
1379 Ok(())
1380 } else {
1381 self.engine
1382 .set_cell_formula(sheet, row, col, ast)
1383 .map_err(crate::engine::EditorError::from)
1384 }
1385 }
1386
1387 #[inline]
1388 pub fn set_row_hidden(
1389 &mut self,
1390 sheet: &str,
1391 row_1based: u32,
1392 hidden: bool,
1393 source: RowVisibilitySource,
1394 ) -> Result<(), crate::engine::EditorError> {
1395 if self.log.is_some() {
1396 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1397 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1398 let old_hidden = self
1399 .engine
1400 .row_visibility
1401 .get(&sheet_id)
1402 .map(|state| state.is_row_hidden(row0, Some(source)))
1403 .unwrap_or(false);
1404 if old_hidden == hidden {
1405 return Ok(());
1406 }
1407
1408 let _ = self
1409 .engine
1410 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1411
1412 let Some(log_ptr) = self.log else {
1413 return Err(crate::engine::EditorError::TransactionFailed {
1414 reason: "action_with_logger: missing ChangeLog".to_string(),
1415 });
1416 };
1417 unsafe { &mut *log_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1418 sheet_id,
1419 row0,
1420 source,
1421 old_hidden,
1422 new_hidden: hidden,
1423 });
1424
1425 Ok(())
1426 } else {
1427 self.engine
1428 .set_row_hidden(sheet, row_1based, hidden, source)
1429 }
1430 }
1431
1432 #[inline]
1433 pub fn set_rows_hidden(
1434 &mut self,
1435 sheet: &str,
1436 start_row_1based: u32,
1437 end_row_1based: u32,
1438 hidden: bool,
1439 source: RowVisibilitySource,
1440 ) -> Result<(), crate::engine::EditorError> {
1441 if self.log.is_some() {
1442 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1443 let (start_row0, end_row0) =
1444 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1445
1446 let Some(log_ptr) = self.log else {
1447 return Err(crate::engine::EditorError::TransactionFailed {
1448 reason: "action_with_logger: missing ChangeLog".to_string(),
1449 });
1450 };
1451 let log = unsafe { &mut *log_ptr };
1452
1453 for row0 in start_row0..=end_row0 {
1454 let old_hidden = self
1455 .engine
1456 .row_visibility
1457 .get(&sheet_id)
1458 .map(|state| state.is_row_hidden(row0, Some(source)))
1459 .unwrap_or(false);
1460 if old_hidden == hidden {
1461 continue;
1462 }
1463
1464 let _ = self
1465 .engine
1466 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1467
1468 log.record(crate::engine::ChangeEvent::SetRowVisibility {
1469 sheet_id,
1470 row0,
1471 source,
1472 old_hidden,
1473 new_hidden: hidden,
1474 });
1475 }
1476
1477 Ok(())
1478 } else {
1479 self.engine
1480 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1481 }
1482 }
1483
1484 #[inline]
1485 pub fn insert_rows(
1486 &mut self,
1487 sheet: &str,
1488 before: u32,
1489 count: u32,
1490 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1491 if count == 0 {
1492 return Ok(crate::engine::ShiftSummary::default());
1493 }
1494 if self.log.is_some() {
1495 let Some(log_ptr) = self.log else {
1496 return Err(crate::engine::EditorError::TransactionFailed {
1497 reason: "action_atomic: missing ChangeLog".to_string(),
1498 });
1499 };
1500
1501 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1502 let before0 = before.saturating_sub(1);
1503 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1504 self.engine
1508 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1509
1510 let summary = {
1512 let log = unsafe { &mut *log_ptr };
1513 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1514 Ok(crate::engine::ShiftSummary::default());
1515 self.engine.edit_with_logger(log, |editor| {
1516 out = editor.insert_rows(sheet_id, before0, count);
1517 })?;
1518 out?
1519 };
1520
1521 self.engine.ensure_arrow_sheet(sheet);
1523 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1524 asheet.insert_rows(before0 as usize, count as usize);
1525 }
1526 self.engine
1527 .shift_row_visibility_insert(sheet_id, before0, count);
1528 self.engine.mark_moved_formula_vertices_dirty(&summary);
1529 self.engine
1530 .clear_computed_overlay_after_row(sheet, before0 as usize);
1531 self.engine
1532 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1533 self.engine.mark_topology_edited();
1534 if let Some(undo_ptr) = self.arrow_undo {
1535 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1536 }
1537 Ok(summary)
1538 } else {
1539 self.engine.insert_rows(sheet, before, count)
1540 }
1541 }
1542
1543 #[inline]
1544 pub fn delete_rows(
1545 &mut self,
1546 sheet: &str,
1547 start: u32,
1548 count: u32,
1549 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1550 if count == 0 {
1551 return Ok(crate::engine::ShiftSummary::default());
1552 }
1553 if self.atomic_policy {
1554 return Err(crate::engine::EditorError::TransactionUnsupported {
1555 reason:
1556 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1557 .to_string(),
1558 });
1559 }
1560 self.engine.delete_rows(sheet, start, count)
1561 }
1562
1563 #[inline]
1564 pub fn insert_columns(
1565 &mut self,
1566 sheet: &str,
1567 before: u32,
1568 count: u32,
1569 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1570 if count == 0 {
1571 return Ok(crate::engine::ShiftSummary::default());
1572 }
1573 if self.log.is_some() {
1574 let Some(log_ptr) = self.log else {
1575 return Err(crate::engine::EditorError::TransactionFailed {
1576 reason: "action_atomic: missing ChangeLog".to_string(),
1577 });
1578 };
1579
1580 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1581 let before0 = before.saturating_sub(1);
1582 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1583 self.engine
1584 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1585
1586 let summary = {
1587 let log = unsafe { &mut *log_ptr };
1588 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1589 Ok(crate::engine::ShiftSummary::default());
1590 self.engine.edit_with_logger(log, |editor| {
1591 out = editor.insert_columns(sheet_id, before0, count);
1592 })?;
1593 out?
1594 };
1595
1596 self.engine.ensure_arrow_sheet(sheet);
1597 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1598 asheet.insert_columns(before0 as usize, count as usize);
1599 }
1600 self.engine.mark_moved_formula_vertices_dirty(&summary);
1601 self.engine
1602 .clear_computed_overlay_after_col(sheet, before0 as usize);
1603 self.engine
1604 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1605 self.engine.mark_topology_edited();
1606 if let Some(undo_ptr) = self.arrow_undo {
1607 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1608 }
1609 Ok(summary)
1610 } else {
1611 self.engine.insert_columns(sheet, before, count)
1612 }
1613 }
1614
1615 #[inline]
1616 pub fn delete_columns(
1617 &mut self,
1618 sheet: &str,
1619 start: u32,
1620 count: u32,
1621 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1622 if count == 0 {
1623 return Ok(crate::engine::ShiftSummary::default());
1624 }
1625 if self.atomic_policy {
1626 return Err(crate::engine::EditorError::TransactionUnsupported {
1627 reason:
1628 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1629 .to_string(),
1630 });
1631 }
1632 self.engine.delete_columns(sheet, start, count)
1633 }
1634
1635 #[inline]
1640 pub fn action<T>(
1641 &mut self,
1642 name: impl AsRef<str>,
1643 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1644 ) -> Result<T, crate::engine::EditorError> {
1645 self.engine.action(name, f)
1646 }
1647}
1648
1649struct ActionDepthGuard<'a, R> {
1650 engine: *mut Engine<R>,
1651 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1652}
1653
1654impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1655 fn drop(&mut self) {
1656 unsafe {
1659 let e = &mut *self.engine;
1660 e.action_depth = e.action_depth.saturating_sub(1);
1661 }
1662 }
1663}
1664
1665#[derive(Default)]
1666struct SourceCache {
1667 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1668 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1669}
1670
1671#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1672struct VisibilityMaskCacheKey {
1673 sheet_id: SheetId,
1674 start_row0: u32,
1675 end_row0: u32,
1676 mode: VisibilityMaskMode,
1677 version: u64,
1678}
1679
1680#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1681enum StructuralScope {
1682 Cell { sheet: SheetId, row: u32, col: u32 },
1683 Region(Region),
1684 Sheet(SheetId),
1685 RemovedSheet(SheetId),
1686 OpaqueGlobal,
1687 AllSheets,
1688}
1689
1690struct SourceCacheSession {
1691 cache: Arc<std::sync::RwLock<SourceCache>>,
1692}
1693
1694impl Drop for SourceCacheSession {
1695 fn drop(&mut self) {
1696 if let Ok(mut g) = self.cache.write() {
1697 *g = SourceCache::default();
1698 }
1699 }
1700}
1701
1702#[derive(Debug)]
1703#[non_exhaustive]
1704pub struct EvalResult {
1705 pub computed_vertices: usize,
1706 pub cycle_errors: usize,
1707 pub elapsed: std::time::Duration,
1708}
1709
1710#[derive(Clone, Debug, PartialEq, Eq)]
1711#[non_exhaustive]
1712pub struct TableMetadata {
1713 pub name: String,
1714 pub sheet: String,
1715 pub start_row: u32,
1716 pub start_col: u32,
1717 pub end_row: u32,
1718 pub end_col: u32,
1719 pub header_row: bool,
1720 pub headers: Vec<String>,
1721 pub totals_row: bool,
1722}
1723
1724#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1729#[non_exhaustive]
1730pub struct EngineBaselineStats {
1731 pub graph_vertex_count: usize,
1732 pub graph_formula_vertex_count: usize,
1733 pub graph_edge_count: usize,
1734 pub dirty_vertex_count: usize,
1735 pub evaluation_vertex_count: usize,
1736 pub formula_ast_root_count: usize,
1737 pub formula_ast_node_count: usize,
1738 pub staged_formula_count: usize,
1739 pub formula_plane_active_span_count: usize,
1740 pub formula_plane_producer_result_entries: usize,
1741 pub formula_plane_consumer_read_entries: usize,
1742 pub formula_plane_mixed_topology_cache_builds: u64,
1743 pub formula_plane_mixed_topology_cache_hits: u64,
1744 pub formula_plane_mixed_topology_cache_overflows: u64,
1745 pub formula_plane_dirty_pending_events: usize,
1746 pub formula_plane_dirty_region_events_recorded: u64,
1747 pub formula_plane_dirty_span_region_events_recorded: u64,
1748 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1749 pub formula_plane_dirty_global_invalidations: u64,
1750 pub formula_plane_structural_span_candidates: u64,
1751 pub formula_plane_cycle_member_span_demotions: u64,
1754}
1755
1756#[derive(Debug, Clone, Default)]
1757#[non_exhaustive]
1758pub struct VirtualDepTelemetry {
1759 pub candidate_vertices_total: usize,
1760 pub vdeps_vertices_total: usize,
1761 pub vdeps_edges_total: usize,
1762 pub builder_elapsed_ms_total: u128,
1763 pub schedule_virtual_passes: usize,
1764 pub schedule_static_passes: usize,
1765 pub schedule_cache_hits: usize,
1766 pub schedule_cache_misses: usize,
1767 pub reused_schedule_vertices_total: usize,
1768 pub replan_iterations: usize,
1769 pub changed_vdeps_total: usize,
1770 pub bailout_reason: Option<&'static str>,
1771 pub fallback_mode_activations: u64,
1772}
1773
1774#[derive(Debug, Clone, Default, PartialEq)]
1783#[non_exhaustive]
1784pub struct CycleTelemetry {
1785 pub static_sccs: usize,
1787 pub phantom_sccs: usize,
1789 pub live_cycles_witnessed: usize,
1791 pub circ_cells_stamped: usize,
1793 pub settle_passes_total: usize,
1796 pub max_passes_single_scc: usize,
1798 pub iterated_sccs: usize,
1801 pub converged_sccs: usize,
1804 pub capped_sccs: usize,
1810 pub max_abs_delta_at_stop: f64,
1814 pub nan_converged: usize,
1817 pub elapsed_ms: u128,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct ScheduleBuildMeta {
1823 candidate_vertices: usize,
1824 vdeps_vertices: usize,
1825 vdeps_edges: usize,
1826 builder_elapsed_ms: u128,
1827 used_virtual_schedule: bool,
1828 schedule_cache_hit: bool,
1829 schedule_cache_eligible: bool,
1830}
1831
1832#[derive(Debug, Clone)]
1833struct CachedScheduleEntry {
1834 topology_epoch: u64,
1835 candidate_vertices: Vec<VertexId>,
1836 schedule: crate::engine::scheduler::Schedule,
1837}
1838
1839#[derive(Clone, Debug, PartialEq, Eq)]
1840struct MixedTopologyCacheKey {
1841 engine_topology_epoch: u64,
1842 graph_topology_revision: u64,
1843 authority_indexes_epoch: u64,
1844 legacy_island_revision: u64,
1848 boundary_index_revision: u64,
1849 function_semantic_epoch: u64,
1850 function_provider_revision: Option<u64>,
1851 max_candidates: usize,
1852 max_edges: usize,
1853 max_memory_bytes: usize,
1854}
1855
1856#[derive(Debug)]
1857struct CachedMixedTopology {
1858 key: MixedTopologyCacheKey,
1859 topology: MixedTopology,
1860 producer_results: FormulaProducerResultIndex,
1861 consumer_reads: FormulaConsumerReadIndex,
1862 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1863 plane_epoch: u64,
1864 island: LegacyIslandPlan,
1865}
1866
1867#[derive(Debug)]
1868struct SkippedMixedTopology {
1869 reason: MixedScheduleFallbackReason,
1870 observed: MixedTopologyCompileStats,
1871 producer_results: FormulaProducerResultIndex,
1872 consumer_reads: FormulaConsumerReadIndex,
1873 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1874 plane_epoch: u64,
1875 island: LegacyIslandPlan,
1876}
1877
1878#[derive(Debug)]
1879enum FormulaPlaneTopologyCompileResult {
1880 Cached(CachedMixedTopology),
1881 CacheSkipped(SkippedMixedTopology),
1882}
1883
1884#[derive(Clone, Copy, Debug)]
1885struct ExactDemandScratchEstimates {
1886 paged: u64,
1887 runs: u64,
1888 native: u64,
1889 repeated: u64,
1890}
1891
1892fn exact_demand_scratch_estimates(
1893 read_count: usize,
1894 closure_base: u64,
1895) -> ExactDemandScratchEstimates {
1896 let read_count = read_count as u64;
1897 let page_count = read_count.div_ceil(128);
1898 let run_count = read_count.div_ceil(256);
1899 let pointer_bytes = std::mem::size_of::<usize>() as u64;
1900 let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
1901 + std::mem::size_of::<Vec<usize>>()
1902 + 12 * std::mem::size_of::<usize>()) as u64;
1903 let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
1904 page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
1905 );
1906 let runs_memory = read_count
1907 .saturating_mul(pointer_bytes)
1908 .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
1909 ExactDemandScratchEstimates {
1910 paged: closure_base.saturating_add(paged_memory),
1911 runs: closure_base.saturating_add(runs_memory),
1912 native: closure_base
1913 .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
1914 repeated: closure_base.saturating_add(512),
1915 }
1916}
1917
1918fn native_exact_demand_allowed(
1919 policy: Option<crate::engine::DiskScratchPolicy>,
1920 target_supports_native: bool,
1921) -> bool {
1922 target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
1923}
1924
1925fn select_exact_demand_strategy(
1926 estimates: ExactDemandScratchEstimates,
1927 native_allowed: bool,
1928 mut can_reserve: impl FnMut(u64) -> bool,
1929) -> (FormulaPlaneTopologyStrategy, u64) {
1930 if can_reserve(estimates.paged) {
1931 (
1932 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1933 estimates.paged,
1934 )
1935 } else if can_reserve(estimates.runs) {
1936 (
1937 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1938 estimates.runs,
1939 )
1940 } else if native_allowed && can_reserve(estimates.native) {
1941 (
1942 FormulaPlaneTopologyStrategy::ExactNativeScratch,
1943 estimates.native,
1944 )
1945 } else {
1946 (
1947 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1948 estimates.repeated,
1949 )
1950 }
1951}
1952
1953#[cfg(test)]
1954mod exact_demand_strategy_tests {
1955 use super::*;
1956
1957 #[test]
1958 fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
1959 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
1960 assert!(estimates.paged > estimates.runs);
1961 assert!(estimates.runs > estimates.native);
1962 assert!(estimates.native > estimates.repeated);
1963 for (limit, native_allowed, expected) in [
1964 (
1965 estimates.paged,
1966 true,
1967 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1968 ),
1969 (
1970 estimates.runs,
1971 true,
1972 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1973 ),
1974 (
1975 estimates.native,
1976 true,
1977 FormulaPlaneTopologyStrategy::ExactNativeScratch,
1978 ),
1979 (
1980 estimates.repeated,
1981 true,
1982 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1983 ),
1984 ] {
1985 assert_eq!(
1986 select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
1987 expected
1988 );
1989 }
1990 }
1991
1992 #[cfg(not(target_arch = "wasm32"))]
1993 #[test]
1994 fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
1995 let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
1996 let path = scratch.path.clone();
1997 let auxiliary_path = scratch.auxiliary_path.clone();
1998 assert!(path.exists());
1999 assert!(auxiliary_path.exists());
2000 drop(scratch);
2001 assert!(!path.exists());
2002 assert!(!auxiliary_path.exists());
2003 }
2004
2005 #[test]
2006 fn native_policy_and_wasm_target_support_gate_native_rung() {
2007 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2008 assert!(!native_exact_demand_allowed(
2009 Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2010 true,
2011 ));
2012 assert!(!native_exact_demand_allowed(
2013 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2014 false,
2015 ));
2016 let native_allowed = native_exact_demand_allowed(
2017 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2018 !cfg!(target_arch = "wasm32"),
2019 );
2020 let expected = if cfg!(target_arch = "wasm32") {
2021 FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2022 } else {
2023 FormulaPlaneTopologyStrategy::ExactNativeScratch
2024 };
2025 assert_eq!(
2026 select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2027 bytes <= estimates.native
2028 })
2029 .0,
2030 expected
2031 );
2032 }
2033}
2034
2035#[cfg(not(target_arch = "wasm32"))]
2036struct NativeTopologyScratch {
2037 path: std::path::PathBuf,
2038 file: Option<std::fs::File>,
2039 auxiliary_path: std::path::PathBuf,
2040 auxiliary_file: Option<std::fs::File>,
2041}
2042
2043#[cfg(not(target_arch = "wasm32"))]
2044impl NativeTopologyScratch {
2045 fn create(request_id: u64) -> std::io::Result<Self> {
2046 use std::fs::OpenOptions;
2047 for attempt in 0..32_u64 {
2048 let path = std::env::temp_dir().join(format!(
2049 "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2050 std::process::id()
2051 ));
2052 match OpenOptions::new()
2053 .read(true)
2054 .write(true)
2055 .create_new(true)
2056 .open(&path)
2057 {
2058 Ok(file) => {
2059 let auxiliary_path = path.with_extension("aux.tmp");
2060 match OpenOptions::new()
2061 .read(true)
2062 .write(true)
2063 .create_new(true)
2064 .open(&auxiliary_path)
2065 {
2066 Ok(auxiliary_file) => {
2067 return Ok(Self {
2068 path,
2069 file: Some(file),
2070 auxiliary_path,
2071 auxiliary_file: Some(auxiliary_file),
2072 });
2073 }
2074 Err(error) => {
2075 drop(file);
2076 let _ = std::fs::remove_file(&path);
2077 if error.kind() == std::io::ErrorKind::AlreadyExists {
2078 continue;
2079 }
2080 return Err(error);
2081 }
2082 }
2083 }
2084 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2085 Err(error) => return Err(error),
2086 }
2087 }
2088 Err(std::io::Error::new(
2089 std::io::ErrorKind::AlreadyExists,
2090 "could not allocate unique topology scratch file",
2091 ))
2092 }
2093}
2094
2095#[cfg(not(target_arch = "wasm32"))]
2096impl Drop for NativeTopologyScratch {
2097 fn drop(&mut self) {
2098 drop(self.file.take());
2099 drop(self.auxiliary_file.take());
2100 let _ = std::fs::remove_file(&self.path);
2101 let _ = std::fs::remove_file(&self.auxiliary_path);
2102 }
2103}
2104
2105fn estimated_span_bindings_bytes(
2106 bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2107) -> Option<usize> {
2108 const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2109 bindings.len().checked_mul(
2110 std::mem::size_of::<FormulaSpanId>()
2111 .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2112 .checked_add(TREE_ENTRY_OVERHEAD)?,
2113 )
2114}
2115
2116type ScheduleBuildOutput = (
2117 crate::engine::scheduler::Schedule,
2118 FxHashMap<VertexId, Vec<VertexId>>,
2119 ScheduleBuildMeta,
2120);
2121
2122#[derive(Debug)]
2124pub struct RecalcPlan {
2125 key: RecalcPlanKey,
2126 kind: RecalcPlanKind,
2127}
2128
2129#[derive(Debug)]
2130struct RecalcPlanKey {
2131 engine_token: Arc<()>,
2132 revisions: PlanningRevisionSnapshot,
2133}
2134
2135#[derive(Clone, Debug, PartialEq, Eq)]
2136struct PlanningRevisionSnapshot {
2137 engine_topology_epoch: u64,
2138 graph_topology_revision: u64,
2139 authority: u64,
2140 authority_indexes: u64,
2141 authority_indexed_plane: u64,
2142 staged: u64,
2143 symbols: u64,
2144 semantic: u64,
2145 provider: Option<u64>,
2146 formula_plane_mode: FormulaPlaneMode,
2147 deterministic_mode: crate::engine::DeterministicMode,
2148 budgets: crate::engine::EvaluationBudgets,
2149 span_refs: Vec<FormulaSpanRef>,
2150}
2151
2152#[derive(Debug)]
2153enum RecalcPlanKind {
2154 CompatibilityFull {
2155 schedule: crate::engine::Schedule,
2156 has_dynamic_refs: bool,
2157 },
2158 Target {
2159 targets: Vec<crate::engine::EvaluationTarget>,
2160 scope: crate::engine::PrepareScope,
2161 topology: RecalcTopology,
2162 dynamic_policy: DynamicPlanPolicy,
2163 },
2164}
2165
2166#[derive(Debug)]
2167enum RecalcTopology {
2168 RunLocalRecipe,
2169 Workbook,
2170}
2171
2172#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2173enum DynamicPlanPolicy {
2174 BoundedTargetReplan,
2175}
2176
2177impl RecalcPlan {
2178 pub fn layer_count(&self) -> usize {
2181 match &self.kind {
2182 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2183 RecalcPlanKind::Target { .. } => 0,
2184 }
2185 }
2186
2187 pub fn has_dynamic_refs(&self) -> bool {
2188 match &self.kind {
2189 RecalcPlanKind::CompatibilityFull {
2190 has_dynamic_refs, ..
2191 } => *has_dynamic_refs,
2192 RecalcPlanKind::Target { .. } => false,
2193 }
2194 }
2195
2196 #[cfg(test)]
2197 pub(crate) fn force_stale_reasons_for_test(
2198 &mut self,
2199 reasons: &[formualizer_common::PlanStaleReason],
2200 ) {
2201 use formualizer_common::PlanStaleReason;
2202 for reason in reasons {
2203 match reason {
2204 PlanStaleReason::Engine => {
2205 self.key.engine_token = Arc::new(());
2206 }
2207 PlanStaleReason::Provider => {
2208 self.key.revisions.provider = Some(
2209 self.key
2210 .revisions
2211 .provider
2212 .unwrap_or_default()
2213 .wrapping_add(1),
2214 );
2215 }
2216 PlanStaleReason::Semantic => {
2217 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2218 }
2219 PlanStaleReason::Budget => {
2220 let current = self.key.revisions.budgets.work.max_work_units;
2221 self.key.revisions.budgets.work.max_work_units =
2222 Some(current.unwrap_or_default().wrapping_add(1));
2223 }
2224 PlanStaleReason::Staged => {
2225 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2226 }
2227 PlanStaleReason::Symbols => {
2228 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2229 }
2230 PlanStaleReason::Authority => {
2231 self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2232 }
2233 PlanStaleReason::SpanGeneration => {
2234 self.key.revisions.span_refs.push(FormulaSpanRef {
2235 id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2236 generation: u32::MAX,
2237 version: u32::MAX,
2238 });
2239 }
2240 PlanStaleReason::Graph => {
2241 self.key.revisions.graph_topology_revision =
2242 self.key.revisions.graph_topology_revision.wrapping_add(1);
2243 }
2244 _ => {}
2245 }
2246 }
2247 }
2248}
2249
2250#[cfg(test)]
2251pub(crate) mod criteria_mask_test_hooks {
2252 use std::cell::Cell;
2253
2254 thread_local! {
2255 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2256 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2257 }
2258
2259 pub fn reset_text_segment_counters() {
2260 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2261 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2262 }
2263
2264 pub fn text_segment_counters() -> (usize, usize) {
2265 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2266 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2267 (a, b)
2268 }
2269
2270 pub(crate) fn inc_total() {
2271 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2272 }
2273 pub(crate) fn inc_all_null() {
2274 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2275 }
2276}
2277
2278#[cfg(test)]
2279pub(crate) mod visibility_mask_test_hooks {
2280 use std::cell::Cell;
2281
2282 thread_local! {
2283 static HITS: Cell<usize> = const { Cell::new(0) };
2284 static MISSES: Cell<usize> = const { Cell::new(0) };
2285 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2286 }
2287
2288 pub fn reset() {
2289 HITS.with(|c| c.set(0));
2290 MISSES.with(|c| c.set(0));
2291 EVICTIONS.with(|c| c.set(0));
2292 }
2293
2294 pub fn counters() -> (usize, usize, usize) {
2295 let hits = HITS.with(|c| c.get());
2296 let misses = MISSES.with(|c| c.get());
2297 let evictions = EVICTIONS.with(|c| c.get());
2298 (hits, misses, evictions)
2299 }
2300
2301 pub(crate) fn inc_hit() {
2302 HITS.with(|c| c.set(c.get() + 1));
2303 }
2304
2305 pub(crate) fn inc_miss() {
2306 MISSES.with(|c| c.set(c.get() + 1));
2307 }
2308
2309 pub(crate) fn inc_eviction() {
2310 EVICTIONS.with(|c| c.set(c.get() + 1));
2311 }
2312}
2313
2314fn compute_criteria_mask(
2315 view: &RangeView<'_>,
2316 col_in_view: usize,
2317 pred: &crate::args::CriteriaPredicate,
2318) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2319 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2320 use arrow::compute::kernels::comparison::{ilike, nilike};
2321 use arrow_array::{
2322 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2323 };
2324
2325 fn apply_numeric_pred(
2327 chunk: &Float64Array,
2328 pred: &crate::args::CriteriaPredicate,
2329 ) -> Option<BooleanArray> {
2330 match pred {
2331 crate::args::CriteriaPredicate::Gt(n) => {
2332 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2333 }
2334 crate::args::CriteriaPredicate::Ge(n) => {
2335 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2336 }
2337 crate::args::CriteriaPredicate::Lt(n) => {
2338 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2339 }
2340 crate::args::CriteriaPredicate::Le(n) => {
2341 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2342 }
2343 crate::args::CriteriaPredicate::Eq(v) => match v {
2344 formualizer_common::LiteralValue::Number(x) => {
2345 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2346 }
2347 formualizer_common::LiteralValue::Int(i) => {
2348 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2349 }
2350 _ => None,
2351 },
2352 crate::args::CriteriaPredicate::Ne(v) => match v {
2353 formualizer_common::LiteralValue::Number(x) => {
2354 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2355 }
2356 formualizer_common::LiteralValue::Int(i) => {
2357 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2358 }
2359 _ => None,
2360 },
2361 _ => None,
2362 }
2363 }
2364
2365 let is_numeric_pred = matches!(
2367 pred,
2368 crate::args::CriteriaPredicate::Gt(_)
2369 | crate::args::CriteriaPredicate::Ge(_)
2370 | crate::args::CriteriaPredicate::Lt(_)
2371 | crate::args::CriteriaPredicate::Le(_)
2372 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2373 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2374 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2375 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2376 );
2377
2378 if is_numeric_pred {
2382 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2383 for res in view.numbers_slices() {
2384 let (_rs, _rl, cols_seg) = res.ok()?;
2385 if col_in_view < cols_seg.len() {
2386 let chunk = cols_seg[col_in_view].as_ref();
2387 let mask = apply_numeric_pred(chunk, pred)?;
2388 bool_parts.push(mask);
2389 }
2390 }
2391
2392 if bool_parts.is_empty() {
2393 return None;
2394 } else if bool_parts.len() == 1 {
2395 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2396 } else {
2397 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2399 .iter()
2400 .map(|a| a as &dyn arrow_array::Array)
2401 .collect();
2402 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2403 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2404 return Some(std::sync::Arc::new(ba));
2405 }
2406 }
2407
2408 let (text_kind, text_pat, empty_special) = match pred {
2411 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2412 (0u8, t.to_lowercase(), t.is_empty())
2413 }
2414 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2415 (1u8, t.to_lowercase(), false)
2416 }
2417 crate::args::CriteriaPredicate::TextLike {
2418 pattern,
2419 case_insensitive,
2420 } => {
2421 let p = if *case_insensitive {
2422 pattern.to_lowercase()
2423 } else {
2424 pattern.clone()
2425 };
2426 (2u8, p.replace('*', "%").replace('?', "_"), false)
2427 }
2428 _ => return None,
2429 };
2430
2431 let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2432 let pat = StringArray::new_scalar(text_pat);
2433 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2434
2435 for res in view.iter_row_chunks() {
2436 let cs = res.ok()?;
2437 if cs.row_len == 0 {
2438 continue;
2439 }
2440 #[cfg(test)]
2441 criteria_mask_test_hooks::inc_total();
2442
2443 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2444 if col_in_view >= slices.len() {
2445 return None;
2446 }
2447
2448 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2449 let seg = match seg_opt {
2450 Some(s) => s,
2451 None => {
2452 #[cfg(test)]
2453 criteria_mask_test_hooks::inc_all_null();
2454 if (text_kind == 0 && empty_special) || ne_matches_blank {
2455 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2457 bb.append_n(cs.row_len, true);
2458 bool_parts.push(bb.finish());
2459 } else {
2460 bool_parts.push(BooleanArray::new_null(cs.row_len));
2462 }
2463 continue;
2464 }
2465 };
2466
2467 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2468 let mut m = match text_kind {
2469 0 => ilike(seg_sa, &pat).ok()?,
2470 1 => nilike(seg_sa, &pat).ok()?,
2471 2 => ilike(seg_sa, &pat).ok()?,
2472 _ => return None,
2473 };
2474
2475 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2481 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2483 for i in 0..seg_sa.len() {
2484 bb.append_value(seg_sa.is_null(i));
2485 }
2486 let nulls = bb.finish();
2487 m = boolean::or_kleene(&m, &nulls).ok()?;
2488 }
2489
2490 bool_parts.push(m);
2491 }
2492
2493 if bool_parts.is_empty() {
2494 None
2495 } else if bool_parts.len() == 1 {
2496 Some(std::sync::Arc::new(bool_parts.remove(0)))
2497 } else {
2498 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2499 .iter()
2500 .map(|a| a as &dyn arrow_array::Array)
2501 .collect();
2502 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2503 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2504 Some(std::sync::Arc::new(ba))
2505 }
2506}
2507
2508#[derive(Debug, Clone)]
2509pub struct LayerInfo {
2510 pub vertex_count: usize,
2511 pub parallel_eligible: bool,
2512 pub sample_cells: Vec<String>, }
2514
2515#[derive(Debug, Clone)]
2516pub struct EvalPlan {
2517 pub total_vertices_to_evaluate: usize,
2518 pub layers: Vec<LayerInfo>,
2519 pub cycles_detected: usize,
2520 pub dirty_count: usize,
2521 pub volatile_count: usize,
2522 pub parallel_enabled: bool,
2523 pub estimated_parallel_layers: usize,
2524 pub target_cells: Vec<String>,
2525}
2526
2527impl<R> Engine<R>
2528where
2529 R: EvaluationContext,
2530{
2531 pub fn new(resolver: R, config: EvalConfig) -> Self {
2538 if let Err(msg) = config.cycle.validate() {
2539 panic!("invalid CycleConfig: {msg}");
2540 }
2541 crate::builtins::load_builtins();
2542 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2543 &config.evaluation_budgets,
2544 config.max_vertices,
2545 config.max_memory_mb,
2546 config.max_eval_time,
2547 );
2548
2549 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2550 #[cfg(feature = "system-clock")]
2551 {
2552 Arc::new(crate::timezone::SystemClock::new(
2553 crate::timezone::TimeZoneSpec::default(),
2554 ))
2555 }
2556 #[cfg(not(feature = "system-clock"))]
2557 {
2558 Arc::new(crate::timezone::FixedClock::new(
2559 chrono::DateTime::UNIX_EPOCH,
2560 crate::timezone::TimeZoneSpec::Utc,
2561 ))
2562 }
2563 });
2564
2565 let thread_pool = if config.enable_parallel {
2567 let mut builder = ThreadPoolBuilder::new();
2568 if let Some(max_threads) = config.max_threads {
2569 builder = builder.num_threads(max_threads);
2570 }
2571
2572 match builder.build() {
2573 Ok(pool) => Some(Arc::new(pool)),
2574 Err(_) => {
2575 None
2577 }
2578 }
2579 } else {
2580 None
2581 };
2582
2583 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2585 let function_provider_revision_seen = resolver.planning_semantic_revision();
2586 let mut engine = Self {
2587 graph: DependencyGraph::new_with_config(config.clone()),
2588 resolver,
2589 config,
2590 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2591 clock: crate::timezone::SnapshotClock::new(clock),
2592 thread_pool,
2593 recalc_epoch: 0,
2594 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2595 topology_epoch: 0,
2596 legacy_island_structural_summaries_trusted: true,
2597 cached_static_schedule: None,
2598 cached_mixed_topology: None,
2599 mixed_topology_cache_builds: 0,
2600 mixed_topology_cache_hits: 0,
2601 mixed_topology_cache_overflows: 0,
2602 mixed_topology_cache_skip_streak: 0,
2603 spill_mgr: ShimSpillManager::default(),
2604 arrow_sheets: SheetStore::default(),
2605 has_edited: false,
2606 overlay_compactions: 0,
2607 computed_overlay_bytes_estimate: 0,
2608 computed_overlay_mirroring_disabled: false,
2609 force_materialize_range_views: false,
2610 row_bounds_cache: std::sync::RwLock::new(None),
2611 used_axis_bounds_cache: std::sync::RwLock::new(None),
2612 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2613 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2614 source_formula_token: Arc::new(()),
2615 recalc_plan_token: Arc::new(()),
2616 staged_formulas: std::collections::HashMap::new(),
2617 staged_formula_index: StagedFormulaIndex::default(),
2618 row_visibility: FxHashMap::default(),
2619 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2620 formula_parse_diagnostics: Vec::new(),
2621 last_formula_ingest_report: None,
2622 formula_ingest_report_total: FormulaIngestReport::default(),
2623 formula_plane_cycle_member_span_demotions: 0,
2624 formula_plane_capacity_bailouts: 0,
2625 formula_plane_structural_span_candidates: 0,
2626 active_cancel_flag: None,
2627 active_evaluation_deadline: None,
2628 action_depth: 0,
2629 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2630 virtual_dep_fallback_activations: 0,
2631 last_cycle_telemetry: CycleTelemetry::default(),
2632 next_evaluation_resource_request_id: 1,
2633 evaluation_resource_request_depth: 0,
2634 active_evaluation_resource_request: None,
2635 last_evaluation_resource_request: None,
2636 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2637 evaluation_resource_request_started_at: None,
2638 evaluation_resource_budgets: resolved_resources.budgets,
2639 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2640 active_resource_ledger: None,
2641 pending_iterative_redirty: Vec::new(),
2642 iterative_state_values: FxHashMap::default(),
2643 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2644 function_provider_revision_seen,
2645 #[cfg(test)]
2646 last_formula_plane_span_eval_report: None,
2647 #[cfg(test)]
2648 evaluation_request_begin_count_for_test: 0,
2649 #[cfg(any(test, feature = "test-support"))]
2650 before_prepared_span_commit_hook: None,
2651 #[cfg(test)]
2652 before_target_preparation_commit_hook: None,
2653 #[cfg(test)]
2654 before_target_planning_snapshot_hook: None,
2655 #[cfg(test)]
2656 inject_target_semantic_stale_once_for_test: false,
2657 #[cfg(test)]
2658 force_virtual_dep_changes_remaining_for_test: 0,
2659 #[cfg(test)]
2660 fail_evaluation_commit_preflight_once_for_test: false,
2661 #[cfg(test)]
2662 force_source_family_fallback: false,
2663 #[cfg(test)]
2664 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2665 #[cfg(test)]
2666 fragmented_commit_fault_for_test: None,
2667 #[cfg(test)]
2668 formula_span_demotion_fault_for_test: None,
2669 #[cfg(test)]
2670 target_preparation_fault_for_test: None,
2671 #[cfg(test)]
2672 force_non_cycle_schedule_fallback_for_test: false,
2673 #[cfg(test)]
2674 mixed_topology_index_builds_for_test: 0,
2675 #[cfg(test)]
2676 before_legacy_fallback_final_provider_sample_hook: None,
2677 #[cfg(test)]
2678 after_eager_proposal_commit_hook: None,
2679 };
2680 engine.config.arrow_storage_enabled = true;
2682 engine.config.delta_overlay_enabled = true;
2683 engine.config.write_formula_overlay_enabled = true;
2684 let default_sheet = engine.graph.default_sheet_name().to_string();
2685 engine.ensure_arrow_sheet(&default_sheet);
2686 engine
2687 }
2688
2689 pub fn with_thread_pool(
2694 resolver: R,
2695 config: EvalConfig,
2696 thread_pool: Arc<rayon::ThreadPool>,
2697 ) -> Self {
2698 if let Err(msg) = config.cycle.validate() {
2699 panic!("invalid CycleConfig: {msg}");
2700 }
2701 crate::builtins::load_builtins();
2702 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2703 &config.evaluation_budgets,
2704 config.max_vertices,
2705 config.max_memory_mb,
2706 config.max_eval_time,
2707 );
2708 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2709 #[cfg(feature = "system-clock")]
2710 {
2711 Arc::new(crate::timezone::SystemClock::new(
2712 crate::timezone::TimeZoneSpec::default(),
2713 ))
2714 }
2715 #[cfg(not(feature = "system-clock"))]
2716 {
2717 Arc::new(crate::timezone::FixedClock::new(
2718 chrono::DateTime::UNIX_EPOCH,
2719 crate::timezone::TimeZoneSpec::Utc,
2720 ))
2721 }
2722 });
2723 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2725 let function_provider_revision_seen = resolver.planning_semantic_revision();
2726 let mut engine = Self {
2727 graph: DependencyGraph::new_with_config(config.clone()),
2728 resolver,
2729 config,
2730 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2731 clock: crate::timezone::SnapshotClock::new(clock),
2732 thread_pool: Some(thread_pool),
2733 recalc_epoch: 0,
2734 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2735 topology_epoch: 0,
2736 legacy_island_structural_summaries_trusted: true,
2737 cached_static_schedule: None,
2738 cached_mixed_topology: None,
2739 mixed_topology_cache_builds: 0,
2740 mixed_topology_cache_hits: 0,
2741 mixed_topology_cache_overflows: 0,
2742 mixed_topology_cache_skip_streak: 0,
2743 spill_mgr: ShimSpillManager::default(),
2744 arrow_sheets: SheetStore::default(),
2745 has_edited: false,
2746 overlay_compactions: 0,
2747 computed_overlay_bytes_estimate: 0,
2748 computed_overlay_mirroring_disabled: false,
2749 force_materialize_range_views: false,
2750 row_bounds_cache: std::sync::RwLock::new(None),
2751 used_axis_bounds_cache: std::sync::RwLock::new(None),
2752 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2753 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2754 source_formula_token: Arc::new(()),
2755 recalc_plan_token: Arc::new(()),
2756 staged_formulas: std::collections::HashMap::new(),
2757 staged_formula_index: StagedFormulaIndex::default(),
2758 row_visibility: FxHashMap::default(),
2759 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2760 formula_parse_diagnostics: Vec::new(),
2761 last_formula_ingest_report: None,
2762 formula_ingest_report_total: FormulaIngestReport::default(),
2763 formula_plane_cycle_member_span_demotions: 0,
2764 formula_plane_capacity_bailouts: 0,
2765 formula_plane_structural_span_candidates: 0,
2766 active_cancel_flag: None,
2767 active_evaluation_deadline: None,
2768 action_depth: 0,
2769 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2770 virtual_dep_fallback_activations: 0,
2771 last_cycle_telemetry: CycleTelemetry::default(),
2772 next_evaluation_resource_request_id: 1,
2773 evaluation_resource_request_depth: 0,
2774 active_evaluation_resource_request: None,
2775 last_evaluation_resource_request: None,
2776 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2777 evaluation_resource_request_started_at: None,
2778 evaluation_resource_budgets: resolved_resources.budgets,
2779 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2780 active_resource_ledger: None,
2781 pending_iterative_redirty: Vec::new(),
2782 iterative_state_values: FxHashMap::default(),
2783 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2784 function_provider_revision_seen,
2785 #[cfg(test)]
2786 last_formula_plane_span_eval_report: None,
2787 #[cfg(test)]
2788 evaluation_request_begin_count_for_test: 0,
2789 #[cfg(any(test, feature = "test-support"))]
2790 before_prepared_span_commit_hook: None,
2791 #[cfg(test)]
2792 before_target_preparation_commit_hook: None,
2793 #[cfg(test)]
2794 before_target_planning_snapshot_hook: None,
2795 #[cfg(test)]
2796 inject_target_semantic_stale_once_for_test: false,
2797 #[cfg(test)]
2798 force_virtual_dep_changes_remaining_for_test: 0,
2799 #[cfg(test)]
2800 fail_evaluation_commit_preflight_once_for_test: false,
2801 #[cfg(test)]
2802 force_source_family_fallback: false,
2803 #[cfg(test)]
2804 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2805 #[cfg(test)]
2806 fragmented_commit_fault_for_test: None,
2807 #[cfg(test)]
2808 formula_span_demotion_fault_for_test: None,
2809 #[cfg(test)]
2810 target_preparation_fault_for_test: None,
2811 #[cfg(test)]
2812 force_non_cycle_schedule_fallback_for_test: false,
2813 #[cfg(test)]
2814 mixed_topology_index_builds_for_test: 0,
2815 #[cfg(test)]
2816 before_legacy_fallback_final_provider_sample_hook: None,
2817 #[cfg(test)]
2818 after_eager_proposal_commit_hook: None,
2819 };
2820 engine.config.arrow_storage_enabled = true;
2822 engine.config.delta_overlay_enabled = true;
2823 engine.config.write_formula_overlay_enabled = true;
2824 let default_sheet = engine.graph.default_sheet_name().to_string();
2825 engine.ensure_arrow_sheet(&default_sheet);
2826 engine
2827 }
2828
2829 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
2830 &self.workbook_load_limits
2831 }
2832
2833 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
2834 self.workbook_load_limits = limits;
2835 }
2836
2837 fn clear_source_cache(&self) {
2838 if let Ok(mut g) = self.source_cache.write() {
2839 *g = SourceCache::default();
2840 }
2841 }
2842
2843 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
2844 &self.last_virtual_dep_telemetry
2845 }
2846
2847 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
2851 &self.last_cycle_telemetry
2852 }
2853
2854 pub fn last_evaluation_resource_request_stats(
2856 &self,
2857 ) -> Option<&EvaluationResourceRequestStats> {
2858 self.last_evaluation_resource_request.as_ref()
2859 }
2860
2861 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
2863 self.evaluation_resource_baseline
2864 }
2865
2866 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
2867 &self.evaluation_resource_budgets
2868 }
2869
2870 pub fn evaluation_resource_config_diagnostic(
2872 &self,
2873 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
2874 self.evaluation_resource_config_diagnostic.as_ref()
2875 }
2876
2877 pub fn reset_evaluation_resource_telemetry(&mut self) {
2879 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
2880 self.last_evaluation_resource_request = None;
2881 }
2882
2883 fn duration_ns(duration: std::time::Duration) -> u64 {
2884 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
2885 }
2886
2887 fn observe_evaluation_resource_request<T>(
2888 &mut self,
2889 kind: EvaluationRequestKind,
2890 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
2891 ) -> Result<T, ExcelError> {
2892 let outermost = self.evaluation_resource_request_depth == 0;
2893 if outermost {
2894 let request_id = self.next_evaluation_resource_request_id;
2895 self.next_evaluation_resource_request_id = request_id
2896 .checked_add(1)
2897 .expect("evaluation resource request ID exhausted");
2898 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
2899 request_id,
2900 kind,
2901 self.config.formula_plane_mode,
2902 self.staged_formula_count(),
2903 ));
2904 self.evaluation_resource_baseline.record_started(request_id);
2905 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
2906 self.active_resource_ledger = Some(ResourceLedger::new(
2907 Some(request_id),
2908 self.evaluation_resource_budgets.clone(),
2909 ));
2910 }
2911 self.evaluation_resource_request_depth =
2912 self.evaluation_resource_request_depth.saturating_add(1);
2913 let result = if outermost {
2914 self.resource_checkpoint(0).and_then(|()| evaluate(self))
2915 } else {
2916 evaluate(self)
2917 };
2918 self.evaluation_resource_request_depth =
2919 self.evaluation_resource_request_depth.saturating_sub(1);
2920
2921 if outermost {
2922 let total_ns = self
2923 .evaluation_resource_request_started_at
2924 .take()
2925 .map(|start| Self::duration_ns(start.elapsed()))
2926 .unwrap_or(0);
2927 let mut stats = self
2928 .active_evaluation_resource_request
2929 .take()
2930 .expect("outer evaluation resource request has active stats");
2931 let mut ledger = self
2932 .active_resource_ledger
2933 .take()
2934 .expect("outer evaluation resource request has active ledger");
2935 ledger.release_all_scratch();
2936 stats.ledger.update(ledger.snapshot());
2937 stats.outcome = match &result {
2938 Ok(_) => EvaluationRequestOutcome::Success,
2939 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
2940 EvaluationRequestOutcome::Cancelled
2941 }
2942 Err(_) => EvaluationRequestOutcome::Error,
2943 };
2944 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
2945 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
2946 FormulaDirtyLeaseOutcome::RetainedOnCancellation
2947 } else {
2948 FormulaDirtyLeaseOutcome::RetainedOnError
2949 };
2950 }
2951 stats.phases.total_ns = total_ns;
2952 let attributed = stats
2953 .phases
2954 .staged_prepare_ns
2955 .saturating_add(stats.phases.topology_ns)
2956 .saturating_add(stats.phases.materialization_ns);
2957 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
2958 self.evaluation_resource_baseline.record_finished(&stats);
2959 self.last_evaluation_resource_request = Some(stats);
2960 }
2961 result
2962 }
2963
2964 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
2965 self.evaluation_resource_budgets = budgets.clone();
2966 self.config.evaluation_budgets = budgets.clone();
2967 self.graph.set_evaluation_budgets(budgets);
2968 }
2969
2970 #[cfg(test)]
2971 pub(crate) fn set_evaluation_budgets_for_test(
2972 &mut self,
2973 budgets: crate::engine::EvaluationBudgets,
2974 ) {
2975 self.set_evaluation_resource_budgets(budgets);
2976 }
2977
2978 fn resource_loop_checkpoint(
2979 ledger: &mut Option<ResourceLedger>,
2980 work_units: u64,
2981 ) -> Result<(), ExcelError> {
2982 ledger
2983 .as_mut()
2984 .map_or(Ok(()), |ledger| {
2985 ledger
2986 .charge_work(work_units)
2987 .and_then(|()| ledger.checkpoint_deadline())
2988 })
2989 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
2990 }
2991
2992 fn preflight_evaluation_commit_window(
2993 &mut self,
2994 bounded_writes: usize,
2995 ) -> Result<crate::instant::FzInstant, ExcelError> {
2996 #[cfg(test)]
2997 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
2998 return Err(crate::engine::ResourceLedgerError::Exhausted(
2999 formualizer_common::ResourceExhaustionDetail {
3000 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3001 limit: 0,
3002 observed: 1,
3003 request_id: self
3004 .active_evaluation_resource_request
3005 .as_ref()
3006 .map(|stats| stats.request_id),
3007 },
3008 )
3009 .into_excel_error());
3010 }
3011 let estimate = std::time::Duration::from_nanos(
3012 u64::try_from(bounded_writes)
3013 .unwrap_or(u64::MAX)
3014 .saturating_mul(100),
3015 );
3016 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3017 ledger
3018 .preflight_commit_window(estimate)
3019 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3020 }
3021 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3022 stats.evaluation_commit_preflight_count =
3023 stats.evaluation_commit_preflight_count.saturating_add(1);
3024 stats.evaluation_commit_estimated_ns = stats
3025 .evaluation_commit_estimated_ns
3026 .saturating_add(Self::duration_ns(estimate));
3027 }
3028 Ok(crate::instant::FzInstant::now())
3029 }
3030
3031 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3032 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3033 stats.evaluation_commit_actual_ns = stats
3034 .evaluation_commit_actual_ns
3035 .saturating_add(Self::duration_ns(started.elapsed()));
3036 }
3037 }
3038
3039 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3040 if self
3041 .active_cancel_flag
3042 .as_ref()
3043 .is_some_and(|cancel| cancel.is_cancelled())
3044 {
3045 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3046 }
3047 if self
3048 .active_evaluation_deadline
3049 .is_some_and(|deadline| Instant::now() >= deadline)
3050 {
3051 return Err(crate::engine::ResourceLedgerError::Exhausted(
3052 formualizer_common::ResourceExhaustionDetail {
3053 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3054 limit: 0,
3055 observed: 1,
3056 request_id: self
3057 .active_evaluation_resource_request
3058 .as_ref()
3059 .map(|request| request.request_id),
3060 },
3061 )
3062 .into_excel_error()
3063 .with_message(message));
3064 }
3065 Ok(())
3066 }
3067
3068 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3069 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3070 return Ok(());
3071 };
3072 ledger
3073 .charge_work(work_units)
3074 .and_then(|()| ledger.checkpoint_deadline())
3075 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3076 }
3077
3078 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3079 if work_units == 0 {
3080 return self.resource_checkpoint(0);
3081 }
3082 while work_units > 0 {
3083 let chunk = work_units.min(256);
3084 self.resource_checkpoint(chunk)?;
3085 work_units -= chunk;
3086 }
3087 Ok(())
3088 }
3089
3090 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3091 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3092 ledger.observe_scratch(bytes);
3095 }
3096 Ok(())
3097 }
3098
3099 fn release_request_scratch(&mut self, bytes: u64) {
3100 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3101 let released = ledger.release_scratch(bytes);
3102 debug_assert!(
3103 released.is_ok(),
3104 "request scratch release exceeded the outstanding reservation"
3105 );
3106 }
3107 }
3108
3109 fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3110 self.active_resource_ledger
3111 .as_ref()
3112 .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3113 }
3114
3115 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3116 self.active_resource_ledger
3117 .as_mut()
3118 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3119 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3120 }
3121
3122 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3123 self.active_resource_ledger
3124 .as_mut()
3125 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3126 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3127 }
3128
3129 fn with_request_scratch<T>(
3130 &mut self,
3131 bytes: u64,
3132 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3133 ) -> Result<T, ExcelError> {
3134 self.reserve_request_scratch(bytes)?;
3135 let result = work(self);
3136 self.release_request_scratch(bytes);
3137 result
3138 }
3139
3140 fn observe_staged_preparation(
3141 &mut self,
3142 selected: usize,
3143 retained: usize,
3144 elapsed: std::time::Duration,
3145 ) {
3146 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3147 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3148 stats.staged_retained = retained as u64;
3149 stats.phases.staged_prepare_ns = stats
3150 .phases
3151 .staged_prepare_ns
3152 .saturating_add(Self::duration_ns(elapsed));
3153 }
3154 }
3155
3156 fn graph_admission_enabled(&self) -> bool {
3157 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3158 }
3159
3160 fn preflight_graph_admission(
3161 &mut self,
3162 usage: crate::engine::resource_ledger::GraphAdmission,
3163 ) -> Result<(), ExcelError> {
3164 let request_id = self
3165 .active_evaluation_resource_request
3166 .as_ref()
3167 .map(|stats| stats.request_id);
3168 crate::engine::resource_ledger::preflight_graph_admission(
3169 &self.evaluation_resource_budgets,
3170 usage,
3171 request_id,
3172 )
3173 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3174 }
3175
3176 fn prepared_legacy_admission(
3177 &mut self,
3178 plan: &PreparedLegacyGraphPlan,
3179 materialization_cells: u64,
3180 ) -> Result<(), ExcelError> {
3181 if !self.graph_admission_enabled() {
3182 return Ok(());
3183 }
3184 let stats = self.graph.baseline_stats();
3185 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3186 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3187 })?;
3188 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3189 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3190 })?;
3191 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3192 final_vertices: stats
3193 .graph_vertex_count
3194 .checked_add(plan.new_vertex_count())
3195 .ok_or_else(|| {
3196 ExcelError::new(ExcelErrorKind::NImpl)
3197 .with_message("graph vertex count overflow")
3198 })?,
3199 final_edges: stats
3200 .graph_edge_count
3201 .checked_sub(removed_edges)
3202 .and_then(|count| count.checked_add(added_edges))
3203 .ok_or_else(|| {
3204 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3205 })?,
3206 materialization_cells,
3207 added_vertices: plan.new_vertex_count(),
3208 added_edges,
3209 })
3210 }
3211
3212 fn observe_target_admission_failure(
3213 &mut self,
3214 reason: formualizer_common::ResourceExhaustionReason,
3215 ) {
3216 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3217 stats.target_admission_failure = Some(reason);
3218 }
3219 }
3220
3221 fn observe_target_preparation_report(
3222 &mut self,
3223 report: &crate::engine::PreparedTargetGraphReport,
3224 ) {
3225 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3226 let index = match reason {
3227 crate::engine::OpaqueReason::DynamicReference => 0,
3228 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3229 crate::engine::OpaqueReason::UnknownFunction => 2,
3230 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3231 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3232 crate::engine::OpaqueReason::UnresolvedName => 5,
3233 crate::engine::OpaqueReason::UnresolvedTable => 6,
3234 crate::engine::OpaqueReason::FormulaName => 7,
3235 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3236 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3237 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3238 };
3239 1u64 << index
3240 };
3241 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3242 stats.staged_selected = report.selected_staged_cells as u64;
3243 stats.staged_retained = report.retained_staged_cells as u64;
3244 stats.target_requested = report.requested_targets as u64;
3245 stats.target_normalized_regions = report.normalized_regions as u64;
3246 stats.target_scope_level = match &report.widened_scope {
3247 crate::engine::PrepareScope::Exact => 0,
3248 crate::engine::PrepareScope::Sheets(_) => 1,
3249 crate::engine::PrepareScope::Workbook => 2,
3250 };
3251 stats.target_widening_reason_bits = report
3252 .widening_reasons
3253 .iter()
3254 .copied()
3255 .fold(0, |bits, reason| bits | reason_bit(reason));
3256 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3257 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3258 stats.target_commit_estimated_work = report.estimated_commit_work;
3259 stats.target_commit_actual_work = report.actual_commit_work;
3260 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3261 stats.phases.staged_prepare_ns = stats
3262 .phases
3263 .staged_prepare_ns
3264 .saturating_add(Self::duration_ns(report.commit_window));
3265 }
3266 }
3267
3268 fn observe_topology(
3269 &mut self,
3270 outcome: FormulaPlaneTopologyCacheOutcome,
3271 strategy: FormulaPlaneTopologyStrategy,
3272 observed: &MixedTopologyCompileStats,
3273 elapsed: std::time::Duration,
3274 ) {
3275 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3276 if outcome.severity() > stats.topology.cache_outcome.severity() {
3277 stats.topology.cache_outcome = outcome;
3278 }
3279 if strategy.severity() > stats.topology.strategy.severity() {
3280 stats.topology.strategy = strategy;
3281 }
3282 match outcome {
3283 FormulaPlaneTopologyCacheOutcome::Hit => {
3284 stats.topology.cache_hit_events =
3285 stats.topology.cache_hit_events.saturating_add(1);
3286 }
3287 FormulaPlaneTopologyCacheOutcome::Built => {
3288 stats.topology.cache_build_events =
3289 stats.topology.cache_build_events.saturating_add(1);
3290 }
3291 FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3292 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3293 stats.topology.cache_build_events =
3294 stats.topology.cache_build_events.saturating_add(1);
3295 stats.topology.cache_skip_events =
3296 stats.topology.cache_skip_events.saturating_add(1);
3297 }
3298 FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3299 }
3300 if matches!(
3301 outcome,
3302 FormulaPlaneTopologyCacheOutcome::Built
3303 | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3304 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3305 ) {
3306 stats.topology.producers_observed = stats
3307 .topology
3308 .producers_observed
3309 .saturating_add(observed.producers as u64);
3310 stats.topology.candidates_observed = stats
3311 .topology
3312 .candidates_observed
3313 .saturating_add(observed.candidates as u64);
3314 stats.topology.edges_observed = stats
3315 .topology
3316 .edges_observed
3317 .saturating_add(observed.relationships as u64);
3318 stats.topology.candidate_cap_hits = stats
3319 .topology
3320 .candidate_cap_hits
3321 .saturating_add(observed.candidate_overflow_count as u64);
3322 stats.topology.edge_cap_hits = stats
3323 .topology
3324 .edge_cap_hits
3325 .saturating_add(observed.edge_overflow_count as u64);
3326 stats.topology.byte_cap_hits = stats
3327 .topology
3328 .byte_cap_hits
3329 .saturating_add(observed.memory_overflow_count as u64);
3330 stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3331 Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3332 } else if stats.topology.edge_cap_hits > 0 {
3333 Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3334 } else if stats.topology.candidate_cap_hits > 0 {
3335 Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3336 } else {
3337 None
3338 };
3339 stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3340 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3341 } else if stats.topology.edge_cap_hits > 0 {
3342 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3343 } else if stats.topology.candidate_cap_hits > 0 {
3344 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3345 } else {
3346 None
3347 };
3348 }
3349 stats.topology.retained_bytes_observed = stats
3350 .topology
3351 .retained_bytes_observed
3352 .max(observed.estimated_memory_bytes as u64);
3353 stats.phases.topology_ns = stats
3354 .phases
3355 .topology_ns
3356 .saturating_add(Self::duration_ns(elapsed));
3357 }
3358 }
3359
3360 fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3361 if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3362 && strategy.severity() > stats.topology.strategy.severity()
3363 {
3364 stats.topology.strategy = strategy;
3365 }
3366 }
3367
3368 fn observe_formula_plane_route(
3369 &mut self,
3370 plan: &LegacyIslandPlan,
3371 phase: FormulaPlaneRoutePhase,
3372 reason: FormulaPlaneRouteTransitionReason,
3373 demotion_generation: usize,
3374 replan_generation: usize,
3375 ) {
3376 if plan.is_empty() {
3377 return;
3378 }
3379 let mut sheet_ids = [0_u16;
3380 crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3381 let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3382 sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3383 let authority = self.graph.formula_authority();
3384 let event = FormulaPlaneRouteEvent {
3385 island_id: plan.island_id,
3386 phase,
3387 route: FormulaPlaneRoute::ContractedLegacyIsland,
3388 sheet_ids,
3389 sheet_count: sheet_count as u8,
3390 authority_epoch: authority.plane.epoch().0,
3391 index_epoch: authority.indexes_epoch(),
3392 transition_reason: reason,
3393 demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3394 replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3395 };
3396 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3397 let index = usize::from(stats.topology.route_event_count);
3398 if index < stats.topology.route_events.len() {
3399 stats.topology.route_events[index] = event;
3400 stats.topology.route_event_count =
3401 stats.topology.route_event_count.saturating_add(1);
3402 } else {
3403 stats.topology.route_events_dropped =
3404 stats.topology.route_events_dropped.saturating_add(1);
3405 }
3406 stats.topology.route_event_bytes_observed = stats
3407 .topology
3408 .route_event_bytes_observed
3409 .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3410 stats.topology.island_membership_vertices = plan.membership.len() as u64;
3411 stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3412 stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3413 stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3414 }
3415 }
3416
3417 fn observe_materialization(
3418 &mut self,
3419 placements: usize,
3420 cycle: bool,
3421 elapsed: std::time::Duration,
3422 ) {
3423 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3424 if cycle {
3425 stats.cycle_materialized_cells = stats
3426 .cycle_materialized_cells
3427 .saturating_add(placements as u64);
3428 } else {
3429 stats.fallback_materialized_cells = stats
3430 .fallback_materialized_cells
3431 .saturating_add(placements as u64);
3432 stats.topology.strategy =
3433 FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3434 }
3435 stats.phases.materialization_ns = stats
3436 .phases
3437 .materialization_ns
3438 .saturating_add(Self::duration_ns(elapsed));
3439 }
3440 }
3441
3442 fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3443 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3444 stats.dirty_lease = if empty {
3445 FormulaDirtyLeaseOutcome::Empty
3446 } else {
3447 FormulaDirtyLeaseOutcome::Acquired
3448 };
3449 }
3450 }
3451
3452 fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3453 let empty = lease.is_empty();
3454 let _ = self.graph.ack_formula_dirty(lease);
3455 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3456 stats.dirty_lease = if empty {
3457 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3458 } else {
3459 FormulaDirtyLeaseOutcome::Acknowledged
3460 };
3461 }
3462 }
3463
3464 fn checked_ack_formula_dirty_observed(
3465 &mut self,
3466 lease: FormulaDirtyLease,
3467 ) -> Result<(), ExcelError> {
3468 self.resource_checkpoint(0)?;
3469 self.ack_formula_dirty_observed(lease);
3470 Ok(())
3471 }
3472
3473 fn checked_ack_formula_dirty_sublease_observed(
3474 &mut self,
3475 lease: FormulaDirtyLease,
3476 owned_events: &[usize],
3477 ) -> Result<(), ExcelError> {
3478 self.resource_checkpoint(owned_events.len() as u64)?;
3479 let empty = lease.is_empty();
3480 let partial = owned_events.len() < lease.len();
3481 let sublease = lease.sublease(owned_events.iter().copied());
3482 if !self.graph.ack_formula_dirty_sublease(sublease) {
3483 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3484 .with_message("FormulaPlane dirty sublease became stale"));
3485 }
3486 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3487 stats.dirty_lease = if empty {
3488 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3489 } else if partial {
3490 FormulaDirtyLeaseOutcome::AcknowledgedPartial
3491 } else {
3492 FormulaDirtyLeaseOutcome::Acknowledged
3493 };
3494 }
3495 Ok(())
3496 }
3497
3498 fn begin_evaluation_request(&mut self) {
3502 #[cfg(test)]
3503 {
3504 self.evaluation_request_begin_count_for_test = self
3505 .evaluation_request_begin_count_for_test
3506 .saturating_add(1);
3507 }
3508 self.last_cycle_telemetry = CycleTelemetry::default();
3509 self.pending_iterative_redirty.clear();
3512 self.clock.refresh();
3516 }
3517
3518 fn redirty_for_next_recalc(&mut self) {
3525 self.graph.redirty_volatiles();
3526 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3527 self.iterative_state_values.clear();
3533 for &vertex in &pending {
3534 if !self.graph.vertex_exists(vertex) {
3535 continue;
3536 }
3537 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3538 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3539 if let Some(value) =
3540 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
3541 && !matches!(value, LiteralValue::Empty)
3542 {
3543 self.iterative_state_values.insert(vertex, value);
3544 }
3545 }
3546 }
3547 if !pending.is_empty() {
3548 self.graph.redirty_iterative_members(&pending);
3549 }
3550 }
3551
3552 pub fn virtual_dep_fallback_activations(&self) -> u64 {
3553 self.virtual_dep_fallback_activations
3554 }
3555
3556 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3557 self.lookup_index_cache.report()
3558 }
3559
3560 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3561 let start_row = view.start_row();
3562 let end_row = view.end_row();
3563 let start_col = view.start_col();
3564 let end_col = view.end_col();
3565 for row in start_row..=end_row {
3566 let Ok(row_u32) = u32::try_from(row) else {
3567 return true;
3568 };
3569 for col in start_col..=end_col {
3570 let Ok(col_u32) = u32::try_from(col) else {
3571 return true;
3572 };
3573 let cell_ref = self
3574 .graph
3575 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3576 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3577 && self.graph.is_volatile(*vertex_id)
3578 {
3579 return true;
3580 }
3581 }
3582 }
3583 false
3584 }
3585
3586 fn build_lookup_index_impl(
3587 &self,
3588 view: &RangeView<'_>,
3589 axis: LookupAxis,
3590 ) -> Option<Arc<LookupIndex>> {
3591 let (rows, cols) = view.dims();
3592 if rows == 0 || cols == 0 {
3593 self.lookup_index_cache.note_skipped_tiny();
3594 return None;
3595 }
3596 let len = match axis {
3597 LookupAxis::ColumnInView(col) => {
3598 if col >= cols {
3599 self.lookup_index_cache.note_skipped_tiny();
3600 return None;
3601 }
3602 rows
3603 }
3604 LookupAxis::RowInView(row) => {
3605 if row >= rows {
3606 self.lookup_index_cache.note_skipped_tiny();
3607 return None;
3608 }
3609 cols
3610 }
3611 };
3612 if len < 64 {
3613 self.lookup_index_cache.note_skipped_tiny();
3614 return None;
3615 }
3616
3617 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3618 let key = LookupIndexKey {
3619 sheet_id,
3620 start_row: u32::try_from(view.start_row()).ok()?,
3621 start_col: u32::try_from(view.start_col()).ok()?,
3622 end_row: u32::try_from(view.end_row()).ok()?,
3623 end_col: u32::try_from(view.end_col()).ok()?,
3624 axis,
3625 snapshot_id: self.data_snapshot_id(),
3626 };
3627 if let Some(index) = self.lookup_index_cache.get(&key) {
3628 return Some(index);
3629 }
3630 if self
3631 .lookup_index_cache
3632 .would_exceed_cap(estimate_bytes(len, 0))
3633 {
3634 self.lookup_index_cache.note_skipped_cap();
3635 return None;
3636 }
3637 if !self.lookup_index_cache.should_build(key) {
3638 return None;
3639 }
3640 if self.lookup_index_cache.is_known_volatile(&key) {
3641 self.lookup_index_cache.note_skipped_volatile();
3642 return None;
3643 }
3644 if self.lookup_view_contains_volatile(view, sheet_id) {
3645 self.lookup_index_cache.note_volatile_key(key);
3646 self.lookup_index_cache.note_skipped_volatile();
3647 return None;
3648 }
3649 match LookupIndex::build(view, axis).ok()? {
3650 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3651 BuildOutcome::ErrorInLookupAxis => {
3652 self.lookup_index_cache.note_skipped_error();
3653 None
3654 }
3655 BuildOutcome::Degenerate => {
3656 self.lookup_index_cache.note_skipped_tiny();
3657 None
3658 }
3659 }
3660 }
3661
3662 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
3663 if !self.config.enable_virtual_dep_telemetry {
3664 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
3665 fallback_mode_activations: self.virtual_dep_fallback_activations,
3666 ..VirtualDepTelemetry::default()
3667 };
3668 }
3669 }
3670
3671 fn source_cache_session(&self) -> SourceCacheSession {
3672 self.clear_source_cache();
3673 SourceCacheSession {
3674 cache: self.source_cache.clone(),
3675 }
3676 }
3677
3678 fn resolve_source_scalar_cached(
3679 &self,
3680 name: &str,
3681 version: Option<u64>,
3682 ) -> Result<LiteralValue, ExcelError> {
3683 let key = (name.to_string(), version);
3684 if let Ok(mut g) = self.source_cache.write() {
3685 if let Some(v) = g.scalars.get(&key) {
3686 return Ok(v.clone());
3687 }
3688
3689 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
3690 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3691 ExcelError::new(ExcelErrorKind::Ref)
3692 .with_message(format!("Unresolved source scalar: {name}"))
3693 } else {
3694 err
3695 }
3696 })?;
3697 g.scalars.insert(key, v.clone());
3698 Ok(v)
3699 } else {
3700 self.resolver.resolve_source_scalar(name).map_err(|err| {
3701 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3702 ExcelError::new(ExcelErrorKind::Ref)
3703 .with_message(format!("Unresolved source scalar: {name}"))
3704 } else {
3705 err
3706 }
3707 })
3708 }
3709 }
3710
3711 fn resolve_source_table_cached(
3712 &self,
3713 name: &str,
3714 version: Option<u64>,
3715 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
3716 let key = (name.to_string(), version);
3717 if let Ok(mut g) = self.source_cache.write() {
3718 if let Some(t) = g.tables.get(&key) {
3719 return Ok(t.clone());
3720 }
3721
3722 let t = self.resolver.resolve_source_table(name).map_err(|err| {
3723 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3724 ExcelError::new(ExcelErrorKind::Ref)
3725 .with_message(format!("Unresolved source table: {name}"))
3726 } else {
3727 err
3728 }
3729 })?;
3730 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
3731 g.tables.insert(key, t.clone());
3732 Ok(t)
3733 } else {
3734 self.resolver
3735 .resolve_source_table(name)
3736 .map_err(|err| {
3737 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3738 ExcelError::new(ExcelErrorKind::Ref)
3739 .with_message(format!("Unresolved source table: {name}"))
3740 } else {
3741 err
3742 }
3743 })
3744 .map(Arc::from)
3745 }
3746 }
3747
3748 fn source_table_to_range_view(
3749 &self,
3750 table: &dyn crate::traits::Table,
3751 spec: &Option<formualizer_parse::parser::TableSpecifier>,
3752 ) -> Result<RangeView<'static>, ExcelError> {
3753 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
3754
3755 let owned = match spec {
3756 Some(TableSpecifier::Column(c)) => {
3757 let c = c.trim();
3758 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
3759 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3760 "Complex structured references not yet supported".to_string(),
3761 ));
3762 }
3763 table.get_column(c)?.materialise().into_owned()
3764 }
3765 Some(TableSpecifier::ColumnRange(start, end)) => {
3766 let cols = table.columns();
3767 let start = start.trim();
3768 let end = end.trim();
3769 let start_key = start.to_lowercase();
3770 let end_key = end.to_lowercase();
3771 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
3772 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
3773 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
3774 if si > ei {
3775 std::mem::swap(&mut si, &mut ei);
3776 }
3777 let h = table.data_height();
3778 let w = ei - si + 1;
3779 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
3780 for (offset, ci) in (si..=ei).enumerate() {
3781 let cname = &cols[ci];
3782 let col_range = table.get_column(cname)?;
3783 let (rh, _) = col_range.dimensions();
3784 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
3785 row[offset] = col_range.get(r, 0)?;
3786 }
3787 }
3788 rows
3789 } else {
3790 return Err(ExcelError::new(ExcelErrorKind::Ref)
3791 .with_message("Column range refers to unknown column(s)".to_string()));
3792 }
3793 }
3794 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
3795 | Some(TableSpecifier::Headers) => table
3796 .headers_row()
3797 .map(|r| r.materialise().into_owned())
3798 .unwrap_or_default(),
3799 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
3800 | Some(TableSpecifier::Totals) => table
3801 .totals_row()
3802 .map(|r| r.materialise().into_owned())
3803 .unwrap_or_default(),
3804 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
3805 table
3806 .data_body()
3807 .map(|r| r.materialise().into_owned())
3808 .unwrap_or_default()
3809 }
3810 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
3811 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
3812 if let Some(h) = table.headers_row() {
3813 out.extend(h.iter_rows());
3814 }
3815 if let Some(body) = table.data_body() {
3816 out.extend(body.iter_rows());
3817 }
3818 if let Some(tr) = table.totals_row() {
3819 out.extend(tr.iter_rows());
3820 }
3821 out
3822 }
3823 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
3824 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3825 "@ (This Row) requires table-aware context; not yet supported".to_string(),
3826 ));
3827 }
3828 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
3829 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3830 .with_message("Complex structured references not yet supported".to_string()));
3831 }
3832 None => {
3833 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3834 .with_message("Table reference without specifier is unsupported".to_string()));
3835 }
3836 };
3837
3838 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
3839 }
3840
3841 pub fn default_sheet_id(&self) -> SheetId {
3842 self.graph.default_sheet_id()
3843 }
3844
3845 pub fn default_sheet_name(&self) -> &str {
3846 self.graph.default_sheet_name()
3847 }
3848
3849 pub fn set_workbook_seed(&mut self, seed: u64) {
3851 self.config.workbook_seed = seed;
3852 }
3853
3854 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
3856 self.config.volatile_level = level;
3857 }
3858
3859 pub fn set_deterministic_mode(
3861 &mut self,
3862 mode: crate::engine::DeterministicMode,
3863 ) -> Result<(), ExcelError> {
3864 let clock = mode.build_clock()?;
3865 self.config.deterministic_mode = mode;
3866 self.clock = crate::timezone::SnapshotClock::new(clock);
3867 Ok(())
3868 }
3869
3870 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
3878 self.clock = crate::timezone::SnapshotClock::new(clock);
3879 }
3880
3881 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
3882 self.config.deterministic_mode.validate()
3883 }
3884
3885 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
3886 self.graph.sheet_id(name)
3887 }
3888
3889 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
3890 self.add_sheet(name)
3891 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
3892 }
3893
3894 pub fn sheet_name(&self, id: SheetId) -> &str {
3895 self.graph.sheet_name(id)
3896 }
3897
3898 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
3899 let id = self.graph.add_sheet(name)?;
3900 self.ensure_arrow_sheet(name);
3901 self.mark_topology_edited();
3903 Ok(id)
3904 }
3905
3906 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
3907 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
3908 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
3909 })?;
3910 if new_name.is_empty() || new_name.len() > 255 {
3911 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
3912 }
3913 if self.graph.sheet_id(new_name).is_some() {
3914 return Err(ExcelError::new(ExcelErrorKind::Value)
3915 .with_message(format!("Sheet '{new_name}' already exists")));
3916 }
3917 self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
3920 .map_err(Self::editor_error_to_excel)?;
3921 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
3922
3923 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
3924 let mut copied_sheet = source_sheet;
3925 copied_sheet.name = Arc::<str>::from(new_name);
3926 self.arrow_sheets.sheets.push(copied_sheet);
3927 } else {
3928 self.ensure_arrow_sheet(new_name);
3929 }
3930
3931 let duplicated_formulas = self
3932 .graph
3933 .formula_vertices()
3934 .into_iter()
3935 .filter(|vertex| {
3936 self.graph
3937 .get_cell_ref(*vertex)
3938 .is_some_and(|cell| cell.sheet_id == new_id)
3939 })
3940 .collect::<Vec<_>>();
3941 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
3942 self.mark_topology_edited();
3943 Ok(new_id)
3944 }
3945
3946 fn ensure_arrow_sheet(&mut self, name: &str) {
3947 if self.arrow_sheets.sheet(name).is_some() {
3948 return;
3949 }
3950 self.arrow_sheets
3951 .sheets
3952 .push(crate::arrow_store::ArrowSheet {
3953 name: std::sync::Arc::<str>::from(name),
3954 date_system: self.config.date_system,
3955 columns: Vec::new(),
3956 nrows: 0,
3957 chunk_starts: Vec::new(),
3958 chunk_rows: 32 * 1024,
3959 });
3960 }
3961
3962 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
3963 let name = self.graph.sheet_name(sheet_id).to_string();
3964 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
3968 .map_err(Self::editor_error_to_excel)?;
3969 self.graph.remove_sheet(sheet_id)?;
3970 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
3971 self.clear_all_computed_overlays();
3975 self.mark_all_formula_vertices_dirty();
3976 self.clear_staged_formulas_for_sheet(&name);
3977 if self.row_visibility.remove(&sheet_id).is_some() {
3978 self.invalidate_row_visibility_mask_cache();
3979 }
3980 self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
3981 self.mark_topology_edited();
3982 Ok(())
3983 }
3984
3985 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
3987 if let Some(asheet) = self
3988 .arrow_sheets
3989 .sheets
3990 .iter_mut()
3991 .find(|s| s.name.as_ref() == target_name)
3992 {
3993 asheet.name = std::sync::Arc::<str>::from(new_name);
3994 return true;
3995 }
3996 false
3997 }
3998
3999 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4000 let old_name = self.graph.sheet_name(sheet_id).to_string();
4001
4002 self.rename_sheet_in_arrow_store(&old_name, new_name);
4005
4006 match self.graph.rename_sheet(sheet_id, new_name) {
4008 Ok(_) => {
4009 self.rename_staged_formula_sheet(&old_name, new_name);
4010 let sheet_vertices: Vec<VertexId> =
4012 self.graph.vertices_in_sheet(sheet_id).collect();
4013 for v_id in sheet_vertices {
4014 self.graph.mark_vertex_dirty(v_id);
4015 }
4016 self.mark_topology_edited();
4018 Ok(())
4019 }
4020 Err(e) => {
4021 self.rename_sheet_in_arrow_store(new_name, &old_name);
4023 Err(e)
4024 }
4025 }
4026 }
4027
4028 pub fn named_ranges_iter(
4029 &self,
4030 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4031 self.graph.named_ranges_iter()
4032 }
4033
4034 pub fn sheet_named_ranges_iter(
4035 &self,
4036 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4037 self.graph.sheet_named_ranges_iter()
4038 }
4039
4040 pub fn resolve_name_entry(
4041 &self,
4042 name: &str,
4043 current_sheet: SheetId,
4044 ) -> Option<&crate::engine::named_range::NamedRange> {
4045 self.graph.resolve_name_entry(name, current_sheet)
4046 }
4047
4048 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4049 let current_sheet = scope_sheet
4050 .and_then(|sheet| self.graph.sheet_id(sheet))
4051 .unwrap_or_else(|| self.graph.default_sheet_id());
4052 self.graph.resolve_name_entry(name, current_sheet).is_some()
4053 }
4054
4055 pub fn resolved_name_value(
4056 &self,
4057 name: &str,
4058 scope_sheet: Option<&str>,
4059 ) -> Option<LiteralValue> {
4060 let current_sheet = scope_sheet
4061 .and_then(|sheet| self.graph.sheet_id(sheet))
4062 .unwrap_or_else(|| self.graph.default_sheet_id());
4063 let entry = self.graph.resolve_name_entry(name, current_sheet)?;
4064 self.graph.get_value(entry.vertex)
4065 }
4066
4067 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4068 let entry = self.graph.resolve_table_entry(name)?;
4069 Some(TableMetadata {
4070 name: entry.name.clone(),
4071 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4072 start_row: entry.range.start.coord.row() + 1,
4073 start_col: entry.range.start.coord.col() + 1,
4074 end_row: entry.range.end.coord.row() + 1,
4075 end_col: entry.range.end.coord.col() + 1,
4076 header_row: entry.header_row,
4077 headers: entry.headers.clone(),
4078 totals_row: entry.totals_row,
4079 })
4080 }
4081
4082 pub fn tables(&self) -> Vec<TableMetadata> {
4084 self.graph
4085 .table_names()
4086 .into_iter()
4087 .filter_map(|name| self.table_metadata(&name))
4088 .collect()
4089 }
4090
4091 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4092 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4093
4094 for (name, named) in self.graph.named_ranges_iter() {
4095 out.push(crate::engine::named_range::NamedRangeSnapshot {
4096 name: name.clone(),
4097 scope: NameScope::Workbook,
4098 definition: named.definition.clone(),
4099 });
4100 }
4101
4102 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4103 out.push(crate::engine::named_range::NamedRangeSnapshot {
4104 name: name.clone(),
4105 scope: NameScope::Sheet(*sheet_id),
4106 definition: named.definition.clone(),
4107 });
4108 }
4109
4110 out.sort_by(|a, b| {
4111 let a_scope = match a.scope {
4112 NameScope::Workbook => (0u8, 0u32),
4113 NameScope::Sheet(id) => (1u8, u32::from(id)),
4114 };
4115 let b_scope = match b.scope {
4116 NameScope::Workbook => (0u8, 0u32),
4117 NameScope::Sheet(id) => (1u8, u32::from(id)),
4118 };
4119 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4120 });
4121
4122 out
4123 }
4124
4125 pub fn named_ranges_snapshot_for_sheet(
4126 &self,
4127 sheet_id: SheetId,
4128 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4129 self.named_ranges_snapshot()
4130 .into_iter()
4131 .filter(|entry| match entry.scope {
4132 NameScope::Workbook => true,
4133 NameScope::Sheet(id) => id == sheet_id,
4134 })
4135 .collect()
4136 }
4137
4138 pub fn define_name(
4139 &mut self,
4140 name: &str,
4141 definition: NamedDefinition,
4142 scope: NameScope,
4143 ) -> Result<(), ExcelError> {
4144 self.graph.validate_define_name(name, scope)?;
4150 let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4151 if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4152 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4153 "formula-name shadowing with active FormulaPlane dependents is not supported",
4154 ));
4155 }
4156 let prepared = self
4157 .prepare_name_dependent_span_demotion([name])
4158 .map_err(Self::editor_error_to_excel)?;
4159 let demoted = self
4160 .commit_name_dependent_span_demotion(prepared)
4161 .map_err(Self::editor_error_to_excel)?;
4162 self.graph.define_name(name, definition, scope)?;
4163 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4164 if !demoted {
4165 self.mark_topology_edited();
4166 }
4167 Ok(())
4168 }
4169
4170 pub fn update_name(
4171 &mut self,
4172 name: &str,
4173 definition: NamedDefinition,
4174 scope: NameScope,
4175 ) -> Result<(), ExcelError> {
4176 self.graph.validate_existing_name(name, scope)?;
4182 let prepared = self
4183 .prepare_name_dependent_span_demotion([name])
4184 .map_err(Self::editor_error_to_excel)?;
4185 let demoted = self
4186 .commit_name_dependent_span_demotion(prepared)
4187 .map_err(Self::editor_error_to_excel)?;
4188 self.graph.update_name(name, definition, scope)?;
4189 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4190 if !demoted {
4191 self.mark_topology_edited();
4192 }
4193 Ok(())
4194 }
4195
4196 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4197 self.graph.validate_existing_name(name, scope)?;
4201 let prepared = self
4202 .prepare_name_dependent_span_demotion([name])
4203 .map_err(Self::editor_error_to_excel)?;
4204 let demoted = self
4205 .commit_name_dependent_span_demotion(prepared)
4206 .map_err(Self::editor_error_to_excel)?;
4207 self.graph.delete_name(name, scope)?;
4208 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4209 if !demoted {
4210 self.mark_topology_edited();
4211 }
4212 Ok(())
4213 }
4214
4215 fn exact_name_dependent_span_refs<'a>(
4216 &self,
4217 names: impl IntoIterator<Item = &'a str>,
4218 ) -> Vec<FormulaSpanRef> {
4219 let authority = self.graph.formula_authority();
4220 let mut refs = names
4221 .into_iter()
4222 .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4223 .collect::<Vec<_>>();
4224 refs.sort_unstable_by_key(|span_ref| {
4225 (span_ref.id.0, span_ref.generation, span_ref.version)
4226 });
4227 refs.dedup();
4228 refs
4229 }
4230
4231 fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4232 events
4233 .iter()
4234 .filter_map(|event| match event {
4235 ChangeEvent::DefineName { name, .. }
4236 | ChangeEvent::UpdateName { name, .. }
4237 | ChangeEvent::DeleteName { name, .. }
4238 | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4239 _ => None,
4240 })
4241 .collect()
4242 }
4243
4244 fn prepare_name_dependent_span_demotion<'a>(
4245 &mut self,
4246 names: impl IntoIterator<Item = &'a str>,
4247 ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4248 {
4249 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4250 return Ok(None);
4251 }
4252 let refs = self.exact_name_dependent_span_refs(names);
4253 if refs.is_empty() {
4254 return Ok(None);
4255 }
4256
4257 let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4261 let clean_cells = {
4262 let authority = self.graph.formula_authority();
4263 refs.iter()
4264 .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4265 .flat_map(|span| span.domain.iter())
4266 .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4267 .map(|coord| {
4268 CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4269 })
4270 .collect::<Vec<_>>()
4271 };
4272 self.prepare_formula_span_demotion(&refs)
4273 .map(|prepared| Some((prepared, clean_cells)))
4274 .map_err(|error| match error {
4275 FormulaSpanDemotionError::Resource(error) => error.into(),
4276 error => crate::engine::EditorError::TransactionFailed {
4277 reason: format!(
4278 "FormulaPlane name-dependent demotion preparation failed: {error}"
4279 ),
4280 },
4281 })
4282 }
4283
4284 fn commit_name_dependent_span_demotion(
4285 &mut self,
4286 prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4287 ) -> Result<bool, crate::engine::EditorError> {
4288 let Some((prepared, clean_cells)) = prepared else {
4289 return Ok(false);
4290 };
4291 self.commit_prepared_formula_span_demotion(prepared)
4292 .map_err(|error| crate::engine::EditorError::TransactionFailed {
4293 reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4294 })?;
4295 for cell in clean_cells {
4296 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4297 self.graph.set_dirty(vertex_id, false);
4298 }
4299 }
4300 Ok(true)
4301 }
4302
4303 pub fn define_table(
4304 &mut self,
4305 name: &str,
4306 range: crate::reference::RangeRef,
4307 header_row: bool,
4308 headers: Vec<String>,
4309 totals_row: bool,
4310 ) -> Result<(), ExcelError> {
4311 self.graph
4312 .define_table(name, range, header_row, headers, totals_row)?;
4313 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4314 self.mark_topology_edited();
4315 Ok(())
4316 }
4317
4318 pub fn define_source_scalar(
4319 &mut self,
4320 name: &str,
4321 version: Option<u64>,
4322 ) -> Result<(), ExcelError> {
4323 self.graph.define_source_scalar(name, version)?;
4324 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4325 self.mark_topology_edited();
4326 Ok(())
4327 }
4328
4329 pub fn define_source_table(
4330 &mut self,
4331 name: &str,
4332 version: Option<u64>,
4333 ) -> Result<(), ExcelError> {
4334 self.graph.define_source_table(name, version)?;
4335 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4336 self.mark_topology_edited();
4337 Ok(())
4338 }
4339
4340 pub fn set_source_scalar_version(
4341 &mut self,
4342 name: &str,
4343 version: Option<u64>,
4344 ) -> Result<(), ExcelError> {
4345 self.graph.set_source_scalar_version(name, version)?;
4346 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4347 self.graph
4348 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4349 }
4350 Ok(())
4351 }
4352
4353 pub fn set_source_table_version(
4354 &mut self,
4355 name: &str,
4356 version: Option<u64>,
4357 ) -> Result<(), ExcelError> {
4358 self.graph.set_source_table_version(name, version)?;
4359 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4360 self.graph
4361 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4362 }
4363 Ok(())
4364 }
4365
4366 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4367 self.graph.invalidate_source(name)?;
4368 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4369 self.graph
4370 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4371 }
4372 Ok(())
4373 }
4374
4375 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4376 self.graph.get_value(vertex)
4377 }
4378
4379 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4380 self.graph.get_cell_value(sheet, row, col)
4381 }
4382
4383 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4384 self.graph.get_vertex_for_cell(cell)
4385 }
4386
4387 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4388 self.graph.get_evaluation_vertices()
4389 }
4390
4391 pub fn baseline_stats(&self) -> EngineBaselineStats {
4393 let graph = self.graph.baseline_stats();
4394 let formula_authority = self.graph.formula_authority();
4395 let formula_dirty = self.graph.formula_dirty_stats();
4396 EngineBaselineStats {
4397 graph_vertex_count: graph.graph_vertex_count,
4398 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4399 graph_edge_count: graph.graph_edge_count,
4400 dirty_vertex_count: graph.dirty_vertex_count,
4401 evaluation_vertex_count: graph.evaluation_vertex_count,
4402 formula_ast_root_count: graph.formula_ast_root_count,
4403 formula_ast_node_count: graph.formula_ast_node_count,
4404 staged_formula_count: self.staged_formula_count(),
4405 formula_plane_active_span_count: formula_authority.active_span_count(),
4406 formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4407 formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4408 formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4409 formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4410 formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4411 formula_plane_dirty_pending_events: formula_dirty.pending_events,
4412 formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4413 formula_plane_dirty_span_region_events_recorded: formula_dirty
4414 .span_region_events_recorded,
4415 formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4416 formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4417 formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4418 formula_plane_cycle_member_span_demotions: self
4419 .formula_plane_cycle_member_span_demotions,
4420 }
4421 }
4422
4423 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4425 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4426 }
4427
4428 #[cfg(test)]
4429 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4430 self.used_axis_bounds_cache
4431 .read()
4432 .ok()
4433 .and_then(|guard| {
4434 guard.as_ref().map(|cache| {
4435 (
4436 cache.row_hits.load(Ordering::Relaxed),
4437 cache.row_misses.load(Ordering::Relaxed),
4438 cache.col_hits.load(Ordering::Relaxed),
4439 cache.col_misses.load(Ordering::Relaxed),
4440 )
4441 })
4442 })
4443 .unwrap_or((0, 0, 0, 0))
4444 }
4445
4446 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4447 self.graph.set_first_load_assume_new(enabled);
4448 }
4449
4450 pub fn first_load_assume_new(&self) -> bool {
4451 self.graph.first_load_assume_new()
4452 }
4453
4454 pub fn reset_ensure_touched(&mut self) {
4455 self.graph.reset_ensure_touched();
4456 }
4457
4458 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4459 self.graph.finalize_sheet_index(sheet);
4460 }
4461
4462 pub fn action<T>(
4471 &mut self,
4472 name: impl AsRef<str>,
4473 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4474 ) -> Result<T, crate::engine::EditorError> {
4475 if self.action_depth != 0 {
4476 return Err(crate::engine::EditorError::TransactionFailed {
4477 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4478 .to_string(),
4479 });
4480 }
4481
4482 self.action_depth = 1;
4483 let engine_ptr: *mut Engine<R> = self;
4484 let _guard = ActionDepthGuard {
4485 engine: engine_ptr,
4486 _marker: std::marker::PhantomData,
4487 };
4488
4489 let mut tx = EngineAction {
4490 engine: self,
4491 name: name.as_ref().to_string(),
4492 log: None,
4493 arrow_undo: None,
4494 atomic_policy: false,
4495 };
4496 f(&mut tx)
4497 }
4498
4499 pub fn action_atomic<T>(
4503 &mut self,
4504 name: impl Into<String>,
4505 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4506 ) -> Result<T, crate::engine::EditorError> {
4507 let (v, _j) = self.action_atomic_journal(name, f)?;
4508 Ok(v)
4509 }
4510
4511 pub fn action_atomic_journal<T>(
4513 &mut self,
4514 name: impl Into<String>,
4515 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4516 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4517 if self.action_depth != 0 {
4518 return Err(crate::engine::EditorError::TransactionFailed {
4519 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4520 .to_string(),
4521 });
4522 }
4523
4524 self.action_depth = 1;
4525 let engine_ptr: *mut Engine<R> = self;
4526 let _guard = ActionDepthGuard {
4527 engine: engine_ptr,
4528 _marker: std::marker::PhantomData,
4529 };
4530
4531 let name_str = name.into();
4532 let mut log = crate::engine::ChangeLog::new();
4533 let start_len = log.len();
4534 self.action_atomic_impl(&mut log, start_len, name_str, f)
4535 }
4536
4537 fn action_atomic_impl<T>(
4538 &mut self,
4539 log: &mut crate::engine::ChangeLog,
4540 start_len: usize,
4541 name: String,
4542 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4543 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4544 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4545 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4546
4547 let log_ptr: *mut crate::engine::ChangeLog = log;
4548 let mut tx = EngineAction {
4549 engine: self,
4550 name: name.clone(),
4551 log: Some(log_ptr),
4552 arrow_undo: Some(arrow_ptr),
4553 atomic_policy: true,
4554 };
4555
4556 let res = f(&mut tx);
4557
4558 let graph_events: Vec<crate::engine::ChangeEvent> =
4560 unsafe { (&*log_ptr).events() }[start_len..].to_vec();
4561 let graph_batch = crate::engine::GraphUndoBatch {
4562 events: graph_events,
4563 };
4564 let affected_cells = arrow_undo.ops.len();
4565 let journal = crate::engine::ActionJournal {
4566 name,
4567 graph: graph_batch,
4568 arrow: arrow_undo,
4569 affected_cells,
4570 };
4571
4572 match res {
4573 Ok(v) => {
4574 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4575 for event in &journal.graph.events {
4576 self.record_formula_plane_change_for_event(event);
4577 }
4578 self.mark_data_edited();
4579 }
4580 Ok((v, journal))
4581 }
4582 Err(e) => {
4583 if let Err(rb) = self.rollback_from_action_journal(&journal) {
4584 return Err(crate::engine::EditorError::TransactionFailed {
4585 reason: format!(
4586 "Engine::action_atomic rollback failed after error '{e}': {rb}"
4587 ),
4588 });
4589 }
4590 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4591 for event in &journal.graph.events {
4592 self.record_formula_plane_change_for_event(event);
4593 }
4594 }
4595 Err(e)
4596 }
4597 }
4598 }
4599
4600 pub fn action_with_logger<T>(
4607 &mut self,
4608 log: &mut crate::engine::ChangeLog,
4609 name: impl AsRef<str>,
4610 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4611 ) -> Result<T, crate::engine::EditorError> {
4612 if self.action_depth != 0 {
4613 return Err(crate::engine::EditorError::TransactionFailed {
4614 reason: "Nested Engine::action calls are not supported (deterministic rule)"
4615 .to_string(),
4616 });
4617 }
4618
4619 self.action_depth = 1;
4620 let engine_ptr: *mut Engine<R> = self;
4621 let _guard = ActionDepthGuard {
4622 engine: engine_ptr,
4623 _marker: std::marker::PhantomData,
4624 };
4625
4626 let start_len = log.len();
4627 let name_str = name.as_ref().to_string();
4628 log.begin_compound(name_str.clone());
4629
4630 let res = self.action_atomic_impl(log, start_len, name_str, f);
4633
4634 match res {
4635 Ok((v, _journal)) => {
4636 log.end_compound();
4637 Ok(v)
4638 }
4639 Err(e) => {
4640 log.end_compound();
4642 log.truncate(start_len);
4643 Err(e)
4644 }
4645 }
4646 }
4647
4648 fn rollback_from_action_journal(
4649 &mut self,
4650 journal: &crate::engine::ActionJournal,
4651 ) -> Result<(), crate::engine::EditorError> {
4652 journal.graph.undo(&mut self.graph)?;
4654 self.apply_inverse_row_visibility_events(&journal.graph.events);
4656 self.apply_arrow_undo_batch(&journal.arrow, true);
4658 Ok(())
4659 }
4660
4661 fn rollback_from_change_events(
4662 &mut self,
4663 events: &[crate::engine::ChangeEvent],
4664 ) -> Result<(), crate::engine::EditorError> {
4665 use crate::engine::ChangeEvent;
4666
4667 {
4669 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4670 let mut compound_stack: Vec<usize> = Vec::new();
4671 for ev in events.iter().rev() {
4672 match ev {
4673 ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
4674 ChangeEvent::CompoundStart { depth, .. } => {
4675 if compound_stack.last() == Some(depth) {
4676 compound_stack.pop();
4677 }
4678 }
4679 ChangeEvent::SetRowVisibility { .. } => {
4680 }
4682 _ => {
4683 editor.apply_inverse(ev.clone())?;
4684 }
4685 }
4686 }
4687 }
4688
4689 for ev in events.iter().rev() {
4691 self.apply_inverse_row_visibility_event(ev);
4692 }
4693
4694 for ev in events.iter().rev() {
4696 self.mirror_inverse_change_to_arrow(ev);
4697 }
4698
4699 Ok(())
4700 }
4701
4702 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
4703 let sheet_id = self.graph.sheet_id(sheet)?;
4704 let coord = Coord::from_excel(row, col, true, true);
4705 let cell = CellRef::new(sheet_id, coord);
4706 let vid = self.graph.get_vertex_for_cell(&cell)?;
4707 let ast_id = self.graph.get_formula_id(vid)?;
4708 self.graph
4709 .data_store()
4710 .retrieve_ast(ast_id, self.graph.sheet_reg())
4711 }
4712
4713 pub fn define_name_with_logger(
4714 &mut self,
4715 log: &mut crate::engine::ChangeLog,
4716 name: &str,
4717 definition: NamedDefinition,
4718 scope: NameScope,
4719 ) -> Result<(), crate::engine::EditorError> {
4720 self.graph
4721 .validate_define_name(name, scope)
4722 .map_err(crate::engine::EditorError::Excel)?;
4723 let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4724 if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4725 return Err(crate::engine::EditorError::TransactionUnsupported {
4726 reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
4727 .to_string(),
4728 });
4729 }
4730 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4731 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4732 {
4733 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4734 editor.define_name(name, definition, scope)?;
4735 }
4736 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4737 if !demoted {
4738 self.mark_topology_edited();
4739 }
4740 Ok(())
4741 }
4742
4743 pub fn update_name_with_logger(
4744 &mut self,
4745 log: &mut crate::engine::ChangeLog,
4746 name: &str,
4747 definition: NamedDefinition,
4748 scope: NameScope,
4749 ) -> Result<(), crate::engine::EditorError> {
4750 self.graph
4751 .validate_existing_name(name, scope)
4752 .map_err(crate::engine::EditorError::Excel)?;
4753 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4754 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4755 {
4756 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4757 editor.update_name(name, definition, scope)?;
4758 }
4759 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4760 if !demoted {
4761 self.mark_topology_edited();
4762 }
4763 Ok(())
4764 }
4765
4766 pub fn delete_name_with_logger(
4767 &mut self,
4768 log: &mut crate::engine::ChangeLog,
4769 name: &str,
4770 scope: NameScope,
4771 ) -> Result<(), crate::engine::EditorError> {
4772 self.graph
4773 .validate_existing_name(name, scope)
4774 .map_err(crate::engine::EditorError::Excel)?;
4775 let prepared = self.prepare_name_dependent_span_demotion([name])?;
4776 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4777 {
4778 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4779 editor.delete_name(name, scope)?;
4780 }
4781 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4782 if !demoted {
4783 self.mark_topology_edited();
4784 }
4785 Ok(())
4786 }
4787
4788 pub fn edit_with_logger<T>(
4789 &mut self,
4790 log: &mut crate::engine::ChangeLog,
4791 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
4792 ) -> Result<T, crate::engine::EditorError> {
4793 let start_len = log.len();
4795
4796 struct ArrowSpillReader<'a> {
4799 sheets: &'a crate::arrow_store::SheetStore,
4800 }
4801 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
4802 fn read_cell_value(
4803 &self,
4804 sheet: &str,
4805 row: u32,
4806 col: u32,
4807 ) -> Option<formualizer_common::LiteralValue> {
4808 use formualizer_common::LiteralValue;
4809 let asheet = self.sheets.sheet(sheet)?;
4810 let r0 = row.saturating_sub(1) as usize;
4811 let c0 = col.saturating_sub(1) as usize;
4812 let v = asheet.get_cell_value(r0, c0);
4813 if matches!(v, LiteralValue::Empty) {
4814 None
4815 } else {
4816 Some(v)
4817 }
4818 }
4819 }
4820
4821 let ret = {
4822 let spill_reader = ArrowSpillReader {
4823 sheets: &self.arrow_sheets,
4824 };
4825 let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
4826 &mut self.graph,
4827 log,
4828 &spill_reader,
4829 );
4830 f(&mut editor)
4831 };
4832
4833 let new_events = log.events()[start_len..].to_vec();
4834 if new_events.iter().any(|event| {
4835 matches!(
4836 event,
4837 ChangeEvent::DefineName { .. }
4838 | ChangeEvent::UpdateName { .. }
4839 | ChangeEvent::DeleteName { .. }
4840 )
4841 }) {
4842 self.rollback_from_change_events(&new_events)?;
4843 log.truncate(start_len);
4844 return Err(crate::engine::EditorError::TransactionUnsupported {
4845 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
4846 });
4847 }
4848
4849 for ev in &new_events {
4852 self.mirror_forward_change_to_arrow(ev);
4853 }
4854 for ev in &new_events {
4855 self.record_formula_plane_change_for_event(ev);
4856 }
4857
4858 Ok(ret)
4859 }
4860
4861 pub(crate) fn preflight_replay_admission(
4862 &mut self,
4863 events: &[ChangeEvent],
4864 forward: bool,
4865 ) -> Result<(), crate::engine::EditorError> {
4866 if !self.graph_admission_enabled() {
4867 return Ok(());
4868 }
4869 let mut vertex_delta = 0i128;
4870 let mut edge_delta = 0i128;
4871 let mut added_vertices = 0usize;
4872 let mut added_edges = 0usize;
4873 let mut formula_cells = BTreeSet::new();
4874 for event in events {
4875 match event {
4876 ChangeEvent::AddVertex {
4877 formula,
4878 coord,
4879 sheet_id,
4880 ..
4881 } => {
4882 let delta = if forward { 1 } else { -1 };
4883 vertex_delta += delta;
4884 if forward {
4885 added_vertices = added_vertices.saturating_add(1);
4886 if formula.is_some() {
4887 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4888 }
4889 }
4890 }
4891 ChangeEvent::RemoveVertex {
4892 old_formula,
4893 coord,
4894 sheet_id,
4895 ..
4896 } => {
4897 let delta = if forward { -1 } else { 1 };
4898 vertex_delta += delta;
4899 if !forward {
4900 added_vertices = added_vertices.saturating_add(1);
4901 if old_formula.is_some()
4902 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
4903 {
4904 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4905 }
4906 }
4907 }
4908 ChangeEvent::EdgeAdded { .. } => {
4909 let delta = if forward { 1 } else { -1 };
4910 edge_delta += delta;
4911 if forward {
4912 added_edges = added_edges.saturating_add(1);
4913 }
4914 }
4915 ChangeEvent::EdgeRemoved { .. } => {
4916 let delta = if forward { -1 } else { 1 };
4917 edge_delta += delta;
4918 if !forward {
4919 added_edges = added_edges.saturating_add(1);
4920 }
4921 }
4922 ChangeEvent::SetFormula {
4923 addr, old_formula, ..
4924 } => {
4925 if forward || old_formula.is_some() {
4926 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
4927 }
4928 }
4929 _ => {}
4930 }
4931 }
4932 let stats = self.graph.baseline_stats();
4933 let final_vertices = i128::try_from(stats.graph_vertex_count)
4934 .ok()
4935 .and_then(|count| count.checked_add(vertex_delta))
4936 .and_then(|count| usize::try_from(count).ok())
4937 .ok_or_else(|| {
4938 crate::engine::EditorError::Excel(
4939 ExcelError::new(ExcelErrorKind::NImpl)
4940 .with_message("replay vertex count overflow"),
4941 )
4942 })?;
4943 let final_edges = i128::try_from(stats.graph_edge_count)
4944 .ok()
4945 .and_then(|count| count.checked_add(edge_delta))
4946 .and_then(|count| usize::try_from(count).ok())
4947 .ok_or_else(|| {
4948 crate::engine::EditorError::Excel(
4949 ExcelError::new(ExcelErrorKind::NImpl)
4950 .with_message("replay edge count overflow"),
4951 )
4952 })?;
4953 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
4954 final_vertices,
4955 final_edges,
4956 materialization_cells: formula_cells.len() as u64,
4957 added_vertices,
4958 added_edges,
4959 })
4960 .map_err(crate::engine::EditorError::Excel)
4961 }
4962
4963 pub fn undo_logged(
4964 &mut self,
4965 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
4966 log: &mut crate::engine::ChangeLog,
4967 ) -> Result<(), crate::engine::EditorError> {
4968 let pending_events = log
4969 .last_group_indices()
4970 .into_iter()
4971 .map(|index| log.events()[index].clone())
4972 .collect::<Vec<_>>();
4973 self.preflight_replay_admission(&pending_events, false)?;
4974 let names = Self::name_event_names(&pending_events);
4975 let prepared =
4976 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
4977 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4978 let batch = undo.undo(&mut self.graph, log)?;
4979 for item in batch.iter().rev() {
4980 self.apply_inverse_row_visibility_event(&item.event);
4981 self.apply_inverse_staged_formula_event(&item.event);
4982 }
4983 self.mirror_undo_batch_to_arrow(&batch);
4984 if !batch.is_empty() {
4985 for item in &batch {
4986 self.record_formula_plane_change_for_event(&item.event);
4987 }
4988 if !demoted && !names.is_empty() {
4989 self.mark_topology_edited();
4990 }
4991 }
4992 Ok(())
4993 }
4994
4995 pub fn redo_logged(
4996 &mut self,
4997 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
4998 log: &mut crate::engine::ChangeLog,
4999 ) -> Result<(), crate::engine::EditorError> {
5000 let pending_events = undo.pending_redo_events();
5001 self.preflight_replay_admission(&pending_events, true)?;
5002 let names = Self::name_event_names(&pending_events);
5003 let prepared =
5004 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5005 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5006 let batch = undo.redo(&mut self.graph, log)?;
5007 for item in &batch {
5008 self.apply_forward_row_visibility_event(&item.event);
5009 self.apply_forward_staged_formula_event(&item.event);
5010 }
5011 self.mirror_redo_batch_to_arrow(&batch);
5012 if !batch.is_empty() {
5013 for item in &batch {
5014 self.record_formula_plane_change_for_event(&item.event);
5015 }
5016 if !demoted && !names.is_empty() {
5017 self.mark_topology_edited();
5018 }
5019 }
5020 Ok(())
5021 }
5022
5023 pub fn undo_action(
5027 &mut self,
5028 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5029 ) -> Result<(), crate::engine::EditorError> {
5030 let Some(journal) = undo.pop_undo_action() else {
5031 return Ok(());
5032 };
5033 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5034 undo.push_done_action(journal);
5035 return Err(error);
5036 }
5037 let names = Self::name_event_names(&journal.graph.events);
5038 let prepared =
5039 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5040 Ok(prepared) => prepared,
5041 Err(error) => {
5042 undo.push_done_action(journal);
5043 return Err(error);
5044 }
5045 };
5046 let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5047 Ok(demoted) => demoted,
5048 Err(error) => {
5049 undo.push_done_action(journal);
5050 return Err(error);
5051 }
5052 };
5053
5054 journal.graph.undo(&mut self.graph)?;
5055 self.apply_inverse_row_visibility_events(&journal.graph.events);
5056 self.apply_arrow_undo_batch(&journal.arrow, true);
5057 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5058 for event in &journal.graph.events {
5059 self.record_formula_plane_change_for_event(event);
5060 }
5061 if !demoted && !names.is_empty() {
5062 self.mark_topology_edited();
5063 } else {
5064 self.mark_data_edited();
5065 }
5066 }
5067
5068 undo.push_redo_action(journal);
5069 Ok(())
5070 }
5071
5072 pub fn redo_action(
5076 &mut self,
5077 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5078 ) -> Result<(), crate::engine::EditorError> {
5079 let Some(journal) = undo.pop_redo_action() else {
5080 return Ok(());
5081 };
5082 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5083 undo.push_redo_action(journal);
5084 return Err(error);
5085 }
5086 let names = Self::name_event_names(&journal.graph.events);
5087 let prepared =
5088 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5089 Ok(prepared) => prepared,
5090 Err(error) => {
5091 undo.push_redo_action(journal);
5092 return Err(error);
5093 }
5094 };
5095 let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5096 Ok(demoted) => demoted,
5097 Err(error) => {
5098 undo.push_redo_action(journal);
5099 return Err(error);
5100 }
5101 };
5102
5103 journal.graph.redo(&mut self.graph)?;
5104 self.apply_forward_row_visibility_events(&journal.graph.events);
5105 self.apply_arrow_undo_batch(&journal.arrow, false);
5106 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5107 for event in &journal.graph.events {
5108 self.record_formula_plane_change_for_event(event);
5109 }
5110 if !demoted && !names.is_empty() {
5111 self.mark_topology_edited();
5112 } else {
5113 self.mark_data_edited();
5114 }
5115 }
5116
5117 undo.push_done_action(journal);
5118 Ok(())
5119 }
5120
5121 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5122 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5123 let row = addr.coord.row() + 1;
5125 let col = addr.coord.col() + 1;
5126 (sheet, row, col)
5127 }
5128
5129 fn mirror_undo_batch_to_arrow(
5130 &mut self,
5131 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5132 ) {
5133 for item in batch.iter().rev() {
5135 self.mirror_inverse_change_to_arrow(&item.event);
5136 }
5137 }
5138
5139 fn mirror_redo_batch_to_arrow(
5140 &mut self,
5141 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5142 ) {
5143 for item in batch.iter() {
5145 self.mirror_forward_change_to_arrow(&item.event);
5146 }
5147 }
5148
5149 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5150 use crate::engine::ChangeEvent;
5151 use formualizer_common::LiteralValue;
5152
5153 match ev {
5154 ChangeEvent::SetValue {
5155 addr,
5156 old_value,
5157 old_formula,
5158 ..
5159 } => {
5160 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5161 if old_formula.is_some() {
5162 self.clear_delta_overlay_cell(&sheet, row, col);
5163 } else {
5164 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5165 self.mirror_value_to_overlay(&sheet, row, col, &v);
5166 }
5167 }
5168 ChangeEvent::SetFormula {
5169 addr,
5170 old_value,
5171 old_formula,
5172 ..
5173 } => {
5174 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5175 if old_formula.is_some() {
5176 self.clear_delta_overlay_cell(&sheet, row, col);
5177 } else {
5178 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5179 self.mirror_value_to_overlay(&sheet, row, col, &v);
5180 }
5181 }
5182 ChangeEvent::SpillCommitted { old, new, .. } => {
5183 self.mirror_spill_snapshot(new, true);
5185 if let Some(snap) = old {
5186 self.mirror_spill_snapshot(snap, false);
5187 }
5188 }
5189 ChangeEvent::SpillCleared { old, .. } => {
5190 self.mirror_spill_snapshot(old, false);
5192 }
5193 ChangeEvent::SetRowVisibility { .. } => {
5194 }
5196 _ => {}
5197 }
5198 }
5199
5200 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5201 use crate::engine::ChangeEvent;
5202
5203 match ev {
5204 ChangeEvent::SetValue { addr, new, .. } => {
5205 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5206 self.mirror_value_to_overlay(&sheet, row, col, new);
5207 }
5208 ChangeEvent::SetFormula { addr, .. } => {
5209 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5210 self.clear_delta_overlay_cell(&sheet, row, col);
5211 }
5213 ChangeEvent::SpillCommitted { old, new, .. } => {
5214 if let Some(snap) = old {
5215 self.mirror_spill_snapshot(snap, true);
5216 }
5217 self.mirror_spill_snapshot(new, false);
5218 }
5219 ChangeEvent::SpillCleared { old, .. } => {
5220 self.mirror_spill_snapshot(old, true);
5221 }
5222 ChangeEvent::SetRowVisibility { .. } => {
5223 }
5225 _ => {
5226 }
5228 }
5229 }
5230
5231 fn mirror_spill_snapshot(
5232 &mut self,
5233 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5234 clear_only: bool,
5235 ) {
5236 use formualizer_common::LiteralValue;
5237
5238 let mut i = 0usize;
5239 for row in &snap.values {
5240 for v in row {
5241 if let Some(cell) = snap.target_cells.get(i) {
5242 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5243 let out = if clear_only {
5244 LiteralValue::Empty
5245 } else {
5246 v.clone()
5247 };
5248 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5249 }
5250 i += 1;
5251 }
5252 }
5253 if clear_only {
5255 for cell in snap.target_cells.iter().skip(i) {
5256 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5257 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5258 }
5259 }
5260 }
5261
5262 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5263 self.graph.set_default_sheet_by_name(name);
5264 }
5265
5266 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5267 self.graph.set_default_sheet_by_id(id);
5268 }
5269
5270 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5271 self.graph.set_sheet_index_mode(mode);
5272 }
5273
5274 fn clear_cached_static_schedule(&mut self) {
5275 self.cached_static_schedule = None;
5276 }
5277
5278 pub fn mark_data_edited(&mut self) {
5281 self.snapshot_id
5282 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5283 self.has_edited = true;
5284 }
5285
5286 pub fn mark_topology_edited(&mut self) {
5288 self.snapshot_id
5289 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5290 self.topology_epoch = self.topology_epoch.wrapping_add(1);
5291 self.graph.bump_topology_revision();
5292 self.clear_cached_static_schedule();
5293 self.cached_mixed_topology = None;
5294 if let Some(ledger) = self.active_resource_ledger.as_mut() {
5295 let released = ledger.account_mixed_cache(0);
5296 debug_assert!(released.is_ok());
5297 }
5298 self.has_edited = true;
5299 }
5300
5301 fn mark_all_formula_vertices_dirty(&mut self) {
5302 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5303 for vertex in vertices {
5304 self.graph.mark_vertex_dirty(vertex);
5305 }
5306 }
5307
5308 fn mark_moved_formula_vertices_dirty(
5309 &mut self,
5310 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5311 ) {
5312 for vertex in &summary.vertices_moved {
5313 if self.graph.get_formula_id(*vertex).is_some() {
5314 self.graph.mark_vertex_dirty(*vertex);
5315 }
5316 }
5317 }
5318
5319 pub fn sheet_store(&self) -> &SheetStore {
5321 &self.arrow_sheets
5322 }
5323
5324 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5326 &mut self.arrow_sheets
5327 }
5328
5329 pub fn has_staged_formulas(&self) -> bool {
5330 !self.staged_formulas.is_empty()
5331 }
5332
5333 pub fn staged_formula_count(&self) -> usize {
5334 self.staged_formulas.values().map(StagedSheet::len).sum()
5335 }
5336
5337 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5339 self.staged_formulas
5340 .entry(sheet.to_string())
5341 .or_default()
5342 .stage(row, col, text);
5343 self.staged_formula_index.stage(sheet, row, col);
5344 }
5345
5346 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5347 for (row, col, _) in &staged.entries {
5348 self.staged_formula_index.remove(sheet, *row, *col);
5349 }
5350 if staged.deferred_package.is_some() {
5351 self.staged_formula_index.set_package(sheet, None);
5352 }
5353 }
5354
5355 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5356 self.staged_formulas.insert(sheet, staged);
5357 }
5358
5359 #[doc(hidden)]
5360 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5361 SourceFormulaIngress { engine: self }
5362 }
5363
5364 #[doc(hidden)]
5365 #[doc(hidden)]
5368 #[cfg(any(test, feature = "test-support"))]
5369 pub fn set_before_prepared_span_commit_hook(
5370 &mut self,
5371 hook: impl FnOnce() + Send + Sync + 'static,
5372 ) {
5373 self.before_prepared_span_commit_hook = Some(Box::new(hook));
5374 }
5375
5376 #[doc(hidden)]
5377 #[cfg(test)]
5379 pub(crate) fn set_before_target_preparation_commit_hook(
5380 &mut self,
5381 hook: impl FnOnce() + Send + Sync + 'static,
5382 ) {
5383 self.before_target_preparation_commit_hook = Some(Box::new(hook));
5384 }
5385
5386 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5387 let sheet = package.sheet_name.clone();
5388 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5389 debug_assert!(staged.deferred_package.is_none());
5390 staged.deferred_package = Some(package);
5391 staged.reconcile_attached_deferred_package();
5392 self.staged_formula_index
5393 .set_package(&sheet, staged.deferred_package.as_ref());
5394 }
5395
5396 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
5397 let mut removed = None;
5398 let mut remove_sheet = false;
5399 let mut had_package = false;
5400 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
5401 had_package = entries.deferred_package.is_some();
5402 removed = entries.remove(row, col);
5403 remove_sheet = entries.is_empty();
5404 }
5405 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
5406 if !ordinary_removed && had_package {
5407 self.staged_formula_index.touch_package(sheet);
5408 }
5409 if remove_sheet {
5410 self.staged_formulas.remove(sheet);
5411 self.staged_formula_index.set_package(sheet, None);
5412 }
5413 removed
5414 }
5415
5416 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
5417 if self.staged_formulas.remove(sheet).is_some() {
5418 self.staged_formula_index.clear_sheet(sheet);
5419 }
5420 }
5421
5422 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
5423 let Some(entries) = self.staged_formulas.remove(old) else {
5424 return;
5425 };
5426 self.staged_formula_index.clear_sheet(old);
5427 let (formulas, mut package) = entries.into_parts();
5428 for (row, col, text) in formulas {
5429 self.stage_formula_text(new, row, col, text);
5430 }
5431 if let Some(package) = package.as_mut() {
5432 package.sheet_name = new.to_string();
5433 }
5434 if let Some(package) = package {
5435 self.stage_deferred_formula_package(package);
5436 }
5437 }
5438
5439 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
5441 self.staged_formulas
5442 .get(sheet)
5443 .and_then(|v| v.get(row, col))
5444 }
5445
5446 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
5447 &self.formula_parse_diagnostics
5448 }
5449
5450 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
5451 std::mem::take(&mut self.formula_parse_diagnostics)
5452 }
5453
5454 pub fn clear_formula_parse_diagnostics(&mut self) {
5455 self.formula_parse_diagnostics.clear();
5456 }
5457
5458 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
5459 self.last_formula_ingest_report.as_ref()
5460 }
5461
5462 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
5463 &self.formula_ingest_report_total
5464 }
5465
5466 #[cfg(test)]
5467 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
5468 &mut self,
5469 hook: impl FnOnce() + Send + Sync + 'static,
5470 ) {
5471 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
5472 }
5473
5474 #[cfg(test)]
5475 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
5476 self.inject_target_semantic_stale_once_for_test = true;
5477 }
5478
5479 #[cfg(test)]
5480 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
5481 self.force_virtual_dep_changes_remaining_for_test = rounds;
5482 }
5483
5484 #[cfg(test)]
5485 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
5486 self.fail_evaluation_commit_preflight_once_for_test = true;
5487 }
5488
5489 #[cfg(test)]
5490 pub(crate) fn set_target_preparation_fault_for_test(
5491 &mut self,
5492 fault: crate::engine::target_preparation::TargetPreparationFault,
5493 ) {
5494 self.target_preparation_fault_for_test = Some(fault);
5495 }
5496
5497 #[cfg(test)]
5498 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
5499 self.staged_formula_index.revision()
5500 }
5501
5502 #[cfg(test)]
5503 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
5504 let ordinary_storage = self
5505 .staged_formulas
5506 .values()
5507 .map(|sheet| sheet.entries.len())
5508 .sum::<usize>();
5509 let package_storage = self
5510 .staged_formulas
5511 .values()
5512 .filter(|sheet| sheet.deferred_package.is_some())
5513 .count();
5514 ordinary_storage == self.staged_formula_index.ordinary_count()
5515 && package_storage == self.staged_formula_index.package_count()
5516 && self.staged_formulas.iter().all(|(name, sheet)| {
5517 sheet.entries.iter().all(|(row, col, _)| {
5518 let leases = self
5519 .staged_formula_index
5520 .leases_in_region(name, *row, *col, *row, *col);
5521 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
5522 })
5523 })
5524 }
5525
5526 #[cfg(test)]
5527 pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
5528 self.last_formula_plane_span_eval_report.as_ref()
5529 }
5530
5531 #[cfg(test)]
5532 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
5533 self.evaluation_request_begin_count_for_test
5534 }
5535
5536 #[cfg(test)]
5537 pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
5538 self.force_source_family_fallback = force;
5539 }
5540
5541 #[cfg(test)]
5542 pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
5543 self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
5544 }
5545
5546 #[cfg(test)]
5547 pub(crate) fn set_fragmented_commit_fault_for_test(
5548 &mut self,
5549 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5550 ) {
5551 self.fragmented_commit_fault_for_test = Some(fault);
5552 }
5553
5554 #[cfg(test)]
5555 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
5556 &mut self,
5557 hook: impl FnOnce() + Send + Sync + 'static,
5558 ) {
5559 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
5560 }
5561
5562 #[cfg(test)]
5563 pub(crate) fn set_after_eager_proposal_commit_hook(
5564 &mut self,
5565 hook: impl FnOnce() + Send + Sync + 'static,
5566 ) {
5567 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
5568 }
5569
5570 #[cfg(test)]
5571 pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
5572 self.graph.formula_authority().indexes_epoch()
5573 }
5574
5575 #[cfg(test)]
5576 pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
5577 self.formula_plane_capacity_bailouts
5578 }
5579
5580 #[cfg(test)]
5581 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
5582 self.topology_epoch
5583 }
5584
5585 #[cfg(test)]
5586 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
5587 self.graph.topology_revision()
5588 }
5589
5590 #[cfg(test)]
5591 pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
5592 self.cached_mixed_topology.is_some()
5593 }
5594
5595 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
5596 self.formula_ingest_report_total.mode = report.mode;
5597 self.formula_ingest_report_total.accumulate(&report);
5598 self.last_formula_ingest_report = Some(report);
5599 }
5600
5601 fn prepare_source_formula_family(
5602 &mut self,
5603 sheet_id: SheetId,
5604 family: &crate::engine::SourceFormulaFamily,
5605 allow_function_closure: bool,
5606 planning_provider: Option<&dyn crate::traits::FunctionProvider>,
5607 ) -> Result<
5608 (
5609 crate::formula_plane::placement::PreparedAnchorOncePlacement,
5610 bool,
5611 ),
5612 SourceFamilyPreparationError,
5613 > {
5614 let transport = family
5615 .validated_complete_domain(&self.workbook_load_limits)
5616 .map_err(SourceFamilyPreparationError::contract)?;
5617 let domain = match transport {
5618 crate::engine::PlacementDomainTransport::RowRun {
5619 row_start,
5620 row_end,
5621 col,
5622 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5623 crate::engine::PlacementDomainTransport::ColRun {
5624 row,
5625 col_start,
5626 col_end,
5627 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5628 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5629 sheet_id,
5630 rect.start.row,
5631 rect.end.row,
5632 rect.start.col,
5633 rect.end.col,
5634 ),
5635 };
5636 let formula = if family.anchor_text.starts_with('=') {
5637 family.anchor_text.to_string()
5638 } else {
5639 format!("={}", family.anchor_text)
5640 };
5641 let ast = formualizer_parse::parser::parse(&formula)
5642 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5643 let relocation = if allow_function_closure {
5644 validate_anchor_once_shadow_relocation(
5645 &ast,
5646 family.anchor_coord0.row,
5647 family.anchor_coord0.col,
5648 &domain,
5649 planning_provider.unwrap_or(&self.resolver),
5650 )
5651 } else {
5652 validate_anchor_once_syntax(
5653 &ast,
5654 family.anchor_coord0.row,
5655 family.anchor_coord0.col,
5656 &domain,
5657 )
5658 };
5659 relocation.map_err(SourceFamilyPreparationError::ast)?;
5660 let ast_id = self.intern_formula_ast(&ast);
5661 let placement = CellRef::new(
5662 sheet_id,
5663 Coord::from_excel(
5664 family.anchor_coord0.row + 1,
5665 family.anchor_coord0.col + 1,
5666 true,
5667 true,
5668 ),
5669 );
5670 let ingested = {
5671 let mut pipeline = match planning_provider {
5672 Some(provider) => self.graph.ingest_pipeline(provider),
5673 None => self.ingest_pipeline(),
5674 };
5675 if allow_function_closure {
5676 pipeline = pipeline.enable_function_semantics();
5677 }
5678 pipeline.ingest_formula(
5679 FormulaAstInput::RawArena(ast_id),
5680 placement,
5681 Some(family.anchor_text.clone()),
5682 )
5683 }
5684 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5685 let function_semantics_used = ingested
5686 .labels
5687 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5688 let candidate = FormulaPlacementCandidate::new(
5689 sheet_id,
5690 family.anchor_coord0.row,
5691 family.anchor_coord0.col,
5692 ingested.ast_id,
5693 Some(family.anchor_text.clone()),
5694 );
5695 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5696 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
5697 prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
5698 .map(|prepared| (prepared, function_semantics_used))
5699 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
5700 }
5701
5702 fn placement_domain_from_transport(
5703 sheet_id: SheetId,
5704 transport: crate::engine::PlacementDomainTransport,
5705 ) -> PlacementDomain {
5706 match transport {
5707 crate::engine::PlacementDomainTransport::RowRun {
5708 row_start,
5709 row_end,
5710 col,
5711 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5712 crate::engine::PlacementDomainTransport::ColRun {
5713 row,
5714 col_start,
5715 col_end,
5716 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5717 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5718 sheet_id,
5719 rect.start.row,
5720 rect.end.row,
5721 rect.start.col,
5722 rect.end.col,
5723 ),
5724 }
5725 }
5726
5727 #[cfg(test)]
5728 pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
5729 &mut self,
5730 sheet_name: &str,
5731 source_id: crate::engine::SourceFamilyId,
5732 expected: crate::engine::PartitionLegacyMember,
5733 replay: crate::engine::DeferredReplayFormula,
5734 ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
5735 {
5736 let sheet_id = self.graph.sheet_id_mut(sheet_name);
5737 self.graph
5738 .analyze_fragmented_exact_replay_record(
5739 &self.resolver,
5740 sheet_id,
5741 source_id,
5742 expected,
5743 replay,
5744 )
5745 .map_err(|error| format!("{error:?}"))
5746 }
5747
5748 pub(crate) fn prepare_fragmented_source_transaction(
5749 &mut self,
5750 source: &crate::engine::PartitionedSourceFormulaFamily,
5751 disposition: &crate::engine::FormulaReplayDisposition,
5752 prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5753 legacy_formulas: Vec<
5754 crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
5755 >,
5756 ) -> Result<
5757 crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5758 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
5759 > {
5760 let transaction = self.graph.prepare_fragmented_source_transaction(
5761 &self.source_formula_token,
5762 source,
5763 disposition,
5764 prepared,
5765 legacy_formulas,
5766 )?;
5767 self.prepared_legacy_admission(
5768 transaction.legacy_graph(),
5769 transaction.materialization_cells(),
5770 )
5771 .map_err(
5772 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
5773 )?;
5774 Ok(transaction)
5775 }
5776
5777 pub(crate) fn commit_fragmented_source_transaction(
5778 &mut self,
5779 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5780 disposition: &crate::engine::FormulaReplayDisposition,
5781 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5782 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5783 let provider_revision = self.resolver.planning_semantic_revision();
5784 let epoch = registry_guard.epoch();
5785 let resolver = &self.resolver;
5786 let decision = self.graph.commit_fragmented_source_transaction(
5787 prepared,
5788 &self.source_formula_token,
5789 disposition,
5790 epoch,
5791 provider_revision,
5792 || resolver.planning_semantic_revision(),
5793 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5794 );
5795 drop(registry_guard);
5796 decision
5797 }
5798
5799 #[cfg(test)]
5800 pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
5801 &mut self,
5802 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5803 disposition: &crate::engine::FormulaReplayDisposition,
5804 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5805 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5806 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5807 let provider_revision = self.resolver.planning_semantic_revision();
5808 let epoch = registry_guard.epoch();
5809 let resolver = &self.resolver;
5810 let decision = self.graph.commit_fragmented_source_transaction(
5811 prepared,
5812 &self.source_formula_token,
5813 disposition,
5814 epoch,
5815 provider_revision,
5816 || resolver.planning_semantic_revision(),
5817 fault,
5818 );
5819 drop(registry_guard);
5820 decision
5821 }
5822
5823 #[cfg(test)]
5824 pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
5825 &mut self,
5826 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5827 disposition: &crate::engine::FormulaReplayDisposition,
5828 before_second_sample: impl FnOnce(),
5829 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5830 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5831 let provider_revision = self.resolver.planning_semantic_revision();
5832 let epoch = registry_guard.epoch();
5833 let resolver = &self.resolver;
5834 let decision = self.graph.commit_fragmented_source_transaction(
5835 prepared,
5836 &self.source_formula_token,
5837 disposition,
5838 epoch,
5839 provider_revision,
5840 || {
5841 before_second_sample();
5842 resolver.planning_semantic_revision()
5843 },
5844 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5845 );
5846 drop(registry_guard);
5847 decision
5848 }
5849
5850 #[cfg(test)]
5851 pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
5852 &mut self,
5853 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5854 disposition: &crate::engine::FormulaReplayDisposition,
5855 function_semantic_epoch: u64,
5856 function_provider_revision: Option<u64>,
5857 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5858 self.graph.commit_fragmented_source_transaction(
5859 prepared,
5860 &self.source_formula_token,
5861 disposition,
5862 function_semantic_epoch,
5863 function_provider_revision,
5864 || function_provider_revision,
5865 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5866 )
5867 }
5868
5869 pub(crate) fn analyze_partitioned_source_family_for_transaction(
5870 &mut self,
5871 sheet_name: &str,
5872 family: &crate::engine::PartitionedSourceFormulaFamily,
5873 ) -> Result<
5874 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5875 SourceFamilyPreparationError,
5876 > {
5877 family
5878 .validate(&self.workbook_load_limits)
5879 .map_err(SourceFamilyPreparationError::contract)?;
5880 let formula = if family.template_text.starts_with('=') {
5881 family.template_text.to_string()
5882 } else {
5883 format!("={}", family.template_text)
5884 };
5885 let ast = formualizer_parse::parser::parse(&formula)
5886 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5887 let mut requests = Vec::new();
5888 Self::collect_planning_function_requests(&ast, &mut requests);
5889 let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
5890 &self.resolver,
5891 requests,
5892 )
5893 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5894 let sheet_id = self.graph.sheet_id_mut(sheet_name);
5895 let domains: Vec<_> = family
5896 .fragments
5897 .iter()
5898 .copied()
5899 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
5900 .collect();
5901 for domain in &domains {
5902 validate_anchor_once_shadow_relocation(
5903 &ast,
5904 family.template_origin0.row,
5905 family.template_origin0.col,
5906 domain,
5907 &snapshot,
5908 )
5909 .map_err(SourceFamilyPreparationError::ast)?;
5910 }
5911 let ast_id = self.intern_formula_ast(&ast);
5912 let placement = CellRef::new(
5913 sheet_id,
5914 Coord::from_excel(
5915 family.template_origin0.row + 1,
5916 family.template_origin0.col + 1,
5917 true,
5918 true,
5919 ),
5920 );
5921 let ingested = self
5922 .graph
5923 .ingest_pipeline(&snapshot)
5924 .enable_function_semantics()
5925 .ingest_formula(
5926 FormulaAstInput::RawArena(ast_id),
5927 placement,
5928 Some(family.template_text.clone()),
5929 )
5930 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5931 let function_semantics_used = ingested
5932 .labels
5933 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5934 let name_assumptions = self
5935 .graph
5936 .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
5937 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5938 let candidate = FormulaPlacementCandidate::new(
5939 sheet_id,
5940 family.template_origin0.row,
5941 family.template_origin0.col,
5942 ingested.ast_id,
5943 Some(family.template_text.clone()),
5944 );
5945 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5946 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
5947 let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
5948 total
5949 .checked_add(domain.cell_count())
5950 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
5951 })?;
5952 let mut placements = Vec::with_capacity(domains.len());
5953 for domain in domains {
5954 let member_count = domain.cell_count();
5955 placements.push(
5956 prepare_anchor_once_fragment(
5957 candidate.clone(),
5958 analysis.clone(),
5959 domain,
5960 member_count,
5961 direct_cells,
5962 )
5963 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
5964 );
5965 }
5966 let fallback_cells = family.legacy_members.shared_member_count() as u64;
5967 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
5968 return Err(SourceFamilyPreparationError::analysis(
5969 "PartitionMemberCountMismatch",
5970 ));
5971 }
5972 Ok(
5973 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
5974 engine_token: Arc::clone(&self.source_formula_token),
5975 sheet_id,
5976 sheet_name: Arc::from(sheet_name),
5977 source: family.clone(),
5978 placements,
5979 function_semantic_epoch: snapshot.epoch(),
5980 function_provider_revision: snapshot.provider_revision(),
5981 function_semantics_used,
5982 name_assumptions,
5983 direct_cells,
5984 },
5985 )
5986 }
5987
5988 fn prepare_partitioned_source_formula_family_shadow(
5989 &mut self,
5990 sheet_id: SheetId,
5991 family: &crate::engine::PartitionedSourceFormulaFamily,
5992 ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
5993 family
5994 .validate(&self.workbook_load_limits)
5995 .map_err(SourceFamilyPreparationError::contract)?;
5996 let formula = if family.template_text.starts_with('=') {
5997 family.template_text.to_string()
5998 } else {
5999 format!("={}", family.template_text)
6000 };
6001 let ast = formualizer_parse::parser::parse(&formula)
6002 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6003 let domains: Vec<_> = family
6004 .fragments
6005 .iter()
6006 .copied()
6007 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6008 .collect();
6009 for domain in &domains {
6010 validate_anchor_once_shadow_relocation(
6011 &ast,
6012 family.template_origin0.row,
6013 family.template_origin0.col,
6014 domain,
6015 &self.resolver,
6016 )
6017 .map_err(SourceFamilyPreparationError::ast)?;
6018 }
6019 let ast_id = self.intern_formula_ast(&ast);
6020 let placement = CellRef::new(
6021 sheet_id,
6022 Coord::from_excel(
6023 family.template_origin0.row + 1,
6024 family.template_origin0.col + 1,
6025 true,
6026 true,
6027 ),
6028 );
6029 let ingested = self
6030 .ingest_pipeline()
6031 .enable_function_semantics()
6032 .ingest_formula(
6033 FormulaAstInput::RawArena(ast_id),
6034 placement,
6035 Some(family.template_text.clone()),
6036 )
6037 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6038 let function_semantics_used = ingested
6039 .labels
6040 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6041 let candidate = FormulaPlacementCandidate::new(
6042 sheet_id,
6043 family.template_origin0.row,
6044 family.template_origin0.col,
6045 ingested.ast_id,
6046 Some(family.template_text.clone()),
6047 );
6048 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6049 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6050 let fragment_count = domains.len() as u64;
6051 let mut direct_cells = 0u64;
6052 for domain in domains {
6053 let member_count = domain.cell_count();
6054 validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6055 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6056 direct_cells = direct_cells
6057 .checked_add(member_count)
6058 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6059 }
6060 let fallback_cells = family.legacy_members.shared_member_count() as u64;
6061 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6062 return Err(SourceFamilyPreparationError::analysis(
6063 "PartitionMemberCountMismatch",
6064 ));
6065 }
6066 Ok(PreparedPartitionShadow {
6067 fragment_count,
6068 direct_cells,
6069 fallback_cells,
6070 function_semantics_used,
6071 })
6072 }
6073
6074 fn analyze_partitioned_formula_plane_shadow(
6075 &mut self,
6076 batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6077 ) -> FormulaIngestReport {
6078 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6079 for (sheet_name, families) in batches {
6080 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6081 for family in families {
6082 report.shadow_candidate_cells = report
6083 .shadow_candidate_cells
6084 .saturating_add(family.surviving_member_count);
6085 match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6086 Ok(prepared) => {
6087 let fragment_count = prepared.fragment_count;
6088 let descendants = family.surviving_member_count.saturating_sub(1);
6089 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6090 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6091 report.source_anchor_analyses =
6092 report.source_anchor_analyses.saturating_add(1);
6093 report.source_partition_analyses_reused = report
6094 .source_partition_analyses_reused
6095 .saturating_add(fragment_count.saturating_sub(1));
6096 report.source_partitioned_families_prepared = report
6097 .source_partitioned_families_prepared
6098 .saturating_add(1);
6099 report.source_partition_fragments_prepared = report
6100 .source_partition_fragments_prepared
6101 .saturating_add(fragment_count);
6102 report.source_partition_span_cells_prepared = report
6103 .source_partition_span_cells_prepared
6104 .saturating_add(prepared.direct_cells);
6105 report.source_partition_fallback_cells = report
6106 .source_partition_fallback_cells
6107 .saturating_add(prepared.fallback_cells);
6108 report.shadow_accepted_span_cells = report
6109 .shadow_accepted_span_cells
6110 .saturating_add(prepared.direct_cells);
6111 report.shadow_fallback_cells = report
6112 .shadow_fallback_cells
6113 .saturating_add(prepared.fallback_cells);
6114 report.source_descendant_strings_avoided = report
6115 .source_descendant_strings_avoided
6116 .saturating_add(descendants);
6117 report.source_descendant_events_avoided = report
6118 .source_descendant_events_avoided
6119 .saturating_add(descendants);
6120 report.source_descendant_analyses_avoided = report
6121 .source_descendant_analyses_avoided
6122 .saturating_add(descendants);
6123 report.graph_formula_vertices_avoided_shadow = report
6124 .graph_formula_vertices_avoided_shadow
6125 .saturating_add(prepared.direct_cells);
6126 report.ast_roots_avoided_shadow = report
6127 .ast_roots_avoided_shadow
6128 .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6129 report.edge_rows_avoided_shadow = report
6130 .edge_rows_avoided_shadow
6131 .saturating_add(prepared.direct_cells);
6132 if prepared.function_semantics_used {
6133 report.source_partition_function_semantics =
6134 report.source_partition_function_semantics.saturating_add(1);
6135 }
6136 }
6137 Err(error) => {
6138 if error.parse_attempted {
6139 report.source_anchor_parses =
6140 report.source_anchor_parses.saturating_add(1);
6141 }
6142 if error.ast_created {
6143 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6144 }
6145 if error.analysis_created {
6146 report.source_anchor_analyses =
6147 report.source_anchor_analyses.saturating_add(1);
6148 }
6149 report.shadow_fallback_cells = report
6150 .shadow_fallback_cells
6151 .saturating_add(family.surviving_member_count);
6152 report.source_partitioned_families_rejected = report
6153 .source_partitioned_families_rejected
6154 .saturating_add(1);
6155 *report.fallback_reasons.entry(error.reason).or_default() += 1;
6156 }
6157 }
6158 }
6159 }
6160 report
6161 }
6162
6163 fn analyze_compressed_formula_plane_shadow(
6164 &mut self,
6165 batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6166 ) -> FormulaIngestReport {
6167 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6168 let _active_epoch = self.graph.formula_authority().plane.epoch();
6169
6170 for (sheet_name, families) in batches {
6171 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6172 for family in families {
6173 let descendants = family.member_count.saturating_sub(1);
6174 report.shadow_candidate_cells = report
6175 .shadow_candidate_cells
6176 .saturating_add(family.member_count);
6177
6178 match self.prepare_source_formula_family(sheet_id, family, true, None) {
6179 Ok(_) => {
6180 report.source_descendant_strings_avoided = report
6181 .source_descendant_strings_avoided
6182 .saturating_add(descendants);
6183 report.source_descendant_events_avoided = report
6184 .source_descendant_events_avoided
6185 .saturating_add(descendants);
6186 report.source_descendant_analyses_avoided = report
6187 .source_descendant_analyses_avoided
6188 .saturating_add(descendants);
6189 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6190 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6191 report.source_anchor_analyses =
6192 report.source_anchor_analyses.saturating_add(1);
6193 report.shadow_accepted_span_cells = report
6194 .shadow_accepted_span_cells
6195 .saturating_add(family.member_count);
6196 report.source_compressed_families_prepared =
6197 report.source_compressed_families_prepared.saturating_add(1);
6198 report.source_compressed_cells_prepared = report
6199 .source_compressed_cells_prepared
6200 .saturating_add(family.member_count);
6201 report.graph_formula_vertices_avoided_shadow = report
6202 .graph_formula_vertices_avoided_shadow
6203 .saturating_add(family.member_count);
6204 report.ast_roots_avoided_shadow =
6205 report.ast_roots_avoided_shadow.saturating_add(descendants);
6206 report.edge_rows_avoided_shadow = report
6207 .edge_rows_avoided_shadow
6208 .saturating_add(family.member_count);
6209 }
6210 Err(error) => {
6211 if error.parse_attempted {
6212 report.source_anchor_parses =
6213 report.source_anchor_parses.saturating_add(1);
6214 }
6215 if error.ast_created {
6216 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6217 }
6218 if error.analysis_created {
6219 report.source_anchor_analyses =
6220 report.source_anchor_analyses.saturating_add(1);
6221 }
6222 report.shadow_fallback_cells = report
6223 .shadow_fallback_cells
6224 .saturating_add(family.member_count);
6225 *report.fallback_reasons.entry(error.reason).or_default() += 1;
6226 }
6227 }
6228 }
6229 }
6230 report
6231 }
6232
6233 fn analyze_formula_plane_shadow_candidates(
6234 &mut self,
6235 batches: &[FormulaIngestBatch],
6236 ) -> FormulaIngestReport {
6237 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6238 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6239
6240 let _active_epoch = self.graph.formula_authority().plane.epoch();
6243
6244 let batch_sheet_ids: Vec<SheetId> = batches
6245 .iter()
6246 .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
6247 .collect();
6248 let mut groups: BTreeMap<
6249 (SheetId, u64, u32),
6250 Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
6251 > = BTreeMap::new();
6252 {
6253 let mut pipeline = self.ingest_pipeline();
6254 for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
6255 for record in &batch.formulas {
6256 if record.row == 0 || record.col == 0 {
6257 report.shadow_candidate_cells =
6258 report.shadow_candidate_cells.saturating_add(1);
6259 report.shadow_fallback_cells =
6260 report.shadow_fallback_cells.saturating_add(1);
6261 Self::record_shadow_fallback_reason(
6262 &mut report,
6263 PlacementFallbackReason::UnsupportedShapeOrGaps,
6264 1,
6265 );
6266 continue;
6267 }
6268
6269 let placement = CellRef::new(
6270 sheet_id,
6271 Coord::from_excel(record.row, record.col, true, true),
6272 );
6273 let ingested = match pipeline.ingest_formula(
6274 FormulaAstInput::RawArena(record.ast_id),
6275 placement,
6276 record.formula_text.clone(),
6277 ) {
6278 Ok(ingested) => ingested,
6279 Err(_) => {
6280 report.shadow_candidate_cells =
6281 report.shadow_candidate_cells.saturating_add(1);
6282 report.shadow_fallback_cells =
6283 report.shadow_fallback_cells.saturating_add(1);
6284 Self::record_shadow_fallback_reason(
6285 &mut report,
6286 PlacementFallbackReason::UnsupportedCanonicalTemplate,
6287 1,
6288 );
6289 continue;
6290 }
6291 };
6292 let candidate = FormulaPlacementCandidate::new(
6293 sheet_id,
6294 record.row - 1,
6295 record.col - 1,
6296 ingested.ast_id,
6297 record.formula_text.clone(),
6298 );
6299 let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
6300 Ok(analysis) => analysis,
6301 Err(reason) => {
6302 report.shadow_candidate_cells =
6303 report.shadow_candidate_cells.saturating_add(1);
6304 report.shadow_fallback_cells =
6305 report.shadow_fallback_cells.saturating_add(1);
6306 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6307 continue;
6308 }
6309 };
6310 groups
6311 .entry((
6312 sheet_id,
6313 ingested.parameterized_canonical_hash,
6314 candidate.col,
6315 ))
6316 .or_default()
6317 .push((candidate, analysis));
6318 }
6319 }
6320 }
6321
6322 let mut scratch_plane = FormulaPlane::default();
6323 for entries in groups.into_values() {
6324 let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
6325 for (component, component_analyses) in
6326 Self::split_candidate_components_with_analyses(candidates, analyses)
6327 {
6328 let placement_report = place_candidate_family_with_analyses(
6329 &mut scratch_plane,
6330 component,
6331 component_analyses,
6332 );
6333 let counters = placement_report.counters;
6334 report.shadow_candidate_cells = report
6335 .shadow_candidate_cells
6336 .saturating_add(counters.formula_cells_seen);
6337 report.shadow_accepted_span_cells = report
6338 .shadow_accepted_span_cells
6339 .saturating_add(counters.accepted_span_cells);
6340 report.shadow_fallback_cells = report
6341 .shadow_fallback_cells
6342 .saturating_add(counters.legacy_cells);
6343 report.shadow_templates_interned = report
6344 .shadow_templates_interned
6345 .saturating_add(counters.templates_interned);
6346 report.shadow_spans_created = report
6347 .shadow_spans_created
6348 .saturating_add(counters.spans_created);
6349 report.graph_formula_vertices_avoided_shadow = report
6350 .graph_formula_vertices_avoided_shadow
6351 .saturating_add(counters.formula_vertices_avoided);
6352 report.ast_roots_avoided_shadow = report
6353 .ast_roots_avoided_shadow
6354 .saturating_add(counters.ast_roots_avoided);
6355 report.edge_rows_avoided_shadow = report
6356 .edge_rows_avoided_shadow
6357 .saturating_add(counters.edge_rows_avoided);
6358 for (reason, count) in counters.fallback_reasons {
6359 Self::record_shadow_fallback_reason(&mut report, reason, count);
6360 }
6361 }
6362 }
6363 report
6364 }
6365
6366 fn record_shadow_fallback_reason(
6367 report: &mut FormulaIngestReport,
6368 reason: PlacementFallbackReason,
6369 count: u64,
6370 ) {
6371 *report
6372 .fallback_reasons
6373 .entry(format!("{reason:?}"))
6374 .or_default() += count;
6375 }
6376
6377 fn analyze_formula_plane_authoritative_ingest(
6378 &mut self,
6379 batches: &[FormulaIngestBatch],
6380 ) -> (
6381 FormulaIngestReport,
6382 Vec<FormulaIngestBatch>,
6383 PlannedFormulaMaterialize,
6384 ) {
6385 let existing_span_refs = self
6386 .graph
6387 .formula_authority()
6388 .active_span_refs()
6389 .into_iter()
6390 .collect::<rustc_hash::FxHashSet<_>>();
6391 let mut report =
6392 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6393 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6394
6395 let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
6396 let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
6397 let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
6398
6399 for batch in batches {
6400 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6401 for record in &batch.formulas {
6402 if record.row == 0 || record.col == 0 {
6403 report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
6404 report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
6405 Self::record_shadow_fallback_reason(
6406 &mut report,
6407 PlacementFallbackReason::UnsupportedShapeOrGaps,
6408 1,
6409 );
6410 fallback
6411 .entry(batch.sheet_name.clone())
6412 .or_default()
6413 .push(record.clone());
6414 continue;
6415 }
6416
6417 pending_candidates.push((
6418 batch.sheet_name.clone(),
6419 FormulaPlacementCandidate::new(
6420 sheet_id,
6421 record.row - 1,
6422 record.col - 1,
6423 record.ast_id,
6424 record.formula_text.clone(),
6425 ),
6426 ));
6427 }
6428 }
6429
6430 let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
6431 let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
6432 (0..pending_candidates.len()).map(|_| None).collect();
6433 let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
6434 (0..pending_candidates.len()).map(|_| None).collect();
6435 {
6436 let mut pipeline = self.ingest_pipeline();
6437 for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
6438 let placement = CellRef::new(
6439 candidate.sheet_id,
6440 Coord::from_excel(
6441 candidate.row.saturating_add(1),
6442 candidate.col.saturating_add(1),
6443 true,
6444 true,
6445 ),
6446 );
6447 let ingested = pipeline.ingest_formula(
6448 FormulaAstInput::RawArena(candidate.ast_id),
6449 placement,
6450 candidate.formula_text.clone(),
6451 );
6452 match ingested {
6453 Ok(ingested) => {
6454 candidate.ast_id = ingested.ast_id;
6455 let dep_plan = ingested.dep_plan.clone();
6456 match CandidateAnalysis::from_ingested(candidate, &ingested) {
6457 Ok(analysis) => {
6458 analyses_by_index[idx] = Some(analysis);
6459 plans_by_index[idx] = Some(dep_plan);
6460 }
6461 Err(reason) => {
6462 report.shadow_candidate_cells =
6463 report.shadow_candidate_cells.saturating_add(1);
6464 report.shadow_fallback_cells =
6465 report.shadow_fallback_cells.saturating_add(1);
6466 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6467 planned_fallback
6468 .entry(sheet_name.clone())
6469 .or_default()
6470 .push((
6471 candidate.row.saturating_add(1),
6472 candidate.col.saturating_add(1),
6473 candidate.ast_id,
6474 dep_plan,
6475 ));
6476 }
6477 }
6478 }
6479 Err(_) => {
6480 let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
6481 report.shadow_candidate_cells =
6482 report.shadow_candidate_cells.saturating_add(1);
6483 report.shadow_fallback_cells =
6484 report.shadow_fallback_cells.saturating_add(1);
6485 Self::record_shadow_fallback_reason(&mut report, reason, 1);
6486 fallback.entry(sheet_name.clone()).or_default().push(
6487 FormulaIngestRecord::new(
6488 candidate.row.saturating_add(1),
6489 candidate.col.saturating_add(1),
6490 candidate.ast_id,
6491 candidate.formula_text.clone(),
6492 ),
6493 );
6494 }
6495 }
6496 }
6497 }
6498
6499 for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
6500 if analyses_by_index[idx].is_none() {
6501 continue;
6502 }
6503 let canonical_hash = analyses_by_index[idx]
6504 .as_ref()
6505 .expect("remaining candidate has an analysis")
6506 .parameterized_canonical_hash();
6507 groups
6508 .entry((candidate.sheet_id, canonical_hash, candidate.col))
6509 .or_default()
6510 .push(idx);
6511 }
6512
6513 for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
6514 let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
6515 let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
6516 for idx in &candidate_indices {
6517 if let Some(plan) = plans_by_index[*idx].take() {
6520 let candidate = &pending_candidates[*idx].1;
6521 plans_by_coord
6522 .entry((candidate.row, candidate.col))
6523 .or_default()
6524 .push(plan);
6525 }
6526 }
6527 let candidates: Vec<_> = candidate_indices
6528 .iter()
6529 .map(|idx| pending_candidates[*idx].1.clone())
6530 .collect();
6531 let components = Self::split_shadow_candidate_components(candidates);
6532 let analyzed_components =
6533 if components.len() == 1 && components[0].len() == candidate_indices.len() {
6534 let component = components.into_iter().next().expect("one component");
6535 let component_analyses = candidate_indices
6536 .iter()
6537 .map(|idx| {
6538 analyses_by_index[*idx]
6539 .take()
6540 .expect("candidate analysis must be used once")
6541 })
6542 .collect();
6543 vec![(component, component_analyses)]
6544 } else {
6545 let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6546 for idx in candidate_indices.iter().rev() {
6547 let candidate = &pending_candidates[*idx].1;
6548 indices_by_coord
6549 .entry((candidate.row, candidate.col))
6550 .or_default()
6551 .push(*idx);
6552 }
6553
6554 components
6555 .into_iter()
6556 .map(|component| {
6557 let mut component_analyses = Vec::with_capacity(component.len());
6558 for candidate in &component {
6559 let idx = indices_by_coord
6560 .get_mut(&(candidate.row, candidate.col))
6561 .and_then(Vec::pop)
6562 .expect("component candidate must have a precomputed analysis");
6563 component_analyses.push(
6564 analyses_by_index[idx]
6565 .take()
6566 .expect("candidate analysis must be used once"),
6567 );
6568 }
6569 (component, component_analyses)
6570 })
6571 .collect()
6572 };
6573
6574 for (component, component_analyses) in analyzed_components {
6575 for (component, component_analyses) in
6576 split_candidate_affine_literal_runs(component, component_analyses)
6577 {
6578 let placement_report = {
6579 let authority = self.graph.formula_authority_mut();
6580 place_candidate_family_with_analyses(
6581 &mut authority.plane,
6582 component.clone(),
6583 component_analyses,
6584 )
6585 };
6586 Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
6587
6588 let mut candidate_by_placement: FxHashMap<
6596 crate::formula_plane::runtime::PlacementCoord,
6597 &FormulaPlacementCandidate,
6598 > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
6599 for candidate in &component {
6600 candidate_by_placement
6601 .entry(candidate.placement())
6602 .or_insert(candidate);
6603 }
6604 for result in &placement_report.results {
6605 let FormulaPlacementResult::Legacy { placement, .. } = result else {
6606 continue;
6607 };
6608 if let Some(&candidate) = candidate_by_placement.get(placement) {
6609 let plan = plans_by_coord
6610 .get_mut(&(candidate.row, candidate.col))
6611 .and_then(Vec::pop);
6612 if let Some(plan) = plan {
6613 planned_fallback
6614 .entry(sheet_name.clone())
6615 .or_default()
6616 .push((
6617 candidate.row.saturating_add(1),
6618 candidate.col.saturating_add(1),
6619 candidate.ast_id,
6620 plan,
6621 ));
6622 } else {
6623 fallback.entry(sheet_name.clone()).or_default().push(
6624 FormulaIngestRecord::new(
6625 candidate.row.saturating_add(1),
6626 candidate.col.saturating_add(1),
6627 candidate.ast_id,
6628 candidate.formula_text.clone(),
6629 ),
6630 );
6631 }
6632 }
6633 }
6634 }
6635 }
6636 }
6637
6638 let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
6639 let new_span_refs = self
6640 .graph
6641 .formula_authority()
6642 .active_span_refs()
6643 .into_iter()
6644 .filter(|span_ref| !existing_span_refs.contains(span_ref))
6645 .collect::<Vec<_>>();
6646 self.graph
6647 .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
6648
6649 let fallback_batches = fallback
6650 .into_iter()
6651 .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
6652 .collect();
6653 (report, fallback_batches, planned_fallback)
6654 }
6655
6656 fn accumulate_formula_plane_placement_report(
6657 report: &mut FormulaIngestReport,
6658 placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
6659 ) {
6660 let counters = &placement_report.counters;
6661 report.shadow_candidate_cells = report
6662 .shadow_candidate_cells
6663 .saturating_add(counters.formula_cells_seen);
6664 report.shadow_accepted_span_cells = report
6665 .shadow_accepted_span_cells
6666 .saturating_add(counters.accepted_span_cells);
6667 report.shadow_fallback_cells = report
6668 .shadow_fallback_cells
6669 .saturating_add(counters.legacy_cells);
6670 report.shadow_templates_interned = report
6671 .shadow_templates_interned
6672 .saturating_add(counters.templates_interned);
6673 report.shadow_spans_created = report
6674 .shadow_spans_created
6675 .saturating_add(counters.spans_created);
6676 report.graph_formula_vertices_avoided_shadow = report
6677 .graph_formula_vertices_avoided_shadow
6678 .saturating_add(counters.formula_vertices_avoided);
6679 report.ast_roots_avoided_shadow = report
6680 .ast_roots_avoided_shadow
6681 .saturating_add(counters.ast_roots_avoided);
6682 report.edge_rows_avoided_shadow = report
6683 .edge_rows_avoided_shadow
6684 .saturating_add(counters.edge_rows_avoided);
6685 for (reason, count) in &counters.fallback_reasons {
6686 Self::record_shadow_fallback_reason(report, *reason, *count);
6687 }
6688 }
6689
6690 fn split_candidate_components_with_analyses(
6691 candidates: Vec<FormulaPlacementCandidate>,
6692 mut analyses: Vec<CandidateAnalysis>,
6693 ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
6694 let components = Self::split_shadow_candidate_components(candidates.clone());
6695 let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
6696 for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
6697 analysis_by_coord
6698 .entry((candidate.row, candidate.col))
6699 .or_default()
6700 .push(analysis);
6701 }
6702 components
6703 .into_iter()
6704 .flat_map(|component| {
6705 let mut component_analyses = Vec::with_capacity(component.len());
6706 for candidate in &component {
6707 let analysis = analysis_by_coord
6708 .get_mut(&(candidate.row, candidate.col))
6709 .and_then(Vec::pop)
6710 .expect("component candidate must have a precomputed analysis");
6711 component_analyses.push(analysis);
6712 }
6713 split_candidate_affine_literal_runs(component, component_analyses)
6714 })
6715 .collect()
6716 }
6717
6718 fn split_shadow_candidate_components(
6719 candidates: Vec<FormulaPlacementCandidate>,
6720 ) -> Vec<Vec<FormulaPlacementCandidate>> {
6721 if candidates.len() <= 1 {
6722 return vec![candidates];
6723 }
6724
6725 let is_row_run = candidates.windows(2).all(|w| {
6729 w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
6730 });
6731 let is_col_run = candidates.windows(2).all(|w| {
6732 w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
6733 });
6734 if is_row_run || is_col_run {
6735 return vec![candidates];
6736 }
6737
6738 let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6739 for (idx, candidate) in candidates.iter().enumerate() {
6740 coord_to_indices
6741 .entry((candidate.row, candidate.col))
6742 .or_default()
6743 .push(idx);
6744 }
6745
6746 let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
6747 let mut components = Vec::new();
6748 while let Some(&start) = remaining.iter().next() {
6749 remaining.remove(&start);
6750 let mut queue = VecDeque::from([start]);
6751 let mut component_indices = Vec::new();
6752
6753 while let Some(idx) = queue.pop_front() {
6754 component_indices.push(idx);
6755 let candidate = &candidates[idx];
6756 let mut neighbor_coords = Vec::with_capacity(5);
6757 neighbor_coords.push((candidate.row, candidate.col));
6758 if let Some(row) = candidate.row.checked_sub(1) {
6759 neighbor_coords.push((row, candidate.col));
6760 }
6761 neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
6762 if let Some(col) = candidate.col.checked_sub(1) {
6763 neighbor_coords.push((candidate.row, col));
6764 }
6765 neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
6766
6767 for coord in neighbor_coords {
6768 if let Some(indices) = coord_to_indices.get(&coord) {
6769 for &neighbor in indices {
6770 if remaining.remove(&neighbor) {
6771 queue.push_back(neighbor);
6772 }
6773 }
6774 }
6775 }
6776 }
6777
6778 component_indices.sort_by_key(|idx| {
6779 let candidate = &candidates[*idx];
6780 (candidate.row, candidate.col, *idx)
6781 });
6782 components.push(
6783 component_indices
6784 .into_iter()
6785 .map(|idx| candidates[idx].clone())
6786 .collect(),
6787 );
6788 }
6789
6790 components
6791 }
6792
6793 fn collect_planning_function_requests(
6794 ast: &ASTNode,
6795 requests: &mut Vec<(String, String, usize)>,
6796 ) {
6797 match &ast.node_type {
6798 ASTNodeType::Function { name, args } => {
6799 requests.push((String::new(), name.clone(), args.len()));
6800 for arg in args {
6801 Self::collect_planning_function_requests(arg, requests);
6802 }
6803 }
6804 ASTNodeType::BinaryOp { left, right, .. } => {
6805 Self::collect_planning_function_requests(left, requests);
6806 Self::collect_planning_function_requests(right, requests);
6807 }
6808 ASTNodeType::UnaryOp { expr, .. } => {
6809 Self::collect_planning_function_requests(expr, requests);
6810 }
6811 ASTNodeType::Call { callee, args } => {
6812 Self::collect_planning_function_requests(callee, requests);
6813 for arg in args {
6814 Self::collect_planning_function_requests(arg, requests);
6815 }
6816 }
6817 ASTNodeType::Array(rows) => {
6818 for cell in rows.iter().flatten() {
6819 Self::collect_planning_function_requests(cell, requests);
6820 }
6821 }
6822 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6823 }
6824 }
6825
6826 fn prepared_placement_function_semantics_changed(
6827 &self,
6828 semantic_epoch: u64,
6829 prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
6830 guard: &crate::function_registry::SemanticEpochReadGuard,
6831 ) -> bool {
6832 if semantic_epoch == guard.epoch() {
6833 return false;
6834 }
6835 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6836 let Some(ast) = self
6837 .graph
6838 .data_store()
6839 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6840 else {
6841 return true;
6843 };
6844 let mut requests = Vec::new();
6845 Self::collect_planning_function_requests(&ast, &mut requests);
6846 guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
6847 }
6848
6849 fn prepared_function_semantics_changed(
6850 &self,
6851 preparation: &crate::engine::FormulaCompressedPreparation,
6852 guard: &crate::function_registry::SemanticEpochReadGuard,
6853 ) -> bool {
6854 if preparation.function_semantic_epoch == guard.epoch() {
6855 return false;
6856 }
6857
6858 let mut requests = Vec::new();
6859 for (_, _, prepared) in &preparation.prepared {
6860 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6861 let Some(ast) = self
6862 .graph
6863 .data_store()
6864 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6865 else {
6866 return true;
6867 };
6868 Self::collect_planning_function_requests(&ast, &mut requests);
6869 }
6870 guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
6871 }
6872
6873 pub(crate) fn prepare_source_formula_families(
6874 &mut self,
6875 sheet_name: &str,
6876 families: &[crate::engine::SourceFormulaFamily],
6877 ) -> crate::engine::FormulaCompressedPreparation {
6878 let mut preparation = crate::engine::FormulaCompressedPreparation {
6879 engine_token: Arc::clone(&self.source_formula_token),
6880 function_semantic_epoch: crate::function_registry::semantic_epoch(),
6881 function_provider_revision: None,
6882 function_semantics_used: false,
6883 sheet_name: Arc::from(sheet_name),
6884 prepared: Vec::new(),
6885 rejected: BTreeMap::new(),
6886 fragmented: Vec::new(),
6887 fragmented_sources: BTreeMap::new(),
6888 eager_replay: Vec::new(),
6889 preparation_spool_replays: 0,
6890 clean_rejected_anchor_counts: [0; 3],
6891 fragmented_rejected_anchor_counts: [0; 3],
6892 exact_replay: None,
6893 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6894 };
6895 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6896 return preparation;
6897 }
6898
6899 let mut requests = Vec::new();
6900 for family in families {
6901 let formula = if family.anchor_text.starts_with('=') {
6902 family.anchor_text.to_string()
6903 } else {
6904 format!("={}", family.anchor_text)
6905 };
6906 if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
6907 Self::collect_planning_function_requests(&ast, &mut requests);
6908 }
6909 }
6910 requests.sort();
6911 requests.dedup();
6912 let snapshot =
6913 match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6914 &self.resolver,
6915 requests,
6916 ) {
6917 Ok(snapshot) => snapshot,
6918 Err(error) => {
6919 let reason = match error {
6920 crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
6921 "FunctionSemanticSnapshotUnavailable"
6922 }
6923 crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
6924 "FunctionProviderRevisionUnavailable"
6925 }
6926 };
6927 for family in families {
6928 preparation
6929 .rejected
6930 .insert(family.source_id, reason.to_string());
6931 }
6932 return preparation;
6933 }
6934 };
6935 preparation.function_semantic_epoch = snapshot.epoch();
6936 preparation.function_provider_revision = snapshot.provider_revision();
6937
6938 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6939 for family in families {
6940 #[cfg(test)]
6941 let force_replay = self.force_source_family_fallback
6942 || cfg!(feature = "benchmark_internal")
6943 && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
6944 #[cfg(all(feature = "benchmark_internal", not(test)))]
6945 let force_replay =
6946 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
6947 #[cfg(any(test, feature = "benchmark_internal"))]
6948 if force_replay {
6949 preparation
6950 .rejected
6951 .insert(family.source_id, "ForcedReplay".to_string());
6952 continue;
6953 }
6954 match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
6955 Ok((prepared, function_semantics_used)) => {
6956 preparation.function_semantics_used |= function_semantics_used;
6957 preparation
6958 .replay_disposition
6959 .set_family_direct(family.source_id);
6960 preparation
6961 .prepared
6962 .push((family.source_id, family.source_order, prepared));
6963 }
6964 Err(error) => {
6965 preparation.clean_rejected_anchor_counts[0] = preparation
6966 .clean_rejected_anchor_counts[0]
6967 .saturating_add(u64::from(error.parse_attempted));
6968 preparation.clean_rejected_anchor_counts[1] = preparation
6969 .clean_rejected_anchor_counts[1]
6970 .saturating_add(u64::from(error.ast_created));
6971 preparation.clean_rejected_anchor_counts[2] = preparation
6972 .clean_rejected_anchor_counts[2]
6973 .saturating_add(u64::from(error.analysis_created));
6974 preparation.rejected.insert(family.source_id, error.reason);
6975 }
6976 }
6977 }
6978 preparation
6979 }
6980
6981 fn prepare_source_formula_proposals(
6982 &mut self,
6983 sheet_name: &str,
6984 families: &[crate::engine::SourceFormulaFamily],
6985 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6986 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6987 formula_record_count: u64,
6988 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6989 suppressed: &BTreeSet<(u32, u32)>,
6990 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6991 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6992 preparation.exact_replay = Some(Arc::clone(&replay));
6993 preparation
6994 .replay_disposition
6995 .extend_suppressed_excel_coords(suppressed.iter().copied());
6996 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6997 return Ok(preparation);
6998 }
6999 #[cfg(feature = "benchmark_internal")]
7000 let benchmark_forced_replay =
7001 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7002 #[cfg(not(feature = "benchmark_internal"))]
7003 let benchmark_forced_replay = false;
7004 let authority_partitions = if benchmark_forced_replay {
7005 for partition in authority_partitions {
7006 preparation
7007 .rejected
7008 .insert(partition.source_id, "ForcedReplay".to_string());
7009 }
7010 &[][..]
7011 } else {
7012 authority_partitions
7013 };
7014 let authority_ids: BTreeSet<_> = authority_partitions
7015 .iter()
7016 .map(|partition| partition.source_id)
7017 .collect();
7018 for partition in replay_partitions {
7019 if !authority_ids.contains(&partition.source_id) {
7020 preparation
7021 .replay_disposition
7022 .register_partition(partition, false)
7023 .map_err(|reason| {
7024 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7025 })?;
7026 }
7027 }
7028 if replay_partitions.is_empty()
7029 && preparation.rejected.is_empty()
7030 && preparation.direct_cell_count() == formula_record_count
7031 {
7032 return Ok(preparation);
7033 }
7034
7035 let mut analyzed = Vec::new();
7036 for source in authority_partitions {
7037 preparation
7038 .fragmented_sources
7039 .insert(source.source_id, source.clone());
7040 match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7041 Ok(prepared) => {
7042 let mut candidate = preparation.replay_disposition.clone();
7043 match candidate.register_partition(source, true) {
7044 Ok(()) => {
7045 preparation.replay_disposition = candidate;
7046 analyzed.push((source.clone(), prepared));
7047 }
7048 Err(reason) => {
7049 preparation
7050 .rejected
7051 .insert(source.source_id, reason.to_string());
7052 preparation
7053 .replay_disposition
7054 .register_partition(source, false)
7055 .map_err(|reason| {
7056 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7057 })?;
7058 }
7059 }
7060 }
7061 Err(error) => {
7062 preparation.fragmented_rejected_anchor_counts[0] = preparation
7063 .fragmented_rejected_anchor_counts[0]
7064 .saturating_add(u64::from(error.parse_attempted));
7065 preparation.fragmented_rejected_anchor_counts[1] = preparation
7066 .fragmented_rejected_anchor_counts[1]
7067 .saturating_add(u64::from(error.ast_created));
7068 preparation.fragmented_rejected_anchor_counts[2] = preparation
7069 .fragmented_rejected_anchor_counts[2]
7070 .saturating_add(u64::from(error.analysis_created));
7071 preparation.rejected.insert(source.source_id, error.reason);
7072 preparation
7073 .replay_disposition
7074 .register_partition(source, false)
7075 .map_err(|reason| {
7076 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7077 })?;
7078 }
7079 }
7080 }
7081
7082 let initial_replay = replay
7083 .lock()
7084 .map_err(|_| {
7085 ExcelError::new(ExcelErrorKind::Value)
7086 .with_message("compressed formula exact replay lock poisoned")
7087 })?
7088 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7089 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7090 preparation.preparation_spool_replays = 1;
7091 let sheet_id = self.graph.sheet_id_mut(sheet_name);
7092 let mut preparation_failed = false;
7093 for (source, prepared_family) in analyzed {
7094 let result = (|| {
7095 let expected: BTreeMap<_, _> = source
7096 .legacy_members
7097 .as_slice()
7098 .iter()
7099 .map(|member| (member.coord, member.kind))
7100 .collect();
7101 let mut actual = BTreeMap::new();
7102 for record in initial_replay
7103 .iter()
7104 .filter(|record| record.partition_owner == Some(source.source_id))
7105 {
7106 let coord = record
7107 .row
7108 .checked_sub(1)
7109 .zip(record.col.checked_sub(1))
7110 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7111 .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7112 let kind = match record.family {
7113 Some(family) if family == source.source_id => {
7114 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7115 }
7116 None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7117 Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7118 };
7119 if actual.insert(coord, (kind, record.clone())).is_some() {
7120 return Err("ExactReplayRecordDuplicate".to_string());
7121 }
7122 }
7123 let actual_kinds: BTreeMap<_, _> = actual
7124 .iter()
7125 .map(|(coord, (kind, _))| (*coord, *kind))
7126 .collect();
7127 if actual_kinds != expected {
7128 return Err("ExactReplayLegacySetMismatch".to_string());
7129 }
7130 let mut legacy = Vec::with_capacity(source.legacy_members.len());
7131 for expected in source.legacy_members.as_slice() {
7132 let record = actual
7133 .remove(&expected.coord)
7134 .map(|(_, record)| record)
7135 .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7136 legacy.push(
7137 self.graph
7138 .analyze_fragmented_exact_replay_record(
7139 &self.resolver,
7140 sheet_id,
7141 source.source_id,
7142 *expected,
7143 record,
7144 )
7145 .map_err(|error| format!("{error:?}"))?,
7146 );
7147 }
7148 Ok::<_, String>((prepared_family, legacy))
7149 })();
7150 match result {
7151 Ok((prepared_family, legacy)) => preparation.fragmented.push(
7152 crate::engine::formula_source::PreparedFragmentedSourceProposal {
7153 source: source.clone(),
7154 prepared: prepared_family,
7155 legacy,
7156 replay: Arc::clone(&replay),
7157 },
7158 ),
7159 Err(reason) => {
7160 preparation_failed = true;
7161 for count in &mut preparation.fragmented_rejected_anchor_counts {
7162 *count = count.saturating_add(1);
7163 }
7164 preparation.rejected.insert(source.source_id, reason);
7165 preparation
7166 .replay_disposition
7167 .force_family_legacy(source.source_id);
7168 }
7169 }
7170 }
7171
7172 preparation.eager_replay = if preparation_failed {
7173 preparation.preparation_spool_replays = 2;
7174 replay
7175 .lock()
7176 .map_err(|_| {
7177 ExcelError::new(ExcelErrorKind::Value)
7178 .with_message("compressed formula exact replay lock poisoned")
7179 })?
7180 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7181 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7182 } else {
7183 initial_replay
7184 };
7185 Ok(preparation)
7186 }
7187
7188 fn formula_batch_from_exact_replay(
7189 &mut self,
7190 sheet_name: &str,
7191 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7192 ) -> Result<FormulaIngestBatch, ExcelError> {
7193 let mut cache = rustc_hash::FxHashMap::default();
7194 let mut formulas = Vec::new();
7195 for record in replayed {
7196 let key = if record.text.starts_with('=') {
7197 record.text
7198 } else {
7199 format!("={}", record.text)
7200 };
7201 let ast_id = if let Some(cached) = cache.get(&key) {
7202 *cached
7203 } else {
7204 let parsed = match formualizer_parse::parser::parse(&key) {
7205 Ok(parsed) => parsed,
7206 Err(error) => {
7207 let Some(parsed) = self.handle_formula_parse_error(
7208 sheet_name,
7209 record.row,
7210 record.col,
7211 &key,
7212 error.to_string(),
7213 )?
7214 else {
7215 continue;
7216 };
7217 parsed
7218 }
7219 };
7220 let ast_id = self.intern_formula_ast(&parsed);
7221 cache.insert(key.clone(), ast_id);
7222 ast_id
7223 };
7224 formulas.push(
7225 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
7226 .with_source_proof(record.source_order, record.family, record.partition_owner),
7227 );
7228 }
7229 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
7230 }
7231
7232 fn replay_prepared_families_exact_records(
7233 &mut self,
7234 preparation: &crate::engine::FormulaCompressedPreparation,
7235 ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
7236 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7237 ExcelError::new(ExcelErrorKind::Value)
7238 .with_message("stale compressed preparation has no exact replay owner")
7239 })?;
7240 let direct: BTreeSet<_> = preparation
7241 .prepared
7242 .iter()
7243 .map(|(family, _, _)| *family)
7244 .collect();
7245 let mut replay_disposition = preparation.replay_disposition.clone();
7246 for family in &direct {
7247 replay_disposition.force_family_legacy(*family);
7248 }
7249 let replayed = replay
7250 .lock()
7251 .map_err(|_| {
7252 ExcelError::new(ExcelErrorKind::Value)
7253 .with_message("compressed formula exact replay lock poisoned")
7254 })?
7255 .replay(&replay_disposition)
7256 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7257 let mut records: Vec<_> = replayed
7258 .into_iter()
7259 .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
7260 .collect();
7261 records.sort_by_key(|record| record.source_order);
7262 let expected: u64 = preparation
7263 .prepared
7264 .iter()
7265 .map(|(_, _, prepared)| prepared.member_count)
7266 .sum();
7267 if records.len() as u64 != expected {
7268 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7269 "compressed formula exact replay incomplete: expected {expected}, got {}",
7270 records.len()
7271 )));
7272 }
7273 Ok(records)
7274 }
7275
7276 fn replay_one_prepared_family_exact(
7277 &mut self,
7278 preparation: &crate::engine::FormulaCompressedPreparation,
7279 family: crate::engine::SourceFamilyId,
7280 member_count: u64,
7281 ) -> Result<FormulaIngestBatch, ExcelError> {
7282 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7283 ExcelError::new(ExcelErrorKind::Value)
7284 .with_message("stale compressed preparation has no exact replay owner")
7285 })?;
7286 let mut replay_disposition = preparation.replay_disposition.clone();
7287 replay_disposition.force_family_legacy(family);
7288 let replayed = replay
7289 .lock()
7290 .map_err(|_| {
7291 ExcelError::new(ExcelErrorKind::Value)
7292 .with_message("compressed formula exact replay lock poisoned")
7293 })?
7294 .replay(&replay_disposition)
7295 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7296 .into_iter()
7297 .filter(|record| record.family == Some(family));
7298 let batch =
7299 self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
7300 if batch.formulas.len() as u64 != member_count {
7301 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7302 "compressed formula exact replay incomplete: expected {member_count}, got {}",
7303 batch.formulas.len()
7304 )));
7305 }
7306 Ok(batch)
7307 }
7308
7309 fn exact_replay_record_is_prepared_partition_legacy(
7310 source: &crate::engine::PartitionedSourceFormulaFamily,
7311 record: &crate::engine::DeferredReplayFormula,
7312 ) -> bool {
7313 if record.partition_owner != Some(source.source_id) {
7314 return false;
7315 }
7316 let Some(coord) = record
7317 .row
7318 .checked_sub(1)
7319 .zip(record.col.checked_sub(1))
7320 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7321 else {
7322 return false;
7323 };
7324 source.legacy_members.as_slice().iter().any(|member| {
7325 member.coord == coord
7326 && match member.kind {
7327 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
7328 record.family == Some(source.source_id)
7329 }
7330 crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
7331 record.family.is_none()
7332 }
7333 }
7334 })
7335 }
7336
7337 fn validate_whole_partition_replay(
7338 source: &crate::engine::PartitionedSourceFormulaFamily,
7339 records: &[crate::engine::DeferredReplayFormula],
7340 ) -> Result<(), ExcelError> {
7341 let mut coordinates = BTreeSet::new();
7342 let mut shared = 0u64;
7343 let mut ordinary = 0u64;
7344 for record in records {
7345 let coord = record
7346 .row
7347 .checked_sub(1)
7348 .zip(record.col.checked_sub(1))
7349 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7350 .ok_or_else(|| {
7351 ExcelError::new(ExcelErrorKind::Value)
7352 .with_message("whole-family replay contains an invalid coordinate")
7353 })?;
7354 if !coordinates.insert(coord) {
7355 return Err(ExcelError::new(ExcelErrorKind::Value)
7356 .with_message("whole-family replay contains a duplicate coordinate"));
7357 }
7358 match (record.family, record.partition_owner) {
7359 (Some(family), Some(owner))
7360 if family == source.source_id && owner == source.source_id =>
7361 {
7362 let direct = source.fragments.iter().any(|fragment| {
7363 let rect = fragment.rect();
7364 coord.row >= rect.start.row
7365 && coord.row <= rect.end.row
7366 && coord.col >= rect.start.col
7367 && coord.col <= rect.end.col
7368 });
7369 let legacy = source.legacy_members.as_slice().iter().any(|member| {
7370 member.coord == coord
7371 && member.kind
7372 == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7373 });
7374 if !direct && !legacy {
7375 return Err(ExcelError::new(ExcelErrorKind::Value)
7376 .with_message("whole-family replay contains an extra shared member"));
7377 }
7378 shared = shared.saturating_add(1);
7379 }
7380 (None, Some(owner)) if owner == source.source_id => {
7381 if !source.legacy_members.as_slice().iter().any(|member| {
7382 member.coord == coord
7383 && member.kind
7384 == crate::engine::PartitionLegacyMemberKind::OrdinaryException
7385 }) {
7386 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7387 "whole-family replay contains an extra ordinary exception",
7388 ));
7389 }
7390 ordinary = ordinary.saturating_add(1);
7391 }
7392 _ => {
7393 return Err(ExcelError::new(ExcelErrorKind::Value)
7394 .with_message("whole-family replay ownership mismatch"));
7395 }
7396 }
7397 }
7398 if shared != source.surviving_member_count
7399 || ordinary != source.reconciliation.ordinary_exceptions
7400 {
7401 return Err(ExcelError::new(ExcelErrorKind::Value)
7402 .with_message("whole-family replay is incomplete"));
7403 }
7404 Ok(())
7405 }
7406
7407 fn prepare_target_combined_legacy_graph(
7408 &self,
7409 packages: &[PreparedTargetSourcePackage],
7410 ordinary: &[PreparedOrdinaryStagedFormula],
7411 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
7412 let mut planned_by_coord = BTreeMap::new();
7413 for package in packages {
7414 for (row, col, ast_id, plan) in &package.legacy {
7415 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
7416 }
7417 }
7418 for formula in ordinary {
7419 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
7420 planned_by_coord.insert(
7421 (formula.sheet_id, formula.lease.row, formula.lease.col),
7422 (ast_id, plan),
7423 );
7424 }
7425 }
7426 let planned = planned_by_coord
7427 .into_iter()
7428 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
7429 .collect::<Vec<_>>();
7430 let formula_count = planned.len();
7431 let graph = self
7432 .graph
7433 .prepare_legacy_graph_plan_multi_sheet(planned)
7434 .map_err(|error| {
7435 ExcelError::new(ExcelErrorKind::Value)
7436 .with_message(format!("target graph preparation failed: {error}"))
7437 })?;
7438 Ok((graph, formula_count))
7439 }
7440
7441 fn materialize_target_package_direct_records(
7442 &mut self,
7443 package: &mut PreparedTargetSourcePackage,
7444 assumptions: &crate::engine::PreparationRevision,
7445 planning_requests: &mut BTreeSet<(String, String, usize)>,
7446 deadline: Option<std::time::Instant>,
7447 ) -> Result<(), ExcelError> {
7448 let existing = package
7449 .legacy
7450 .iter()
7451 .map(|(row, col, _, _)| (*row, *col))
7452 .collect::<BTreeSet<_>>();
7453 let mut missing = BTreeMap::new();
7454 for record in &package.replay_records {
7455 if !existing.contains(&(record.row, record.col)) {
7456 missing.insert((record.row, record.col), record.clone());
7457 }
7458 }
7459 let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
7460 for record in batch.formulas {
7461 self.target_preparation_checkpoint(deadline, 1)?;
7462 let ast = self
7463 .graph
7464 .data_store()
7465 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
7466 .ok_or_else(|| {
7467 ExcelError::new(ExcelErrorKind::Value)
7468 .with_message("target fallback AST is unavailable")
7469 })?;
7470 let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
7471 if let Some(reason) =
7472 Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
7473 {
7474 return Err(Self::preparation_stale(
7475 reason,
7476 "target fallback planning snapshot became stale during materialization",
7477 ));
7478 }
7479 let placement = CellRef::new(
7480 package.sheet_id,
7481 Coord::from_excel(record.row, record.col, true, true),
7482 );
7483 let ingested = self
7484 .graph
7485 .ingest_pipeline(&snapshot)
7486 .enable_function_semantics()
7487 .ingest_formula(
7488 FormulaAstInput::RawArena(record.ast_id),
7489 placement,
7490 record.formula_text,
7491 )?;
7492 package
7493 .legacy
7494 .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7495 }
7496 package.direct_families = 0;
7497 package.direct_cells = 0;
7498 package.direct_fragments = 0;
7499 package.direct_complete_families = 0;
7500 package.direct_complete_cells = 0;
7501 package.direct_partition_families = 0;
7502 package.direct_partition_cells = 0;
7503 Ok(())
7504 }
7505
7506 fn prepare_target_source_package(
7507 &mut self,
7508 sheet: &str,
7509 lease: StagedPackageLease,
7510 deadline: Option<std::time::Instant>,
7511 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
7512 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
7513 ExcelError::new(ExcelErrorKind::Ref)
7514 .with_message(format!("deferred source sheet not found: {sheet}"))
7515 })?;
7516 let (
7517 mut source_report,
7518 families,
7519 partitions,
7520 replay,
7521 invalidated,
7522 suppressed,
7523 reconciliation_replay,
7524 ) = {
7525 let package = self
7526 .staged_formulas
7527 .get(sheet)
7528 .and_then(|staged| staged.deferred_package.as_ref())
7529 .ok_or_else(|| {
7530 ExcelError::new(ExcelErrorKind::Value)
7531 .with_message("staged deferred source package is unavailable")
7532 })?;
7533 if package.sheet_name != sheet {
7534 return Err(ExcelError::new(ExcelErrorKind::Value)
7535 .with_message("deferred formula package sheet mismatch"));
7536 }
7537 (
7538 package.report.clone(),
7539 package.families.clone(),
7540 package.partitioned_families.clone(),
7541 Arc::clone(&package.replay),
7542 package.invalidated.clone(),
7543 package.suppressed.clone(),
7544 package.reconciliation_replay.clone(),
7545 )
7546 };
7547
7548 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
7549 for partition in &partitions {
7550 replay_disposition
7551 .register_partition(partition, false)
7552 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
7553 }
7554 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
7555 self.target_preparation_checkpoint(deadline, 1)?;
7556 let mut replay_records = if let Some(mut records) = reconciliation_replay {
7557 records.retain(|record| {
7558 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
7559 else {
7560 return true;
7561 };
7562 let coord = crate::engine::SourceCoord { row, col };
7563 let disposition = record.family.map_or_else(
7564 || replay_disposition.ordinary_disposition(coord).0,
7565 |family| replay_disposition.shared_disposition(family, coord),
7566 );
7567 !matches!(
7568 disposition,
7569 crate::engine::FormulaReplayCoordinateDisposition::Direct
7570 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
7571 )
7572 });
7573 records
7574 } else {
7575 replay
7576 .lock()
7577 .map_err(|_| {
7578 ExcelError::new(ExcelErrorKind::Value)
7579 .with_message("deferred formula spool lock poisoned")
7580 })?
7581 .replay_partitioned(&replay_disposition, &partitions)
7582 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7583 };
7584 replay_records.sort_by_key(|record| record.source_order);
7585 for chunk in replay_records.chunks(256) {
7586 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
7587 }
7588 if replay_records
7589 .windows(2)
7590 .any(|records| records[0].source_order == records[1].source_order)
7591 {
7592 return Err(ExcelError::new(ExcelErrorKind::Value)
7593 .with_message("duplicate deferred source-order proof"));
7594 }
7595
7596 let mut disposition = replay_disposition;
7597 let mut ordered_placements = Vec::new();
7598 let mut direct_families = 0usize;
7599 let mut direct_cells = 0u64;
7600 let mut direct_fragments = 0u64;
7601 let mut direct_complete_families = 0u64;
7602 let mut direct_complete_cells = 0u64;
7603 let mut direct_partition_families = 0u64;
7604 let mut direct_partition_cells = 0u64;
7605 let mut anchor_parses = 0u64;
7606 let mut anchor_asts = 0u64;
7607 let mut anchor_analyses = 0u64;
7608 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
7609 for family in &families {
7610 self.target_preparation_checkpoint(deadline, 1)?;
7611 if invalidated.contains(&family.source_id) {
7612 continue;
7613 }
7614 match self.prepare_source_formula_family(sheet_id, family, true, None) {
7615 Ok((prepared, _)) => {
7616 anchor_parses = anchor_parses.saturating_add(1);
7617 anchor_asts = anchor_asts.saturating_add(1);
7618 anchor_analyses = anchor_analyses.saturating_add(1);
7619 direct_families = direct_families.saturating_add(1);
7620 direct_cells = direct_cells.saturating_add(prepared.member_count);
7621 direct_complete_families = direct_complete_families.saturating_add(1);
7622 direct_complete_cells =
7623 direct_complete_cells.saturating_add(prepared.member_count);
7624 if self.config.formula_plane_mode
7625 == FormulaPlaneMode::AuthoritativeExperimental
7626 {
7627 disposition.set_family_direct(family.source_id);
7628 }
7629 ordered_placements.push((family.source_order, vec![prepared]));
7630 }
7631 Err(error) => {
7632 anchor_parses =
7633 anchor_parses.saturating_add(u64::from(error.parse_attempted));
7634 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7635 anchor_analyses =
7636 anchor_analyses.saturating_add(u64::from(error.analysis_created));
7637 *source_report
7638 .fallback_reasons
7639 .entry(error.reason)
7640 .or_default() += 1;
7641 }
7642 }
7643 }
7644
7645 for source in &partitions {
7646 self.target_preparation_checkpoint(deadline, 1)?;
7647 if invalidated.contains(&source.source_id) {
7648 continue;
7649 }
7650 let prepared =
7651 match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
7652 Ok(prepared) => {
7653 anchor_parses = anchor_parses.saturating_add(1);
7654 anchor_asts = anchor_asts.saturating_add(1);
7655 anchor_analyses = anchor_analyses.saturating_add(1);
7656 prepared
7657 }
7658 Err(error) => {
7659 anchor_parses =
7660 anchor_parses.saturating_add(u64::from(error.parse_attempted));
7661 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7662 anchor_analyses =
7663 anchor_analyses.saturating_add(u64::from(error.analysis_created));
7664 *source_report
7665 .fallback_reasons
7666 .entry(error.reason)
7667 .or_default() += 1;
7668 continue;
7669 }
7670 };
7671 let fragment_regions: Vec<_> = prepared
7672 .placements
7673 .iter()
7674 .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
7675 .collect();
7676 let crosses_fragment =
7677 prepared
7678 .placements
7679 .iter()
7680 .enumerate()
7681 .any(|(fragment_index, placement)| {
7682 placement
7683 .fragment_dependency_proof()
7684 .1
7685 .dependencies
7686 .iter()
7687 .any(|dependency| {
7688 fragment_regions.iter().enumerate().any(
7689 |(candidate_index, candidate)| {
7690 candidate_index != fragment_index
7691 && dependency.read_region.intersects(candidate)
7692 },
7693 )
7694 })
7695 });
7696 let family_records = replay_records
7697 .iter()
7698 .filter(|record| {
7699 record.family == Some(source.source_id)
7700 || record.partition_owner == Some(source.source_id)
7701 })
7702 .cloned()
7703 .collect::<Vec<_>>();
7704 if crosses_fragment {
7705 *source_report
7706 .fallback_reasons
7707 .entry("CrossFragmentDependency".to_string())
7708 .or_default() += 1;
7709 continue;
7710 }
7711 if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
7712 *source_report
7713 .fallback_reasons
7714 .entry(format!("TargetPartitionReplay:{error}"))
7715 .or_default() += 1;
7716 continue;
7717 }
7718 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
7719 let mut candidate = disposition.clone();
7720 if let Err(reason) = candidate.register_partition(source, true) {
7721 *source_report
7722 .fallback_reasons
7723 .entry(reason.to_string())
7724 .or_default() += 1;
7725 continue;
7726 }
7727 disposition = candidate;
7728 }
7729 direct_families = direct_families.saturating_add(1);
7730 direct_cells = direct_cells.saturating_add(prepared.direct_cells);
7731 direct_partition_families = direct_partition_families.saturating_add(1);
7732 direct_partition_cells =
7733 direct_partition_cells.saturating_add(prepared.direct_cells);
7734 direct_fragments =
7735 direct_fragments.saturating_add(prepared.placements.len() as u64);
7736 ordered_placements.push((source.source_order, prepared.placements));
7737 }
7738 }
7739 ordered_placements.sort_by_key(|(source_order, _)| *source_order);
7740 let placements = ordered_placements
7741 .into_iter()
7742 .flat_map(|(_, placements)| placements)
7743 .collect();
7744
7745 Ok(PreparedTargetSourcePackage {
7746 sheet: sheet.to_string(),
7747 sheet_id,
7748 lease,
7749 source_report,
7750 replay_records,
7751 disposition,
7752 placements,
7753 legacy: Vec::new(),
7754 direct_families,
7755 direct_cells,
7756 direct_fragments,
7757 direct_complete_families,
7758 direct_complete_cells,
7759 direct_partition_families,
7760 direct_partition_cells,
7761 anchor_parses,
7762 anchor_asts,
7763 anchor_analyses,
7764 })
7765 }
7766
7767 fn fallback_planning_snapshot(
7768 &self,
7769 batch: &FormulaIngestBatch,
7770 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7771 let mut requests = Vec::new();
7772 for formula in &batch.formulas {
7773 let ast = self
7774 .graph
7775 .data_store()
7776 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
7777 .ok_or_else(|| {
7778 ExcelError::new(ExcelErrorKind::Value)
7779 .with_message("ordered fallback AST is unavailable")
7780 })?;
7781 Self::collect_planning_function_requests(&ast, &mut requests);
7782 }
7783 requests.sort();
7784 requests.dedup();
7785 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7786 &self.resolver,
7787 requests,
7788 )
7789 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
7790 }
7791
7792 fn prepare_legacy_batch_fallback(
7793 &mut self,
7794 batch: FormulaIngestBatch,
7795 function_provider: &dyn crate::traits::FunctionProvider,
7796 ) -> Result<
7797 (
7798 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
7799 u64,
7800 ),
7801 ExcelError,
7802 > {
7803 let formula_count = batch.formulas.len() as u64;
7804 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7805 let mut planned = Vec::with_capacity(batch.formulas.len());
7806 for record in batch.formulas {
7807 let placement = CellRef::new(
7808 sheet_id,
7809 Coord::from_excel(record.row, record.col, true, true),
7810 );
7811 let ingested = self
7812 .graph
7813 .ingest_pipeline(function_provider)
7814 .enable_function_semantics()
7815 .ingest_formula(
7816 FormulaAstInput::RawArena(record.ast_id),
7817 placement,
7818 record.formula_text,
7819 )
7820 .map_err(|error| {
7821 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
7822 })?;
7823 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7824 }
7825 let plan = self
7826 .graph
7827 .prepare_legacy_graph_plan(sheet_id, planned)
7828 .map_err(|error| {
7829 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
7830 })?;
7831 Ok((plan, formula_count))
7832 }
7833
7834 fn publish_compressed_partial_report(
7835 &mut self,
7836 report: &FormulaIngestReport,
7837 direct_report: &FormulaIngestReport,
7838 ) {
7839 if direct_report.source_family_promoted == 0
7840 && direct_report.graph_formula_cells_materialized == 0
7841 {
7842 return;
7843 }
7844 let mut published = report.clone();
7845 published.accumulate(direct_report);
7846 self.record_formula_ingest_report(published);
7847 }
7848
7849 fn finish_compressed_formula_sources(
7850 &mut self,
7851 batches: Vec<(
7852 FormulaIngestBatch,
7853 crate::engine::FormulaCompressedSourceReport,
7854 crate::engine::FormulaCompressedPreparation,
7855 )>,
7856 ) -> Result<FormulaIngestReport, ExcelError> {
7857 self.observe_function_semantic_epoch()?;
7858 if batches.iter().any(|(_, _, preparation)| {
7859 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7860 }) {
7861 return Err(ExcelError::new(ExcelErrorKind::Value)
7862 .with_message("compressed source preparation belongs to another engine"));
7863 }
7864 if batches.iter().any(|(fallback, _, preparation)| {
7865 preparation.sheet_name.as_ref() != fallback.sheet_name
7866 }) {
7867 return Err(ExcelError::new(ExcelErrorKind::Value)
7868 .with_message("compressed source preparation sheet mismatch"));
7869 }
7870 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7871 let initial_provider_revision = self.resolver.planning_semantic_revision();
7872 let mut fallback_batches = Vec::with_capacity(batches.len());
7873 let mut stale_fallback_batches = Vec::new();
7874 let mut pending_preparations = Vec::new();
7875 for (mut fallback, mut source, mut preparation) in batches {
7876 for formula in fallback.formulas.drain(..) {
7877 let source_order = formula.source_order.ok_or_else(|| {
7878 ExcelError::new(ExcelErrorKind::Value).with_message(
7879 "compressed source supplied formulas without source-order proof",
7880 )
7881 })?;
7882 let text = formula.formula_text.ok_or_else(|| {
7883 ExcelError::new(ExcelErrorKind::Value).with_message(
7884 "ordered compressed fallback formula has no exact source text",
7885 )
7886 })?;
7887 preparation
7888 .eager_replay
7889 .push(crate::engine::DeferredReplayFormula {
7890 source_order,
7891 row: formula.row,
7892 col: formula.col,
7893 text: text.to_string(),
7894 family: formula.source_family,
7895 partition_owner: formula.partition_owner,
7896 });
7897 }
7898 preparation
7899 .eager_replay
7900 .sort_by_key(|record| record.source_order);
7901 source.source_spool_replays = source
7902 .source_spool_replays
7903 .saturating_add(preparation.preparation_spool_replays);
7904 let stale_reason = preparation
7905 .function_semantics_used
7906 .then(|| {
7907 if preparation.function_provider_revision != initial_provider_revision {
7908 Some("FunctionProviderRevisionChanged")
7909 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7910 {
7911 Some("FunctionSemanticEpochChanged")
7912 } else {
7913 None
7914 }
7915 })
7916 .flatten();
7917 let stale_semantics = stale_reason.is_some();
7918 if let Some(reason) = stale_reason {
7919 for (family, _, _) in &preparation.prepared {
7920 preparation.rejected.insert(*family, reason.to_string());
7921 }
7922 }
7923 for reason in preparation.rejected.values() {
7924 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7925 }
7926 let clean_direct_cells: u64 = preparation
7927 .prepared
7928 .iter()
7929 .map(|(_, _, prepared)| prepared.member_count)
7930 .sum();
7931 let fragmented_direct_cells: u64 = preparation
7932 .fragmented
7933 .iter()
7934 .map(|fragment| fragment.source.surviving_member_count)
7935 .sum();
7936 let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
7937 source.replay_families = source.families_seen.saturating_sub(
7938 preparation
7939 .prepared
7940 .len()
7941 .saturating_add(preparation.fragmented.len()) as u64,
7942 );
7943 source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
7944 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7945 fallback.sheet_name.clone(),
7946 source,
7947 );
7948 if stale_semantics {
7949 stale_fallback_batches.push((fallback, compressed));
7950 } else {
7951 fallback_batches.push((fallback, compressed));
7952 }
7953 pending_preparations.push((preparation, stale_semantics));
7954 }
7955 drop(initial_guard);
7956
7957 let configured_mode = self.config.formula_plane_mode;
7959 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7960 let stale_result =
7961 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7962 self.config.formula_plane_mode = configured_mode;
7963 let mut report = stale_result?;
7964 report.mode = configured_mode;
7965
7966 self.config.formula_plane_mode = FormulaPlaneMode::Off;
7967 let fallback_result =
7968 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7969 self.config.formula_plane_mode = configured_mode;
7970 match fallback_result {
7971 Ok(fallback_report) => report.accumulate(&fallback_report),
7972 Err(error) => {
7973 self.record_formula_ingest_report(report);
7974 return Err(error);
7975 }
7976 }
7977
7978 let mut direct_report =
7979 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7980 for (preparation, _) in &pending_preparations {
7981 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7982 preparation.clean_rejected_anchor_counts[0]
7983 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7984 );
7985 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7986 preparation.clean_rejected_anchor_counts[1]
7987 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7988 );
7989 direct_report.source_anchor_analyses =
7990 direct_report.source_anchor_analyses.saturating_add(
7991 preparation.clean_rejected_anchor_counts[2]
7992 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7993 );
7994 let rejected_partitions: Vec<_> = preparation
7995 .rejected
7996 .keys()
7997 .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
7998 .collect();
7999 direct_report.source_partitioned_families_rejected = direct_report
8000 .source_partitioned_families_rejected
8001 .saturating_add(rejected_partitions.len() as u64);
8002 direct_report.source_partition_failures = direct_report
8003 .source_partition_failures
8004 .saturating_add(rejected_partitions.len() as u64);
8005 direct_report.source_partition_fallback_cells = direct_report
8006 .source_partition_fallback_cells
8007 .saturating_add(
8008 rejected_partitions
8009 .iter()
8010 .map(|source| source.surviving_member_count)
8011 .sum::<u64>(),
8012 );
8013 }
8014 loop {
8015 #[cfg(any(test, feature = "test-support"))]
8016 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8017 hook();
8018 }
8019 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8020 let commit_provider_revision = self.resolver.planning_semantic_revision();
8021 let mut newly_stale = Vec::new();
8022 let mut current = Vec::new();
8023 for pending in pending_preparations.drain(..) {
8024 let stale_reason = pending
8025 .0
8026 .function_semantics_used
8027 .then(|| {
8028 if pending.0.function_provider_revision != commit_provider_revision {
8029 Some("FunctionProviderRevisionChanged")
8030 } else if self
8031 .prepared_function_semantics_changed(&pending.0, &commit_guard)
8032 {
8033 Some("FunctionSemanticEpochChanged")
8034 } else {
8035 None
8036 }
8037 })
8038 .flatten();
8039 if let Some(reason) = stale_reason {
8040 newly_stale.push((pending, reason));
8041 } else {
8042 current.push(pending);
8043 }
8044 }
8045 if !newly_stale.is_empty() {
8046 drop(commit_guard);
8047 for ((mut preparation, was_initially_stale), reason) in newly_stale {
8048 for (family, _, _) in &preparation.prepared {
8049 preparation.rejected.insert(*family, reason.to_string());
8050 }
8051 let exact = match self.replay_prepared_families_exact_records(&preparation) {
8052 Ok(exact) => exact,
8053 Err(error) => {
8054 self.publish_compressed_partial_report(&report, &direct_report);
8055 return Err(error);
8056 }
8057 };
8058 preparation.eager_replay.extend(exact);
8059 preparation
8060 .eager_replay
8061 .sort_by_key(|record| record.source_order);
8062 direct_report.source_spool_replays =
8063 direct_report.source_spool_replays.saturating_add(1);
8064 if !was_initially_stale {
8065 *direct_report
8066 .fallback_reasons
8067 .entry(reason.to_string())
8068 .or_default() += preparation.prepared.len() as u64;
8069 }
8070 direct_report.source_family_fallback = direct_report
8071 .source_family_fallback
8072 .saturating_add(preparation.prepared.len() as u64);
8073 direct_report.source_family_fallback_cells =
8074 direct_report.source_family_fallback_cells.saturating_add(
8075 preparation
8076 .prepared
8077 .iter()
8078 .map(|(_, _, prepared)| prepared.member_count)
8079 .sum(),
8080 );
8081 preparation.prepared.clear();
8082 preparation.function_semantics_used = false;
8083 if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8084 current.push((preparation, false));
8085 }
8086 }
8087 pending_preparations = current;
8088 continue;
8089 }
8090 drop(commit_guard);
8091
8092 enum SourceProposal {
8095 KnownFallback {
8096 source_order: crate::engine::SourceFormulaOrder,
8097 records: Vec<crate::engine::DeferredReplayFormula>,
8098 },
8099 Clean(
8100 crate::engine::SourceFamilyId,
8101 crate::engine::SourceFormulaOrder,
8102 crate::formula_plane::placement::PreparedAnchorOncePlacement,
8103 ),
8104 Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8105 }
8106
8107 impl SourceProposal {
8108 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8109 match self {
8110 Self::KnownFallback { source_order, .. } => *source_order,
8111 Self::Clean(_, source_order, _) => *source_order,
8112 Self::Fragment(fragment) => fragment.source.source_order,
8113 }
8114 }
8115 }
8116
8117 for (mut preparation, _) in current {
8118 let mut proposals = Vec::with_capacity(
8119 preparation
8120 .prepared
8121 .len()
8122 .saturating_add(preparation.fragmented.len())
8123 .saturating_add(preparation.eager_replay.len()),
8124 );
8125 proposals.extend(preparation.prepared.drain(..).map(
8126 |(source_id, source_order, prepared)| {
8127 SourceProposal::Clean(source_id, source_order, prepared)
8128 },
8129 ));
8130
8131 let accepted_fragments: BTreeMap<_, _> = preparation
8132 .fragmented
8133 .iter()
8134 .map(|fragment| (fragment.source.source_id, &fragment.source))
8135 .collect();
8136 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8137 for record in preparation.eager_replay.drain(..).filter(|record| {
8138 !record.partition_owner.is_some_and(|owner| {
8139 accepted_fragments.get(&owner).is_some_and(|source| {
8140 Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8141 })
8142 })
8143 }) {
8144 if let Some(owner) = record.partition_owner.or(record.family) {
8145 family_fallbacks.entry(owner).or_default().push(record);
8146 } else {
8147 proposals.push(SourceProposal::KnownFallback {
8148 source_order: record.source_order,
8149 records: vec![record],
8150 });
8151 }
8152 }
8153 for (_, mut records) in family_fallbacks {
8154 records.sort_by_key(|record| record.source_order);
8155 if records
8156 .windows(2)
8157 .any(|window| window[0].source_order == window[1].source_order)
8158 {
8159 self.publish_compressed_partial_report(&report, &direct_report);
8160 return Err(ExcelError::new(ExcelErrorKind::Value)
8161 .with_message("duplicate exact-replay source-order proof"));
8162 }
8163 let Some(source_order) = records.first().map(|record| record.source_order)
8164 else {
8165 self.publish_compressed_partial_report(&report, &direct_report);
8166 return Err(ExcelError::new(ExcelErrorKind::Value)
8167 .with_message("empty exact-replay fallback family"));
8168 };
8169 proposals.push(SourceProposal::KnownFallback {
8170 source_order,
8171 records,
8172 });
8173 }
8174 proposals.extend(
8175 preparation
8176 .fragmented
8177 .drain(..)
8178 .map(SourceProposal::Fragment),
8179 );
8180 proposals.sort_by_key(SourceProposal::source_order);
8181 if proposals
8182 .windows(2)
8183 .any(|window| window[0].source_order() == window[1].source_order())
8184 {
8185 self.publish_compressed_partial_report(&report, &direct_report);
8186 return Err(ExcelError::new(ExcelErrorKind::Value)
8187 .with_message("ambiguous compressed source-order proof"));
8188 }
8189
8190 for proposal in proposals {
8191 match proposal {
8192 SourceProposal::KnownFallback { records, .. } => {
8193 let batch = match self.formula_batch_from_exact_replay(
8194 preparation.sheet_name.as_ref(),
8195 records,
8196 ) {
8197 Ok(batch) => batch,
8198 Err(error) => {
8199 self.publish_compressed_partial_report(&report, &direct_report);
8200 return Err(error);
8201 }
8202 };
8203 if batch.is_empty() {
8204 continue;
8205 }
8206 let snapshot = match self.fallback_planning_snapshot(&batch) {
8207 Ok(snapshot) => snapshot,
8208 Err(error) => {
8209 self.publish_compressed_partial_report(&report, &direct_report);
8210 return Err(error);
8211 }
8212 };
8213 let commit_guard =
8214 crate::function_registry::semantic_epoch_read_guard();
8215 let provider_revision_initial =
8216 self.resolver.planning_semantic_revision();
8217 if (commit_guard.epoch() != snapshot.epoch()
8218 && snapshot.semantic_changes_affect_requests_since_guarded(
8219 &commit_guard,
8220 snapshot.epoch(),
8221 ))
8222 || snapshot.provider_revision().is_some_and(|revision| {
8223 Some(revision) != provider_revision_initial
8224 })
8225 {
8226 drop(commit_guard);
8227 self.publish_compressed_partial_report(&report, &direct_report);
8228 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8229 "ordered fallback planning snapshot became stale",
8230 ));
8231 }
8232 let (plan, formula_count) = match self
8233 .prepare_legacy_batch_fallback(batch, &snapshot)
8234 {
8235 Ok(plan) => plan,
8236 Err(error) => {
8237 drop(commit_guard);
8238 self.publish_compressed_partial_report(&report, &direct_report);
8239 return Err(error);
8240 }
8241 };
8242 let provider_revision_after =
8243 self.resolver.planning_semantic_revision();
8244 if provider_revision_after != provider_revision_initial {
8245 drop(commit_guard);
8246 self.publish_compressed_partial_report(&report, &direct_report);
8247 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8248 "function provider changed while preparing ordered fallback",
8249 ));
8250 }
8251 let graph_vertices = plan.new_vertex_count();
8252 let Some(graph_edges) = plan.planned_edge_count() else {
8253 drop(commit_guard);
8254 self.publish_compressed_partial_report(&report, &direct_report);
8255 return Err(ExcelError::new(ExcelErrorKind::Value)
8256 .with_message("prepared ordered fallback size overflow"));
8257 };
8258 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
8259 {
8260 drop(commit_guard);
8261 self.publish_compressed_partial_report(&report, &direct_report);
8262 return Err(error);
8263 }
8264 if let Err(error) =
8265 self.graph.validate_prepared_legacy_graph_plan(&plan)
8266 {
8267 drop(commit_guard);
8268 self.publish_compressed_partial_report(&report, &direct_report);
8269 return Err(ExcelError::new(ExcelErrorKind::Value)
8270 .with_message(error.to_string()));
8271 }
8272 #[cfg(test)]
8273 if let Some(hook) = self
8274 .before_legacy_fallback_final_provider_sample_hook
8275 .take()
8276 {
8277 hook();
8278 }
8279 let provider_revision_final =
8280 self.resolver.planning_semantic_revision();
8281 if provider_revision_final != provider_revision_initial {
8282 drop(commit_guard);
8283 self.publish_compressed_partial_report(&report, &direct_report);
8284 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8285 "function provider changed after ordered fallback validation",
8286 ));
8287 }
8288 let graph_formulas =
8289 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8290 direct_report.formula_cells_seen = direct_report
8291 .formula_cells_seen
8292 .saturating_add(formula_count);
8293 direct_report.graph_formula_cells_materialized = direct_report
8294 .graph_formula_cells_materialized
8295 .saturating_add(graph_formulas as u64);
8296 direct_report.graph_vertices_created = direct_report
8297 .graph_vertices_created
8298 .saturating_add(graph_vertices as u64);
8299 direct_report.graph_edges_created = direct_report
8300 .graph_edges_created
8301 .saturating_add(graph_edges as u64);
8302 }
8303 SourceProposal::Clean(source_id, _, prepared) => {
8304 let commit_guard =
8305 crate::function_registry::semantic_epoch_read_guard();
8306 let commit_provider_revision =
8307 self.resolver.planning_semantic_revision();
8308 let semantic_changed = preparation.function_semantics_used
8309 && self.prepared_placement_function_semantics_changed(
8310 preparation.function_semantic_epoch,
8311 &prepared,
8312 &commit_guard,
8313 );
8314 let cells = prepared.member_count;
8315 let stats = self.graph.baseline_stats();
8316 self.preflight_graph_admission(
8317 crate::engine::resource_ledger::GraphAdmission {
8318 final_vertices: stats.graph_vertex_count,
8319 final_edges: stats.graph_edge_count,
8320 materialization_cells: 0,
8321 added_vertices: 0,
8322 added_edges: 0,
8323 },
8324 )?;
8325 let append = self
8326 .graph
8327 .formula_authority()
8328 .prepare_formula_plane_append(
8329 vec![prepared],
8330 self.graph.data_store(),
8331 self.graph.sheet_reg(),
8332 )
8333 .map_err(|error| error.to_string())
8334 .and_then(|append| {
8335 if preparation.function_semantics_used
8336 && preparation.function_provider_revision
8337 != commit_provider_revision
8338 {
8339 return Err("FunctionProviderRevisionChanged".to_string());
8340 }
8341 if preparation.function_semantics_used
8342 && self.resolver.planning_semantic_revision()
8343 != commit_provider_revision
8344 {
8345 return Err("FunctionProviderRevisionChanged".to_string());
8346 }
8347 if semantic_changed {
8348 return Err("FunctionSemanticEpochChanged".to_string());
8349 }
8350 self.graph
8351 .formula_authority()
8352 .validate_prepared_formula_plane_append(
8353 &append,
8354 self.graph.data_store(),
8355 self.graph.sheet_reg(),
8356 )
8357 .map_err(|error| error.to_string())?;
8358 Ok(append)
8359 });
8360 match append {
8361 Ok(append) => {
8362 let placement_report = self
8363 .graph
8364 .formula_authority_mut()
8365 .apply_prevalidated_formula_plane_append(append);
8366 self.graph.mark_formula_spans_dirty(
8367 placement_report.spans.iter().copied(),
8368 WholeSpanDirtyReason::NewSpan,
8369 );
8370 direct_report.formula_cells_seen =
8371 direct_report.formula_cells_seen.saturating_add(cells);
8372 direct_report.shadow_candidate_cells =
8373 direct_report.shadow_candidate_cells.saturating_add(cells);
8374 direct_report.shadow_accepted_span_cells = direct_report
8375 .shadow_accepted_span_cells
8376 .saturating_add(cells);
8377 direct_report.shadow_templates_interned =
8378 direct_report.shadow_templates_interned.saturating_add(
8379 placement_report.work.templates_to_append as u64,
8380 );
8381 direct_report.shadow_spans_created = direct_report
8382 .shadow_spans_created
8383 .saturating_add(placement_report.spans.len() as u64);
8384 direct_report.graph_formula_vertices_avoided_shadow =
8385 direct_report
8386 .graph_formula_vertices_avoided_shadow
8387 .saturating_add(cells);
8388 direct_report.ast_roots_avoided_shadow = direct_report
8389 .ast_roots_avoided_shadow
8390 .saturating_add(cells.saturating_sub(1));
8391 direct_report.edge_rows_avoided_shadow = direct_report
8392 .edge_rows_avoided_shadow
8393 .saturating_add(cells);
8394 direct_report.source_family_promoted =
8395 direct_report.source_family_promoted.saturating_add(1);
8396 direct_report.source_family_promoted_cells = direct_report
8397 .source_family_promoted_cells
8398 .saturating_add(cells);
8399 direct_report.source_anchor_parses =
8400 direct_report.source_anchor_parses.saturating_add(1);
8401 direct_report.source_anchor_asts =
8402 direct_report.source_anchor_asts.saturating_add(1);
8403 direct_report.source_anchor_analyses =
8404 direct_report.source_anchor_analyses.saturating_add(1);
8405 let descendants = cells.saturating_sub(1);
8406 direct_report.source_descendant_strings_avoided = direct_report
8407 .source_descendant_strings_avoided
8408 .saturating_add(descendants);
8409 direct_report.source_descendant_events_avoided = direct_report
8410 .source_descendant_events_avoided
8411 .saturating_add(descendants);
8412 direct_report.source_descendant_analyses_avoided =
8413 direct_report
8414 .source_descendant_analyses_avoided
8415 .saturating_add(descendants);
8416 direct_report.source_compressed_families_prepared =
8417 direct_report
8418 .source_compressed_families_prepared
8419 .saturating_add(1);
8420 direct_report.source_compressed_cells_prepared = direct_report
8421 .source_compressed_cells_prepared
8422 .saturating_add(cells);
8423 }
8424 Err(error) => {
8425 drop(commit_guard);
8426 let reason = format!("CleanFormulaPlaneAppend:{error}");
8427 let batch = match self.replay_one_prepared_family_exact(
8428 &preparation,
8429 source_id,
8430 cells,
8431 ) {
8432 Ok(batch) => batch,
8433 Err(error) => {
8434 self.publish_compressed_partial_report(
8435 &report,
8436 &direct_report,
8437 );
8438 return Err(error);
8439 }
8440 };
8441 let snapshot = match self.fallback_planning_snapshot(&batch) {
8442 Ok(snapshot) => snapshot,
8443 Err(error) => {
8444 self.publish_compressed_partial_report(
8445 &report,
8446 &direct_report,
8447 );
8448 return Err(error);
8449 }
8450 };
8451 let commit_guard =
8452 crate::function_registry::semantic_epoch_read_guard();
8453 let provider_revision =
8454 self.resolver.planning_semantic_revision();
8455 if (commit_guard.epoch() != snapshot.epoch()
8456 && snapshot.semantic_changes_affect_requests_since_guarded(
8457 &commit_guard,
8458 snapshot.epoch(),
8459 ))
8460 || snapshot.provider_revision().is_some_and(|revision| {
8461 Some(revision) != provider_revision
8462 })
8463 {
8464 drop(commit_guard);
8465 self.publish_compressed_partial_report(
8466 &report,
8467 &direct_report,
8468 );
8469 return Err(ExcelError::new(ExcelErrorKind::Value)
8470 .with_message(
8471 "clean-family fallback planning snapshot became stale",
8472 ));
8473 }
8474 let (plan, formula_count) = match self
8475 .prepare_legacy_batch_fallback(batch, &snapshot)
8476 {
8477 Ok(plan) => plan,
8478 Err(error) => {
8479 drop(commit_guard);
8480 self.publish_compressed_partial_report(
8481 &report,
8482 &direct_report,
8483 );
8484 return Err(error);
8485 }
8486 };
8487 let provider_revision_after =
8488 self.resolver.planning_semantic_revision();
8489 if provider_revision_after != provider_revision {
8490 drop(commit_guard);
8491 self.publish_compressed_partial_report(
8492 &report,
8493 &direct_report,
8494 );
8495 return Err(ExcelError::new(ExcelErrorKind::Value)
8496 .with_message(
8497 "function provider changed while preparing clean-family fallback",
8498 ));
8499 }
8500 let graph_vertices = plan.new_vertex_count();
8501 let Some(graph_edges) = plan.planned_edge_count() else {
8502 drop(commit_guard);
8503 self.publish_compressed_partial_report(
8504 &report,
8505 &direct_report,
8506 );
8507 return Err(ExcelError::new(ExcelErrorKind::Value)
8508 .with_message(
8509 "prepared clean-family fallback size overflow",
8510 ));
8511 };
8512 if let Err(error) =
8513 self.prepared_legacy_admission(&plan, formula_count)
8514 {
8515 drop(commit_guard);
8516 self.publish_compressed_partial_report(
8517 &report,
8518 &direct_report,
8519 );
8520 return Err(error);
8521 }
8522 if let Err(error) =
8523 self.graph.validate_prepared_legacy_graph_plan(&plan)
8524 {
8525 drop(commit_guard);
8526 self.publish_compressed_partial_report(
8527 &report,
8528 &direct_report,
8529 );
8530 return Err(ExcelError::new(ExcelErrorKind::Value)
8531 .with_message(error.to_string()));
8532 }
8533 #[cfg(test)]
8534 if let Some(hook) = self
8535 .before_legacy_fallback_final_provider_sample_hook
8536 .take()
8537 {
8538 hook();
8539 }
8540 let provider_revision_final =
8541 self.resolver.planning_semantic_revision();
8542 if provider_revision_final != provider_revision {
8543 drop(commit_guard);
8544 self.publish_compressed_partial_report(
8545 &report,
8546 &direct_report,
8547 );
8548 return Err(ExcelError::new(ExcelErrorKind::Value)
8549 .with_message(
8550 "function provider changed after clean-family fallback validation",
8551 ));
8552 }
8553 let graph_formulas =
8554 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8555 direct_report.formula_cells_seen = direct_report
8556 .formula_cells_seen
8557 .saturating_add(formula_count);
8558 direct_report.graph_formula_cells_materialized = direct_report
8559 .graph_formula_cells_materialized
8560 .saturating_add(graph_formulas as u64);
8561 direct_report.graph_vertices_created = direct_report
8562 .graph_vertices_created
8563 .saturating_add(graph_vertices as u64);
8564 direct_report.graph_edges_created = direct_report
8565 .graph_edges_created
8566 .saturating_add(graph_edges as u64);
8567 direct_report.source_spool_replays =
8568 direct_report.source_spool_replays.saturating_add(1);
8569 direct_report.source_family_fallback =
8570 direct_report.source_family_fallback.saturating_add(1);
8571 direct_report.source_family_fallback_cells = direct_report
8572 .source_family_fallback_cells
8573 .saturating_add(cells);
8574 direct_report.source_anchor_parses =
8575 direct_report.source_anchor_parses.saturating_add(1);
8576 direct_report.source_anchor_asts =
8577 direct_report.source_anchor_asts.saturating_add(1);
8578 direct_report.source_anchor_analyses =
8579 direct_report.source_anchor_analyses.saturating_add(1);
8580 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8581 }
8582 }
8583 }
8584 SourceProposal::Fragment(fragment) => {
8585 let commit_guard =
8586 crate::function_registry::semantic_epoch_read_guard();
8587 let commit_epoch = commit_guard.epoch();
8588 let commit_provider_revision =
8589 self.resolver.planning_semantic_revision();
8590 let source = fragment.source;
8591 let source_id = source.source_id;
8592 #[cfg(test)]
8593 let commit_fault = self
8594 .fragmented_commit_fault_for_test
8595 .take()
8596 .unwrap_or(
8597 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
8598 );
8599 #[cfg(not(test))]
8600 let commit_fault =
8601 crate::engine::fragmented_transaction::FragmentedCommitFault::None;
8602 let outcome = self
8603 .prepare_fragmented_source_transaction(
8604 &source,
8605 &preparation.replay_disposition,
8606 fragment.prepared,
8607 fragment.legacy,
8608 )
8609 .map(|transaction| {
8610 let resolver = &self.resolver;
8611 self.graph.commit_fragmented_source_transaction(
8612 transaction,
8613 &self.source_formula_token,
8614 &preparation.replay_disposition,
8615 commit_epoch,
8616 commit_provider_revision,
8617 || resolver.planning_semantic_revision(),
8618 commit_fault,
8619 )
8620 });
8621 let fallback_reason = match outcome {
8622 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
8623 let mut delta = success.report_delta;
8624 delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
8625 delta.source_family_promoted = 1;
8626 delta.source_family_promoted_cells =
8627 source.surviving_member_count;
8628 delta.source_partitioned_families_seen = 0;
8629 delta.source_partition_holes = 0;
8630 delta.source_partition_ordinary_exceptions = 0;
8631 delta.source_partition_surviving_cells = 0;
8632 delta.shadow_templates_interned =
8633 success.plane.work.templates_to_append as u64;
8634 delta.shadow_spans_created = success.plane.spans.len() as u64;
8635 let descendants =
8636 source.surviving_member_count.saturating_sub(1);
8637 delta.source_descendant_strings_avoided = descendants;
8638 delta.source_descendant_events_avoided = descendants;
8639 delta.source_descendant_analyses_avoided = descendants;
8640 direct_report.accumulate(&delta);
8641 None
8642 }
8643 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
8644 Some(format!("{reason:?}"))
8645 }
8646 Err(error) => Some(format!("{error:?}")),
8647 };
8648 if let Some(reason) = fallback_reason {
8649 drop(commit_guard);
8650 let mut replay_disposition = preparation.replay_disposition.clone();
8651 replay_disposition.force_family_legacy(source_id);
8652 let replayed = (|| {
8653 let records = fragment
8654 .replay
8655 .lock()
8656 .map_err(|_| {
8657 ExcelError::new(ExcelErrorKind::Value).with_message(
8658 "compressed formula exact replay lock poisoned",
8659 )
8660 })?
8661 .replay_partitioned(
8662 &replay_disposition,
8663 std::slice::from_ref(&source),
8664 )
8665 .map_err(|message| {
8666 ExcelError::new(ExcelErrorKind::Value)
8667 .with_message(message)
8668 })?
8669 .into_iter()
8670 .filter(|record| {
8671 record.family == Some(source_id)
8672 || (record.family.is_none()
8673 && record.partition_owner == Some(source_id))
8674 })
8675 .collect::<Vec<_>>();
8676 Self::validate_whole_partition_replay(&source, &records)?;
8677 Ok::<_, ExcelError>(records)
8678 })();
8679 let replayed = match replayed {
8680 Ok(replayed) => replayed,
8681 Err(error) => {
8682 self.publish_compressed_partial_report(
8683 &report,
8684 &direct_report,
8685 );
8686 return Err(error);
8687 }
8688 };
8689 let batch = match self.formula_batch_from_exact_replay(
8690 preparation.sheet_name.as_ref(),
8691 replayed,
8692 ) {
8693 Ok(batch) => batch,
8694 Err(error) => {
8695 self.publish_compressed_partial_report(
8696 &report,
8697 &direct_report,
8698 );
8699 return Err(error);
8700 }
8701 };
8702 let snapshot = match self.fallback_planning_snapshot(&batch) {
8703 Ok(snapshot) => snapshot,
8704 Err(error) => {
8705 self.publish_compressed_partial_report(
8706 &report,
8707 &direct_report,
8708 );
8709 return Err(error);
8710 }
8711 };
8712 let commit_guard =
8713 crate::function_registry::semantic_epoch_read_guard();
8714 let provider_revision = self.resolver.planning_semantic_revision();
8715 if (commit_guard.epoch() != snapshot.epoch()
8716 && snapshot.semantic_changes_affect_requests_since_guarded(
8717 &commit_guard,
8718 snapshot.epoch(),
8719 ))
8720 || snapshot
8721 .provider_revision()
8722 .is_some_and(|revision| Some(revision) != provider_revision)
8723 {
8724 self.publish_compressed_partial_report(&report, &direct_report);
8725 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8726 format!(
8727 "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
8728 snapshot.epoch(),
8729 commit_guard.epoch(),
8730 snapshot.provider_revision(),
8731 provider_revision,
8732 ),
8733 ));
8734 }
8735 let (plan, formula_count) =
8736 match self.prepare_legacy_batch_fallback(batch, &snapshot) {
8737 Ok(plan) => plan,
8738 Err(error) => {
8739 drop(commit_guard);
8740 self.publish_compressed_partial_report(
8741 &report,
8742 &direct_report,
8743 );
8744 return Err(error);
8745 }
8746 };
8747 let provider_revision_after =
8748 self.resolver.planning_semantic_revision();
8749 if provider_revision_after != provider_revision {
8750 drop(commit_guard);
8751 self.publish_compressed_partial_report(&report, &direct_report);
8752 return Err(ExcelError::new(ExcelErrorKind::Value)
8753 .with_message(
8754 "function provider changed while preparing whole-family fallback",
8755 ));
8756 }
8757 let graph_vertices = plan.new_vertex_count();
8758 let Some(graph_edges) = plan.planned_edge_count() else {
8759 drop(commit_guard);
8760 self.publish_compressed_partial_report(&report, &direct_report);
8761 return Err(ExcelError::new(ExcelErrorKind::Value)
8762 .with_message(
8763 "prepared whole-family fallback size overflow",
8764 ));
8765 };
8766 if let Err(error) =
8767 self.prepared_legacy_admission(&plan, formula_count)
8768 {
8769 drop(commit_guard);
8770 self.publish_compressed_partial_report(&report, &direct_report);
8771 return Err(error);
8772 }
8773 if let Err(error) =
8774 self.graph.validate_prepared_legacy_graph_plan(&plan)
8775 {
8776 drop(commit_guard);
8777 self.publish_compressed_partial_report(&report, &direct_report);
8778 return Err(ExcelError::new(ExcelErrorKind::Value)
8779 .with_message(error.to_string()));
8780 }
8781 #[cfg(test)]
8782 if let Some(hook) = self
8783 .before_legacy_fallback_final_provider_sample_hook
8784 .take()
8785 {
8786 hook();
8787 }
8788 let provider_revision_final =
8789 self.resolver.planning_semantic_revision();
8790 if provider_revision_final != provider_revision {
8791 drop(commit_guard);
8792 self.publish_compressed_partial_report(&report, &direct_report);
8793 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8794 "function provider changed after whole-family fallback validation",
8795 ));
8796 }
8797 let graph_formulas =
8798 self.graph.apply_prevalidated_legacy_graph_plan(plan);
8799 direct_report.formula_cells_seen = direct_report
8800 .formula_cells_seen
8801 .saturating_add(formula_count);
8802 direct_report.graph_formula_cells_materialized = direct_report
8803 .graph_formula_cells_materialized
8804 .saturating_add(graph_formulas as u64);
8805 direct_report.graph_vertices_created = direct_report
8806 .graph_vertices_created
8807 .saturating_add(graph_vertices as u64);
8808 direct_report.graph_edges_created = direct_report
8809 .graph_edges_created
8810 .saturating_add(graph_edges as u64);
8811 direct_report.source_spool_replays =
8812 direct_report.source_spool_replays.saturating_add(1);
8813 direct_report.source_family_fallback =
8814 direct_report.source_family_fallback.saturating_add(1);
8815 direct_report.source_family_fallback_cells = direct_report
8816 .source_family_fallback_cells
8817 .saturating_add(source.surviving_member_count);
8818 direct_report.source_partitioned_families_rejected = direct_report
8819 .source_partitioned_families_rejected
8820 .saturating_add(1);
8821 direct_report.source_partition_failures =
8822 direct_report.source_partition_failures.saturating_add(1);
8823 direct_report.source_partition_fallback_cells = direct_report
8824 .source_partition_fallback_cells
8825 .saturating_add(source.surviving_member_count);
8826 direct_report.source_anchor_parses =
8827 direct_report.source_anchor_parses.saturating_add(1);
8828 direct_report.source_anchor_asts =
8829 direct_report.source_anchor_asts.saturating_add(1);
8830 direct_report.source_anchor_analyses =
8831 direct_report.source_anchor_analyses.saturating_add(1);
8832 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8833 }
8834 }
8835 }
8836 #[cfg(test)]
8837 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
8838 hook();
8839 }
8840 }
8841 }
8842 break;
8843 }
8844 report.accumulate(&direct_report);
8845 self.record_formula_ingest_report(report.clone());
8846 Ok(report)
8847 }
8848 pub(crate) fn ingest_compressed_formula_source_batches(
8850 &mut self,
8851 batches: Vec<(
8852 FormulaIngestBatch,
8853 crate::engine::FormulaCompressedSourceBatch,
8854 )>,
8855 ) -> Result<FormulaIngestReport, ExcelError> {
8856 self.ingest_compressed_formula_source_batches_inner(batches, true)
8857 }
8858
8859 fn ingest_compressed_formula_source_batches_inner(
8860 &mut self,
8861 batches: Vec<(
8862 FormulaIngestBatch,
8863 crate::engine::FormulaCompressedSourceBatch,
8864 )>,
8865 publish_report: bool,
8866 ) -> Result<FormulaIngestReport, ExcelError> {
8867 let mut source_counts = [0_u64; 11];
8868 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
8869 let mut formula_batches = Vec::with_capacity(batches.len());
8870 let mut compressed_families = Vec::new();
8871 let mut partitioned_families = Vec::new();
8872 for (batch, compressed_batch) in batches {
8873 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
8874 if sheet_name.as_ref() != batch.sheet_name {
8875 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8876 "compressed formula source sheet does not match its replay batch",
8877 ));
8878 }
8879 compressed_families.push((batch.sheet_name.clone(), families));
8880 partitioned_families.push((batch.sheet_name.clone(), partitions));
8881 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
8882 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
8883 source_counts[2] =
8884 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
8885 source_counts[3] =
8886 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
8887 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
8888 source_counts[5] =
8889 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
8890 source_counts[6] =
8891 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
8892 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
8893 source_counts[8] =
8894 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
8895 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
8896 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
8897 source_report.families_seen = source_report
8898 .families_seen
8899 .saturating_add(compressed.families_seen);
8900 source_report.family_cells_seen = source_report
8901 .family_cells_seen
8902 .saturating_add(compressed.family_cells_seen);
8903 source_report.source_clean_families = source_report
8904 .source_clean_families
8905 .saturating_add(compressed.source_clean_families);
8906 source_report.source_clean_cells = source_report
8907 .source_clean_cells
8908 .saturating_add(compressed.source_clean_cells);
8909 source_report.source_fragmentable_families = source_report
8910 .source_fragmentable_families
8911 .saturating_add(compressed.source_fragmentable_families);
8912 source_report.source_fragmentable_cells = source_report
8913 .source_fragmentable_cells
8914 .saturating_add(compressed.source_fragmentable_cells);
8915 source_report.source_fragment_count = source_report
8916 .source_fragment_count
8917 .saturating_add(compressed.source_fragment_count);
8918 source_report.source_isolated_fallback_cells = source_report
8919 .source_isolated_fallback_cells
8920 .saturating_add(compressed.source_isolated_fallback_cells);
8921 source_report.source_hole_exclusions = source_report
8922 .source_hole_exclusions
8923 .saturating_add(compressed.source_hole_exclusions);
8924 source_report.source_ordinary_exclusions = source_report
8925 .source_ordinary_exclusions
8926 .saturating_add(compressed.source_ordinary_exclusions);
8927 source_report.source_partition_failures = source_report
8928 .source_partition_failures
8929 .saturating_add(compressed.source_partition_failures);
8930 source_report.replay_families = source_report
8931 .replay_families
8932 .saturating_add(compressed.replay_families);
8933 source_report.replay_cells = source_report
8934 .replay_cells
8935 .saturating_add(compressed.replay_cells);
8936 source_report.forward_descendants = source_report
8937 .forward_descendants
8938 .saturating_add(compressed.forward_descendants);
8939 source_report.evidence_limit_fallbacks = source_report
8940 .evidence_limit_fallbacks
8941 .saturating_add(compressed.evidence_limit_fallbacks);
8942 source_report.evidence_peak_bytes = source_report
8943 .evidence_peak_bytes
8944 .max(compressed.evidence_peak_bytes);
8945 for (reason, count) in compressed.fallback_reasons {
8946 *source_report.fallback_reasons.entry(reason).or_default() += count;
8947 }
8948 formula_batches.push(batch);
8949 }
8950 self.ingest_formula_batches_inner(
8951 formula_batches,
8952 source_counts,
8953 Some(source_report),
8954 compressed_families,
8955 partitioned_families,
8956 publish_report,
8957 )
8958 }
8959
8960 pub fn ingest_formula_batches(
8961 &mut self,
8962 batches: Vec<FormulaIngestBatch>,
8963 ) -> Result<FormulaIngestReport, ExcelError> {
8964 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
8965 let report = self.ingest_formula_batches_inner(
8966 batches,
8967 [0; 11],
8968 None,
8969 Vec::new(),
8970 Vec::new(),
8971 true,
8972 )?;
8973 if has_formulas {
8974 self.mark_topology_edited();
8975 }
8976 Ok(report)
8977 }
8978
8979 fn ingest_formula_batches_unpublished(
8980 &mut self,
8981 batches: Vec<FormulaIngestBatch>,
8982 ) -> Result<FormulaIngestReport, ExcelError> {
8983 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
8984 }
8985
8986 fn ingest_formula_batches_inner(
8987 &mut self,
8988 batches: Vec<FormulaIngestBatch>,
8989 source_counts: [u64; 11],
8990 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
8991 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
8992 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
8993 publish_report: bool,
8994 ) -> Result<FormulaIngestReport, ExcelError> {
8995 self.observe_function_semantic_epoch()?;
8996 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
8997 #[cfg(feature = "benchmark_internal")]
8998 let benchmark_forced_replay =
8999 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9000 #[cfg(not(feature = "benchmark_internal"))]
9001 let benchmark_forced_replay = false;
9002 let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9003 && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9004 {
9005 let mut report =
9006 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9007 report
9008 .fallback_reasons
9009 .insert("ForcedReplay".to_string(), formula_cells_seen);
9010 (report, batches, BTreeMap::new())
9011 } else {
9012 match self.config.formula_plane_mode {
9013 FormulaPlaneMode::Off => (
9014 FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9015 batches,
9016 BTreeMap::new(),
9017 ),
9018 FormulaPlaneMode::Shadow => (
9019 if partitioned_families
9020 .iter()
9021 .all(|(_, families)| families.is_empty())
9022 {
9023 if compressed_families.is_empty() {
9024 self.analyze_formula_plane_shadow_candidates(&batches)
9025 } else {
9026 self.analyze_compressed_formula_plane_shadow(&compressed_families)
9027 }
9028 } else {
9029 let mut partition_report =
9030 self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9031 if compressed_families
9032 .iter()
9033 .any(|(_, families)| !families.is_empty())
9034 {
9035 partition_report.accumulate(
9036 &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9037 );
9038 }
9039 partition_report
9040 },
9041 batches,
9042 BTreeMap::new(),
9043 ),
9044 FormulaPlaneMode::AuthoritativeExperimental => {
9045 self.analyze_formula_plane_authoritative_ingest(&batches)
9046 }
9047 }
9048 };
9049 report.formula_cells_seen = formula_cells_seen;
9050 report.source_formula_events = source_counts[0];
9051 report.source_ordinary_events = source_counts[1];
9052 report.source_shared_anchor_events = source_counts[2];
9053 report.source_shared_descendant_events = source_counts[3];
9054 report.source_unknown_events = source_counts[4];
9055 report.source_formula_records_spooled = source_counts[5];
9056 report.source_spool_encoded_bytes = source_counts[6];
9057 report.source_spool_peak_memory_bytes = source_counts[7];
9058 report.source_spool_spilled_bytes = source_counts[8];
9059 report.source_spool_spill_files = source_counts[9];
9060 report.source_spool_replays = source_counts[10];
9061 if let Some(source) = source_report {
9062 report.source_families_seen = source.families_seen;
9063 report.source_family_cells_seen = source.family_cells_seen;
9064 report.source_family_shadow_eligible = source.source_clean_families;
9065 report.source_family_shadow_eligible_cells = source.source_clean_cells;
9066 report.source_partitioned_families_seen = source.source_fragmentable_families;
9067 report.source_partition_holes = source.source_hole_exclusions;
9068 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9069 report.source_partition_failures = source.source_partition_failures;
9070 report.source_partition_surviving_cells = source.source_fragmentable_cells;
9071 report.source_family_fallback = report
9072 .source_family_fallback
9073 .saturating_add(source.replay_families);
9074 report.source_family_fallback_cells = report
9075 .source_family_fallback_cells
9076 .saturating_add(source.replay_cells);
9077 report.source_forward_descendants = source.forward_descendants;
9078 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9079 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9080 for (reason, count) in source.fallback_reasons {
9081 let total = report.fallback_reasons.entry(reason).or_default();
9082 *total = total.saturating_add(count);
9083 }
9084 }
9085
9086 if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9087 || !planned_materialize.is_empty()
9088 {
9089 let mut prepared_by_sheet = planned_materialize;
9090 for batch in materialize_batches {
9091 if batch.is_empty() {
9092 continue;
9093 }
9094 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9095 ExcelError::new(ExcelErrorKind::Ref)
9096 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9097 })?;
9098 let mut pipeline = self.ingest_pipeline();
9099 let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9100 let placement = CellRef::new(
9101 sheet_id,
9102 Coord::from_excel(record.row, record.col, true, true),
9103 );
9104 (
9105 FormulaAstInput::RawArena(record.ast_id),
9106 placement,
9107 record.formula_text,
9108 )
9109 }))?;
9110 prepared_by_sheet
9111 .entry(batch.sheet_name)
9112 .or_default()
9113 .extend(ingested.into_iter().map(|formula| {
9114 (
9115 formula.placement.coord.row() + 1,
9116 formula.placement.coord.col() + 1,
9117 formula.ast_id,
9118 formula.dep_plan,
9119 )
9120 }));
9121 }
9122 let admission_preflighted = self.graph_admission_enabled();
9123 if admission_preflighted {
9124 let mut preview = Vec::new();
9125 for (sheet_name, formulas) in &prepared_by_sheet {
9126 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9127 ExcelError::new(ExcelErrorKind::Ref)
9128 .with_message(format!("unknown ingest sheet: {sheet_name}"))
9129 })?;
9130 preview.extend(
9131 formulas
9132 .iter()
9133 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9134 );
9135 }
9136 let admission = self.graph.preview_formula_mutations(&preview)?;
9137 self.preflight_graph_admission(admission)?;
9138 }
9139
9140 let mut builder = self.begin_bulk_ingest();
9141 if admission_preflighted {
9142 builder.mark_admission_preflighted();
9143 }
9144 for (sheet_name, formulas) in prepared_by_sheet {
9145 if formulas.is_empty() {
9146 continue;
9147 }
9148 let sheet_id = builder.add_sheet(&sheet_name);
9149 builder.add_formula_plans(sheet_id, formulas);
9150 }
9151 let summary = builder.finish()?;
9152 report.graph_formula_cells_materialized = summary.formulas as u64;
9153 report.graph_vertices_created = summary.vertices as u64;
9154 report.graph_edges_created = summary.edges as u64;
9155 }
9156
9157 if publish_report {
9158 self.record_formula_ingest_report(report.clone());
9159 }
9160 Ok(report)
9161 }
9162
9163 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9164 let mut unique = Vec::new();
9165 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9166 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9167 prior.sheet == diagnostic.sheet
9168 && prior.row == diagnostic.row
9169 && prior.col == diagnostic.col
9170 && prior.formula == diagnostic.formula
9171 && prior.policy == diagnostic.policy
9172 });
9173 if !duplicate {
9174 unique.push(diagnostic);
9175 }
9176 }
9177 self.formula_parse_diagnostics.extend(unique);
9178 }
9179
9180 pub fn handle_formula_parse_error(
9181 &mut self,
9182 sheet: &str,
9183 row: u32,
9184 col: u32,
9185 formula: &str,
9186 message: String,
9187 ) -> Result<Option<ASTNode>, ExcelError> {
9188 let policy = self.config.formula_parse_policy;
9189
9190 if policy == FormulaParsePolicy::Strict {
9191 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9192 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9193 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9194 )));
9195 }
9196
9197 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9198 sheet: sheet.to_string(),
9199 row,
9200 col,
9201 formula: formula.to_string(),
9202 message: message.clone(),
9203 policy,
9204 });
9205
9206 match policy {
9207 FormulaParsePolicy::Strict => unreachable!(),
9208 FormulaParsePolicy::KeepCachedValue => Ok(None),
9209 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9210 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9211 None,
9212 ))),
9213 FormulaParsePolicy::CoerceToError => {
9214 let err = ExcelError::new(ExcelErrorKind::Error)
9215 .with_message(format!("Malformed formula: {message}"));
9216 Ok(Some(ASTNode::new(
9217 ASTNodeType::Literal(LiteralValue::Error(err)),
9218 None,
9219 )))
9220 }
9221 }
9222 }
9223
9224 #[cfg(test)]
9225 fn target_preparation_fault(
9226 &mut self,
9227 seam: crate::engine::target_preparation::TargetPreparationFault,
9228 ) -> Result<(), ExcelError> {
9229 if self.target_preparation_fault_for_test == Some(seam) {
9230 self.target_preparation_fault_for_test = None;
9231 Err(ExcelError::new(ExcelErrorKind::Value)
9232 .with_message(format!("injected target preparation fault: {seam:?}")))
9233 } else {
9234 Ok(())
9235 }
9236 }
9237
9238 fn preparation_stale(
9239 reason: formualizer_common::PreparationStaleReason,
9240 message: impl Into<String>,
9241 ) -> ExcelError {
9242 ExcelError::new(ExcelErrorKind::Value)
9243 .with_message(message)
9244 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
9245 }
9246
9247 fn preparation_revision_stale_reason(
9248 assumptions: &crate::engine::PreparationRevision,
9249 current: &crate::engine::PreparationRevision,
9250 planning_requests: &BTreeSet<(String, String, usize)>,
9251 staged_leases_match: bool,
9252 ) -> Option<formualizer_common::PreparationStaleReason> {
9253 if assumptions.graph != current.graph {
9254 Some(formualizer_common::PreparationStaleReason::Graph)
9255 } else if assumptions.authority != current.authority
9256 || assumptions.authority_indexes != current.authority_indexes
9257 || assumptions.authority_indexed_plane != current.authority_indexed_plane
9258 {
9259 Some(formualizer_common::PreparationStaleReason::Authority)
9260 } else if assumptions.staged != current.staged || !staged_leases_match {
9261 Some(formualizer_common::PreparationStaleReason::Staged)
9262 } else if assumptions.symbols != current.symbols {
9263 Some(formualizer_common::PreparationStaleReason::Symbols)
9264 } else if assumptions.provider != current.provider {
9265 Some(formualizer_common::PreparationStaleReason::Provider)
9266 } else if assumptions.semantic != current.semantic
9267 && crate::function_registry::semantic_changes_affect_requests_since(
9268 assumptions.semantic,
9269 planning_requests.iter().cloned(),
9270 )
9271 {
9272 Some(formualizer_common::PreparationStaleReason::Semantic)
9273 } else {
9274 None
9275 }
9276 }
9277
9278 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
9279 let (authority, authority_indexes, authority_indexed_plane) =
9280 self.graph.authority_revisions();
9281 crate::engine::PreparationRevision {
9282 graph: self.graph.topology_revision(),
9283 authority,
9284 authority_indexes,
9285 authority_indexed_plane,
9286 staged: self.staged_formula_index.revision(),
9287 symbols: self.graph.symbol_revision(),
9288 semantic: crate::function_registry::semantic_epoch(),
9289 provider: self.resolver.planning_semantic_revision(),
9290 }
9291 }
9292
9293 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
9294 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
9295 let provider = self.resolver.planning_semantic_revision();
9296 let semantic = registry_guard.epoch();
9297 let (authority, authority_indexes, authority_indexed_plane) =
9298 self.graph.authority_revisions();
9299 let mut span_refs = self.graph.formula_authority().active_span_refs();
9300 span_refs.sort_unstable_by_key(|span_ref| {
9301 (span_ref.id.0, span_ref.generation, span_ref.version)
9302 });
9303 PlanningRevisionSnapshot {
9304 engine_topology_epoch: self.topology_epoch,
9305 graph_topology_revision: self.graph.topology_revision(),
9306 authority,
9307 authority_indexes,
9308 authority_indexed_plane,
9309 staged: self.staged_formula_index.revision(),
9310 symbols: self.graph.symbol_revision(),
9311 semantic,
9312 provider,
9313 formula_plane_mode: self.config.formula_plane_mode,
9314 deterministic_mode: self.config.deterministic_mode.clone(),
9315 budgets: self.evaluation_resource_budgets.clone(),
9316 span_refs,
9317 }
9318 }
9319
9320 fn recalc_plan_key(&self) -> RecalcPlanKey {
9321 RecalcPlanKey {
9322 engine_token: Arc::clone(&self.recalc_plan_token),
9323 revisions: self.planning_revision_snapshot(),
9324 }
9325 }
9326
9327 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
9328 ExcelError::new(ExcelErrorKind::Value)
9329 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
9330 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
9331 }
9332
9333 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
9334 use formualizer_common::PlanStaleReason;
9335
9336 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
9337 return Err(Self::plan_stale(PlanStaleReason::Engine));
9338 }
9339
9340 let current = self.planning_revision_snapshot();
9341 let expected = &key.revisions;
9342 let stale = if expected.provider != current.provider {
9343 Some(PlanStaleReason::Provider)
9344 } else if expected.semantic != current.semantic {
9345 Some(PlanStaleReason::Semantic)
9346 } else if expected.budgets != current.budgets
9347 || expected.deterministic_mode != current.deterministic_mode
9348 {
9349 Some(PlanStaleReason::Budget)
9350 } else if expected.staged != current.staged {
9351 Some(PlanStaleReason::Staged)
9352 } else if expected.symbols != current.symbols {
9353 Some(PlanStaleReason::Symbols)
9354 } else if expected.formula_plane_mode != current.formula_plane_mode
9355 || expected.authority != current.authority
9356 || expected.authority_indexes != current.authority_indexes
9357 || expected.authority_indexed_plane != current.authority_indexed_plane
9358 {
9359 Some(PlanStaleReason::Authority)
9360 } else if expected.span_refs != current.span_refs {
9361 Some(PlanStaleReason::SpanGeneration)
9362 } else if expected.graph_topology_revision != current.graph_topology_revision
9363 || expected.engine_topology_epoch != current.engine_topology_epoch
9364 {
9365 Some(PlanStaleReason::Graph)
9366 } else {
9367 None
9368 };
9369 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
9370 }
9371
9372 fn target_preparation_checkpoint(
9373 &mut self,
9374 deadline: Option<std::time::Instant>,
9375 work: u64,
9376 ) -> Result<(), ExcelError> {
9377 if self
9378 .active_cancel_flag
9379 .as_ref()
9380 .is_some_and(|cancel| cancel.is_cancelled())
9381 {
9382 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
9383 .with_message("target graph preparation cancelled"));
9384 }
9385 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
9386 return Err(crate::engine::ResourceLedgerError::Exhausted(
9387 formualizer_common::ResourceExhaustionDetail {
9388 reason: formualizer_common::ResourceExhaustionReason::Deadline,
9389 limit: 0,
9390 observed: 1,
9391 request_id: self
9392 .active_evaluation_resource_request
9393 .as_ref()
9394 .map(|stats| stats.request_id),
9395 },
9396 )
9397 .into_excel_error());
9398 }
9399 self.charge_bounded_work(work)
9400 }
9401
9402 fn opaque_reason_in_ast(
9403 &self,
9404 ast: &ASTNode,
9405 provider: &dyn crate::traits::FunctionProvider,
9406 ) -> Option<crate::engine::OpaqueReason> {
9407 match &ast.node_type {
9408 ASTNodeType::Function { name, args } => {
9409 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
9410 if canonical == "INDIRECT" {
9411 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
9412 }
9413 let Some(function) = provider.get_function_for_planning("", name) else {
9414 return Some(crate::engine::OpaqueReason::UnknownFunction);
9415 };
9416 let caps = function.caps();
9417 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9418 || caps.contains(FnCaps::RETURNS_REFERENCE)
9419 {
9420 return Some(crate::engine::OpaqueReason::DynamicReference);
9421 }
9422 args.iter()
9423 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
9424 }
9425 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
9426 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
9427 ASTNodeType::BinaryOp { left, right, .. } => self
9428 .opaque_reason_in_ast(left, provider)
9429 .or_else(|| self.opaque_reason_in_ast(right, provider)),
9430 ASTNodeType::Array(rows) => rows
9431 .iter()
9432 .flat_map(|row| row.iter())
9433 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
9434 ASTNodeType::Reference {
9435 reference:
9436 ReferenceType::Cell {
9437 sheet: Some(sheet), ..
9438 }
9439 | ReferenceType::Range {
9440 sheet: Some(sheet), ..
9441 },
9442 ..
9443 } if self.graph.sheet_id(sheet).is_none() => {
9444 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
9445 }
9446 ASTNodeType::Reference {
9447 reference:
9448 ReferenceType::External(_)
9449 | ReferenceType::Cell3D { .. }
9450 | ReferenceType::Range3D { .. },
9451 ..
9452 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
9453 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
9454 }
9455 }
9456
9457 fn target_planning_snapshot(
9458 &mut self,
9459 ast: &ASTNode,
9460 planning_requests: &mut BTreeSet<(String, String, usize)>,
9461 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
9462 #[cfg(test)]
9463 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
9464 return Err(Self::preparation_stale(
9465 formualizer_common::PreparationStaleReason::Semantic,
9466 "injected target semantic preparation movement",
9467 ));
9468 }
9469 #[cfg(test)]
9470 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
9471 hook();
9472 }
9473 let mut requests = Vec::new();
9474 Self::collect_planning_function_requests(ast, &mut requests);
9475 requests.sort();
9476 requests.dedup();
9477 planning_requests.extend(requests.iter().cloned());
9478 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
9479 &self.resolver,
9480 requests,
9481 )
9482 .map_err(|error| {
9483 ExcelError::new(ExcelErrorKind::Value)
9484 .with_message(format!("target planning snapshot unavailable: {error:?}"))
9485 })
9486 }
9487
9488 fn target_planning_snapshot_stale_reason(
9489 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
9490 assumptions: &crate::engine::PreparationRevision,
9491 ) -> Option<formualizer_common::PreparationStaleReason> {
9492 if snapshot
9493 .provider_revision()
9494 .is_some_and(|revision| Some(revision) != assumptions.provider)
9495 {
9496 Some(formualizer_common::PreparationStaleReason::Provider)
9497 } else if snapshot.epoch() != assumptions.semantic
9498 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
9499 {
9500 Some(formualizer_common::PreparationStaleReason::Semantic)
9501 } else {
9502 None
9503 }
9504 }
9505
9506 fn widen_target_preparation(
9507 policy: crate::engine::OpaquePreparePolicy,
9508 scope: &mut crate::engine::PrepareScope,
9509 reasons: &mut Vec<crate::engine::OpaqueReason>,
9510 reason: crate::engine::OpaqueReason,
9511 ) -> Result<bool, ExcelError> {
9512 if policy == crate::engine::OpaquePreparePolicy::Error {
9513 return Err(ExcelError::new(ExcelErrorKind::NImpl)
9514 .with_message(format!("opaque target preparation semantics: {reason:?}")));
9515 }
9516 if !reasons.contains(&reason) {
9517 reasons.push(reason);
9518 }
9519 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
9520 *scope = crate::engine::PrepareScope::Workbook;
9521 Ok(true)
9522 } else {
9523 Ok(false)
9524 }
9525 }
9526
9527 fn widen_target_preparation_to_sheet(
9528 policy: crate::engine::OpaquePreparePolicy,
9529 scope: &mut crate::engine::PrepareScope,
9530 reasons: &mut Vec<crate::engine::OpaqueReason>,
9531 reason: crate::engine::OpaqueReason,
9532 sheet: &str,
9533 ) -> Result<bool, ExcelError> {
9534 if policy == crate::engine::OpaquePreparePolicy::Error {
9535 return Err(ExcelError::new(ExcelErrorKind::NImpl)
9536 .with_message(format!("opaque target preparation semantics: {reason:?}")));
9537 }
9538 if !reasons.contains(&reason) {
9539 reasons.push(reason);
9540 }
9541 match scope {
9542 crate::engine::PrepareScope::Exact => {
9543 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
9544 Ok(true)
9545 }
9546 crate::engine::PrepareScope::Sheets(sheets) => {
9547 if sheets.iter().any(|candidate| candidate == sheet) {
9548 Ok(false)
9549 } else {
9550 sheets.push(sheet.to_string());
9551 sheets.sort();
9552 Ok(true)
9553 }
9554 }
9555 crate::engine::PrepareScope::Workbook => Ok(false),
9556 }
9557 }
9558
9559 fn ast_has_proven_sheet_local_dynamic(
9560 ast: &ASTNode,
9561 provider: &dyn crate::traits::FunctionProvider,
9562 ) -> bool {
9563 fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
9564 match &ast.node_type {
9565 ASTNodeType::Function { name, args } => {
9566 let Some(function) = provider.get_function_for_planning("", name) else {
9567 return (false, false);
9568 };
9569 let caps = function.caps();
9570 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9571 || caps.contains(FnCaps::RETURNS_REFERENCE);
9572 let canonical = name.rsplit('.').next().unwrap_or(name);
9573 if dynamic
9574 && !canonical.eq_ignore_ascii_case("OFFSET")
9575 && !canonical.eq_ignore_ascii_case("INDEX")
9576 {
9577 return (false, true);
9578 }
9579 let mut has_dynamic = dynamic;
9580 for arg in args {
9581 let (safe, child_dynamic) = classify(arg, provider);
9582 if !safe {
9583 return (false, has_dynamic || child_dynamic);
9584 }
9585 has_dynamic |= child_dynamic;
9586 }
9587 (true, has_dynamic)
9588 }
9589 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
9590 ASTNodeType::BinaryOp { left, right, .. } => {
9591 let (left_safe, left_dynamic) = classify(left, provider);
9592 let (right_safe, right_dynamic) = classify(right, provider);
9593 (left_safe && right_safe, left_dynamic || right_dynamic)
9594 }
9595 ASTNodeType::Array(rows) => {
9596 let mut has_dynamic = false;
9597 for item in rows.iter().flatten() {
9598 let (safe, child_dynamic) = classify(item, provider);
9599 if !safe {
9600 return (false, has_dynamic || child_dynamic);
9601 }
9602 has_dynamic |= child_dynamic;
9603 }
9604 (true, has_dynamic)
9605 }
9606 ASTNodeType::Reference {
9607 reference:
9608 ReferenceType::Cell { sheet: None, .. }
9609 | ReferenceType::Range { sheet: None, .. },
9610 ..
9611 }
9612 | ASTNodeType::Literal(_)
9613 | ASTNodeType::Omitted => (true, false),
9614 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
9615 }
9616 }
9617
9618 let (safe, dynamic) = classify(ast, provider);
9619 safe && dynamic
9620 }
9621
9622 fn table_selection_region(
9623 &self,
9624 entry: &crate::engine::graph::TableEntry,
9625 selection: &crate::engine::TableSelection,
9626 ) -> Result<PreparationRegion, ExcelError> {
9627 let mut start_row = entry.range.start.coord.row() + 1;
9628 let mut end_row = entry.range.end.coord.row() + 1;
9629 let mut start_col = entry.range.start.coord.col() + 1;
9630 let mut end_col = entry.range.end.coord.col() + 1;
9631 match selection {
9632 crate::engine::TableSelection::Whole => {}
9633 crate::engine::TableSelection::Headers => {
9634 if !entry.header_row {
9635 return Err(ExcelError::new(ExcelErrorKind::Value)
9636 .with_message(format!("table {} has no header row", entry.name)));
9637 }
9638 end_row = start_row;
9639 }
9640 crate::engine::TableSelection::Data => {
9641 if entry.header_row {
9642 start_row = start_row.saturating_add(1);
9643 }
9644 if entry.totals_row {
9645 end_row = end_row.saturating_sub(1);
9646 }
9647 }
9648 crate::engine::TableSelection::Totals => {
9649 if !entry.totals_row {
9650 return Err(ExcelError::new(ExcelErrorKind::Value)
9651 .with_message(format!("table {} has no totals row", entry.name)));
9652 }
9653 start_row = end_row;
9654 }
9655 crate::engine::TableSelection::Column(column) => {
9656 let index = entry.col_index(column).ok_or_else(|| {
9657 ExcelError::new(ExcelErrorKind::Name)
9658 .with_message(format!("unknown table column: {column}"))
9659 })?;
9660 start_col = start_col.saturating_add(index as u32);
9661 end_col = start_col;
9662 }
9663 crate::engine::TableSelection::Columns { start, end } => {
9664 let first = entry.col_index(start).ok_or_else(|| {
9665 ExcelError::new(ExcelErrorKind::Name)
9666 .with_message(format!("unknown table column: {start}"))
9667 })?;
9668 let last = entry.col_index(end).ok_or_else(|| {
9669 ExcelError::new(ExcelErrorKind::Name)
9670 .with_message(format!("unknown table column: {end}"))
9671 })?;
9672 if first > last {
9673 return Err(ExcelError::new(ExcelErrorKind::Value)
9674 .with_message("table column selection is reversed"));
9675 }
9676 start_col = start_col.saturating_add(first as u32);
9677 end_col = entry
9678 .range
9679 .start
9680 .coord
9681 .col()
9682 .saturating_add(last as u32)
9683 .saturating_add(1);
9684 }
9685 }
9686 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
9687 start_row = entry.range.start.coord.row() + 1;
9688 end_row = start_row;
9689 }
9690 if start_row > end_row || start_col > end_col {
9691 return Err(
9692 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
9693 );
9694 }
9695 Ok(PreparationRegion {
9696 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
9697 sheet_id: entry.sheet_id(),
9698 start_row,
9699 start_col,
9700 end_row,
9701 end_col,
9702 })
9703 }
9704
9705 #[cfg(test)]
9706 pub(crate) fn reset_target_root_dedup_probes_for_test() {
9707 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
9708 }
9709
9710 #[cfg(test)]
9711 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
9712 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
9713 }
9714
9715 pub(crate) fn resolve_target_producers(
9716 &mut self,
9717 targets: &[crate::engine::EvaluationTarget],
9718 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9719 use crate::engine::target_preparation::TargetProducer;
9720
9721 let request_id = self
9722 .active_evaluation_resource_request
9723 .as_ref()
9724 .map(|request| request.request_id);
9725 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
9726 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
9727 let active_span_refs = self
9728 .graph
9729 .formula_authority()
9730 .active_span_refs()
9731 .into_iter()
9732 .map(|span_ref| (span_ref.id, span_ref))
9733 .collect::<BTreeMap<_, _>>();
9734 let resolve_region = |engine: &mut Self,
9735 region: Region,
9736 value_only: Option<CellRef>,
9737 roots: &mut OrderedTargetProducers|
9738 -> Result<(), ExcelError> {
9739 let before = roots.len();
9740 let authority = engine.graph.formula_authority();
9741 for matched in authority.producer_results.query(region).matches {
9742 if let FormulaProducerId::Span(span_id) = matched.value.producer
9743 && let Some(span_ref) = active_span_refs.get(&span_id).copied()
9744 && let Some(demanded) = matched.value.result_region.intersection(region)
9745 {
9746 roots
9747 .push(TargetProducer::Span { span_ref, demanded })
9748 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9749 }
9750 }
9751 for anchor in engine.graph.spill_anchors_in_region(
9752 region.sheet_id(),
9753 region.axis_ranges().0.query_bounds().0,
9754 region.axis_ranges().1.query_bounds().0,
9755 region.axis_ranges().0.query_bounds().1,
9756 region.axis_ranges().1.query_bounds().1,
9757 ) {
9758 roots
9759 .push(TargetProducer::Legacy(anchor))
9760 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9761 }
9762 for vertex in engine.graph.vertices_in_region(
9763 region.sheet_id(),
9764 region.axis_ranges().0.query_bounds().0,
9765 region.axis_ranges().0.query_bounds().1,
9766 region.axis_ranges().1.query_bounds().0,
9767 region.axis_ranges().1.query_bounds().1,
9768 ) {
9769 let vertex = engine
9770 .graph
9771 .get_cell_ref(vertex)
9772 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
9773 .unwrap_or(vertex);
9774 match engine.graph.get_vertex_kind(vertex) {
9775 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
9776 roots.push(TargetProducer::Legacy(vertex)).map_err(|_| {
9777 target_root_allocation_error(roots.len() + 1, request_id)
9778 })?;
9779 }
9780 VertexKind::NamedScalar
9781 | VertexKind::NamedArray
9782 | VertexKind::Range
9783 | VertexKind::InfiniteRange
9784 | VertexKind::Table => {
9785 roots.push(TargetProducer::Symbol(vertex)).map_err(|_| {
9786 target_root_allocation_error(roots.len() + 1, request_id)
9787 })?;
9788 }
9789 VertexKind::Empty | VertexKind::Cell | VertexKind::External => {}
9790 }
9791 }
9792 if roots.len() == before
9793 && let Some(cell) = value_only
9794 {
9795 roots
9796 .push(TargetProducer::ValueOnly(cell))
9797 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9798 }
9799 Ok(())
9800 };
9801
9802 for target in targets {
9803 match target {
9804 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
9805 if *row == 0 || *col == 0 {
9806 return Err(ExcelError::new(ExcelErrorKind::Ref)
9807 .with_message("target cell coordinates are one-based"));
9808 }
9809 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
9810 ExcelError::new(ExcelErrorKind::Ref)
9811 .with_message(format!("target sheet not found: {sheet}"))
9812 })?;
9813 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
9814 resolve_region(
9815 self,
9816 Region::point(sheet_id, *row - 1, *col - 1),
9817 Some(cell),
9818 &mut roots,
9819 )?;
9820 }
9821 crate::engine::EvaluationTarget::Range(range) => {
9822 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
9823 ExcelError::new(ExcelErrorKind::Ref)
9824 .with_message(format!("target sheet not found: {}", range.sheet))
9825 })?;
9826 resolve_region(
9827 self,
9828 Region::rect(
9829 sheet_id,
9830 range.start_row - 1,
9831 range.end_row - 1,
9832 range.start_col - 1,
9833 range.end_col - 1,
9834 ),
9835 None,
9836 &mut roots,
9837 )?;
9838 }
9839 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
9840 let sheet_id = scope_sheet
9841 .as_deref()
9842 .and_then(|sheet| self.graph.sheet_id(sheet))
9843 .unwrap_or_else(|| self.graph.default_sheet_id());
9844 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9845 roots
9846 .push(TargetProducer::Symbol(entry.vertex))
9847 .map_err(|_| {
9848 target_root_allocation_error(roots.len() + 1, request_id)
9849 })?;
9850 }
9851 }
9852 crate::engine::EvaluationTarget::Table { name, .. } => {
9853 if let Some(entry) = self.graph.resolve_table_entry(name) {
9854 roots
9855 .push(TargetProducer::Symbol(entry.vertex))
9856 .map_err(|_| {
9857 target_root_allocation_error(roots.len() + 1, request_id)
9858 })?;
9859 }
9860 }
9861 }
9862 }
9863 Ok(roots.into_vec())
9864 }
9865
9866 fn resolve_target_producers_from_existing_roots(
9867 &mut self,
9868 roots: &[crate::engine::target_preparation::TargetProducer],
9869 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9870 use crate::engine::target_preparation::TargetProducer;
9871 let request_id = self
9872 .active_evaluation_resource_request
9873 .as_ref()
9874 .map(|request| request.request_id);
9875 let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
9876 .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
9877 for root in roots {
9878 match *root {
9879 TargetProducer::Span { demanded, .. } => {
9880 let (rows, cols) = demanded.axis_ranges();
9881 for vertex in self.graph.vertices_in_region(
9882 demanded.sheet_id(),
9883 rows.query_bounds().0,
9884 rows.query_bounds().1,
9885 cols.query_bounds().0,
9886 cols.query_bounds().1,
9887 ) {
9888 if matches!(
9889 self.graph.get_vertex_kind(vertex),
9890 VertexKind::FormulaScalar | VertexKind::FormulaArray
9891 ) {
9892 refreshed
9893 .push(TargetProducer::Legacy(vertex))
9894 .map_err(|_| {
9895 target_root_allocation_error(refreshed.len() + 1, request_id)
9896 })?;
9897 }
9898 }
9899 }
9900 other => {
9901 refreshed.push(other).map_err(|_| {
9902 target_root_allocation_error(refreshed.len() + 1, request_id)
9903 })?;
9904 }
9905 }
9906 }
9907 Ok(refreshed.into_vec())
9908 }
9909
9910 pub fn prepare_graph_for_targets(
9914 &mut self,
9915 targets: &[crate::engine::EvaluationTarget],
9916 options: crate::engine::TargetEvalOptions<'_>,
9917 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9918 let previous_budgets = self.evaluation_resource_budgets.clone();
9919 let diagnostics_len = self.formula_parse_diagnostics.len();
9920 let previous_report = self.last_formula_ingest_report.clone();
9921 if let Some(budgets) = options.budgets {
9922 self.evaluation_resource_budgets = budgets.clone();
9923 }
9924 let previous_graph_budget_override = self
9925 .graph
9926 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
9927 let previous_cancel = self.active_cancel_flag.take();
9931 self.active_cancel_flag = options.cancel.clone();
9932 let result = self.observe_evaluation_resource_request(
9933 EvaluationRequestKind::TargetPreparation,
9934 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
9935 );
9936 self.active_cancel_flag = previous_cancel;
9937 self.graph
9938 .set_admission_budget_override(previous_graph_budget_override);
9939 self.evaluation_resource_budgets = previous_budgets;
9940 if result.is_err() {
9941 self.formula_parse_diagnostics.truncate(diagnostics_len);
9942 self.last_formula_ingest_report = previous_report;
9943 }
9944 result
9945 }
9946
9947 fn prepare_graph_for_targets_unobserved(
9948 &mut self,
9949 targets: &[crate::engine::EvaluationTarget],
9950 options: &crate::engine::TargetEvalOptions<'_>,
9951 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9952 let scratch_checkpoint = self
9953 .active_resource_ledger
9954 .as_ref()
9955 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
9956 let result = self.prepare_graph_for_targets_transaction(targets, options);
9957 let release = self
9958 .active_resource_ledger
9959 .as_mut()
9960 .map_or(Ok(()), |ledger| {
9961 ledger.release_scratch_to(scratch_checkpoint)
9962 });
9963 match (result, release) {
9964 (result, Ok(())) => result,
9965 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
9966 }
9967 }
9968
9969 fn prepare_graph_for_targets_transaction(
9970 &mut self,
9971 targets: &[crate::engine::EvaluationTarget],
9972 options: &crate::engine::TargetEvalOptions<'_>,
9973 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9974 use crate::engine::{
9975 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
9976 TableSelection,
9977 };
9978
9979 self.target_preparation_checkpoint(options.deadline, 0)?;
9980 self.observe_function_semantic_epoch()?;
9981 let assumptions = self.preparation_revisions();
9982 let ledger_at_start = self
9983 .active_resource_ledger
9984 .as_ref()
9985 .map(|ledger| ledger.snapshot());
9986 let diagnostics_len = self.formula_parse_diagnostics.len();
9987 let report_len = self.last_formula_ingest_report.clone();
9988 let request_id = options.request_id.or_else(|| {
9989 self.active_evaluation_resource_request
9990 .as_ref()
9991 .map(|stats| stats.request_id)
9992 });
9993
9994 let mut scope = PrepareScope::Exact;
9995 let mut reasons = Vec::new();
9996 let mut regions = VecDeque::new();
9997 let mut normalized = Vec::with_capacity(targets.len());
9998 let mut symbol_vertices = VecDeque::new();
9999 for target in targets {
10000 self.target_preparation_checkpoint(options.deadline, 1)?;
10001 match target {
10002 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10003 if *row == 0 || *col == 0 {
10004 return Err(ExcelError::new(ExcelErrorKind::Ref)
10005 .with_message("target cell coordinates are one-based"));
10006 }
10007 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10008 ExcelError::new(ExcelErrorKind::Ref)
10009 .with_message(format!("target sheet not found: {sheet}"))
10010 })?;
10011 regions.push_back(PreparationRegion {
10012 sheet: sheet.clone(),
10013 sheet_id,
10014 start_row: *row,
10015 start_col: *col,
10016 end_row: *row,
10017 end_col: *col,
10018 });
10019 normalized.push(target.clone());
10020 }
10021 crate::engine::EvaluationTarget::Range(range) => {
10022 if range.start_row == 0
10023 || range.start_col == 0
10024 || range.end_row < range.start_row
10025 || range.end_col < range.start_col
10026 {
10027 return Err(ExcelError::new(ExcelErrorKind::Ref)
10028 .with_message("invalid target range"));
10029 }
10030 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10031 ExcelError::new(ExcelErrorKind::Ref)
10032 .with_message(format!("target sheet not found: {}", range.sheet))
10033 })?;
10034 regions.push_back(PreparationRegion {
10035 sheet: range.sheet.clone(),
10036 sheet_id,
10037 start_row: range.start_row,
10038 start_col: range.start_col,
10039 end_row: range.end_row,
10040 end_col: range.end_col,
10041 });
10042 normalized.push(target.clone());
10043 }
10044 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10045 let current_sheet = match scope_sheet {
10046 Some(sheet) => self.graph.sheet_id(sheet).ok_or_else(|| {
10047 ExcelError::new(ExcelErrorKind::Ref)
10048 .with_message(format!("name scope sheet not found: {sheet}"))
10049 })?,
10050 None => self.graph.default_sheet_id(),
10051 };
10052 if let Some(entry) = self.graph.resolve_name_entry(name, current_sheet) {
10053 symbol_vertices.push_back(entry.vertex);
10054 } else {
10055 Self::widen_target_preparation(
10056 options.opaque_policy,
10057 &mut scope,
10058 &mut reasons,
10059 OpaqueReason::UnresolvedName,
10060 )?;
10061 }
10062 normalized.push(target.clone());
10063 }
10064 crate::engine::EvaluationTarget::Table { name, selection } => {
10065 if let Some(entry) = self.graph.resolve_table_entry(name) {
10066 let region = self.table_selection_region(entry, selection)?;
10067 symbol_vertices.push_back(entry.vertex);
10068 regions.push_back(region);
10069 } else {
10070 Self::widen_target_preparation(
10071 options.opaque_policy,
10072 &mut scope,
10073 &mut reasons,
10074 OpaqueReason::UnresolvedTable,
10075 )?;
10076 }
10077 normalized.push(target.clone());
10078 }
10079 }
10080 }
10081
10082 let mut visited_regions = FxHashSet::default();
10083 let mut visited_vertices = FxHashSet::default();
10084 let mut selected = FxHashSet::default();
10085 let mut prepared = Vec::new();
10086 let mut pending_diagnostics = Vec::new();
10087 let mut planning_requests = BTreeSet::new();
10088 let mut selected_cells = Vec::new();
10089 let mut workbook_seeded = false;
10090 let mut sheet_scope_seeded = BTreeSet::new();
10091 let mut indexed_query_sheets = FxHashSet::default();
10092 let mut discovery_scratch_reserved = 0u64;
10093 let mut package_encountered = false;
10094 let mut selected_package_sheets = FxHashSet::default();
10095 let mut prepared_packages = Vec::new();
10096 let authoritative_with_ordinary = self.config.formula_plane_mode
10097 == FormulaPlaneMode::AuthoritativeExperimental
10098 && self.staged_formula_index.ordinary_count() != 0;
10099 let has_unknown_package_sheet = self
10100 .staged_formula_index
10101 .package_sheets()
10102 .any(|sheet| self.graph.sheet_id(sheet).is_none());
10103
10104 loop {
10105 if let PrepareScope::Sheets(sheets) = &scope {
10106 for sheet in sheets.clone() {
10107 if !sheet_scope_seeded.insert(sheet.clone()) {
10108 continue;
10109 }
10110 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10111 package_encountered = true;
10112 Self::widen_target_preparation(
10113 options.opaque_policy,
10114 &mut scope,
10115 &mut reasons,
10116 OpaqueReason::UnsupportedSourceSemantics,
10117 )?;
10118 continue;
10119 };
10120 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10121 self.target_preparation_checkpoint(options.deadline, 1)?;
10122 regions.push_back(PreparationRegion {
10123 sheet: sheet.clone(),
10124 sheet_id,
10125 start_row: lease.row,
10126 start_col: lease.col,
10127 end_row: lease.row,
10128 end_col: lease.col,
10129 });
10130 }
10131 if self
10132 .staged_formula_index
10133 .package_lease_for_sheet(&sheet)
10134 .is_some()
10135 {
10136 regions.push_back(PreparationRegion {
10137 sheet: sheet.clone(),
10138 sheet_id,
10139 start_row: 1,
10140 start_col: 1,
10141 end_row: self.workbook_load_limits.max_sheet_rows,
10142 end_col: self.workbook_load_limits.max_sheet_cols,
10143 });
10144 }
10145 }
10146 }
10147 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10148 workbook_seeded = true;
10149 for (sheet, lease) in self.staged_formula_index.all_leases() {
10150 self.target_preparation_checkpoint(options.deadline, 1)?;
10151 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10152 package_encountered = true;
10153 Self::widen_target_preparation(
10154 options.opaque_policy,
10155 &mut scope,
10156 &mut reasons,
10157 OpaqueReason::UnsupportedSourceSemantics,
10158 )?;
10159 continue;
10160 };
10161 regions.push_back(PreparationRegion {
10162 sheet,
10163 sheet_id,
10164 start_row: lease.row,
10165 start_col: lease.col,
10166 end_row: lease.row,
10167 end_col: lease.col,
10168 });
10169 }
10170 let package_sheets = self
10171 .staged_formula_index
10172 .package_sheets()
10173 .map(str::to_string)
10174 .collect::<Vec<_>>();
10175 for sheet in package_sheets {
10176 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10177 package_encountered = true;
10178 Self::widen_target_preparation(
10179 options.opaque_policy,
10180 &mut scope,
10181 &mut reasons,
10182 OpaqueReason::UnsupportedSourceSemantics,
10183 )?;
10184 continue;
10185 };
10186 regions.push_back(PreparationRegion {
10187 sheet,
10188 sheet_id,
10189 start_row: 1,
10190 start_col: 1,
10191 end_row: self.workbook_load_limits.max_sheet_rows,
10192 end_col: self.workbook_load_limits.max_sheet_cols,
10193 });
10194 }
10195 }
10196
10197 let Some(region) = regions.pop_front() else {
10198 if let Some(vertex) = symbol_vertices.pop_front() {
10199 self.target_preparation_checkpoint(options.deadline, 1)?;
10200 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10201 continue;
10202 }
10203 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10204 if let Some(ast) = self.graph.get_formula(vertex) {
10205 let snapshot =
10206 self.target_planning_snapshot(&ast, &mut planning_requests)?;
10207 if let Some(reason) =
10208 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10209 {
10210 return Err(Self::preparation_stale(
10211 reason,
10212 "target planning snapshot became stale during discovery",
10213 ));
10214 }
10215 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10216 if let Some(reason) =
10217 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10218 {
10219 if reason == OpaqueReason::DynamicReference
10220 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10221 {
10222 let sheet = self.graph.get_vertex_sheet_id(vertex);
10223 let sheet = self.graph.sheet_name(sheet).to_string();
10224 Self::widen_target_preparation_to_sheet(
10225 options.opaque_policy,
10226 &mut scope,
10227 &mut reasons,
10228 reason,
10229 &sheet,
10230 )?;
10231 } else {
10232 Self::widen_target_preparation(
10233 options.opaque_policy,
10234 &mut scope,
10235 &mut reasons,
10236 reason,
10237 )?;
10238 }
10239 }
10240 } else if vertex_is_dynamic {
10241 Self::widen_target_preparation(
10242 options.opaque_policy,
10243 &mut scope,
10244 &mut reasons,
10245 OpaqueReason::DynamicReference,
10246 )?;
10247 }
10248 if let Some(anchor) = self
10249 .graph
10250 .get_cell_ref(vertex)
10251 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
10252 {
10253 symbol_vertices.push_back(anchor);
10254 }
10255 if let Some(cell) = self.graph.get_cell_ref(vertex) {
10256 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
10257 regions.push_back(PreparationRegion {
10258 sheet,
10259 sheet_id: cell.sheet_id,
10260 start_row: cell.coord.row() + 1,
10261 start_col: cell.coord.col() + 1,
10262 end_row: cell.coord.row() + 1,
10263 end_col: cell.coord.col() + 1,
10264 });
10265 }
10266 for dependency in self.graph.get_dependencies(vertex) {
10267 self.target_preparation_checkpoint(options.deadline, 1)?;
10268 symbol_vertices.push_back(dependency);
10269 }
10270 if let Some(range_dependencies) = self
10271 .graph
10272 .formula_range_dependencies(vertex)
10273 .map(<[_]>::to_vec)
10274 {
10275 for range in range_dependencies {
10276 self.target_preparation_checkpoint(options.deadline, 1)?;
10277 let sheet_id = match range.sheet {
10278 crate::reference::SharedSheetLocator::Id(id) => id,
10279 crate::reference::SharedSheetLocator::Current => {
10280 self.graph.get_vertex_sheet_id(vertex)
10281 }
10282 crate::reference::SharedSheetLocator::Name(ref name) => {
10283 let Some(id) = self.graph.sheet_id(name) else {
10284 if Self::widen_target_preparation(
10285 options.opaque_policy,
10286 &mut scope,
10287 &mut reasons,
10288 OpaqueReason::UnresolvedCrossSheetBinding,
10289 )? {
10290 break;
10291 }
10292 continue;
10293 };
10294 id
10295 }
10296 };
10297 let sheet = self.graph.sheet_name(sheet_id).to_string();
10298 regions.push_back(PreparationRegion {
10299 sheet,
10300 sheet_id,
10301 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10302 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10303 end_row: range
10304 .end_row
10305 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10306 bound.index + 1
10307 }),
10308 end_col: range
10309 .end_col
10310 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10311 bound.index + 1
10312 }),
10313 });
10314 }
10315 }
10316 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
10317 match &name.definition {
10318 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
10319 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
10320 sheet_id: cell.sheet_id,
10321 start_row: cell.coord.row() + 1,
10322 start_col: cell.coord.col() + 1,
10323 end_row: cell.coord.row() + 1,
10324 end_col: cell.coord.col() + 1,
10325 }),
10326 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
10327 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
10328 sheet_id: range.start.sheet_id,
10329 start_row: range.start.coord.row() + 1,
10330 start_col: range.start.coord.col() + 1,
10331 end_row: range.end.coord.row() + 1,
10332 end_col: range.end.coord.col() + 1,
10333 }),
10334 NamedDefinition::Formula {
10335 ast,
10336 dependencies,
10337 range_deps,
10338 } => {
10339 let snapshot =
10340 self.target_planning_snapshot(ast, &mut planning_requests)?;
10341 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
10342 &snapshot,
10343 &assumptions,
10344 ) {
10345 return Err(Self::preparation_stale(
10346 reason,
10347 "target planning snapshot became stale during discovery",
10348 ));
10349 }
10350 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
10351 Self::widen_target_preparation(
10352 options.opaque_policy,
10353 &mut scope,
10354 &mut reasons,
10355 reason,
10356 )?;
10357 }
10358 for dependency in dependencies {
10359 self.target_preparation_checkpoint(options.deadline, 1)?;
10360 symbol_vertices.push_back(*dependency);
10361 }
10362 for range in range_deps {
10363 self.target_preparation_checkpoint(options.deadline, 1)?;
10364 let sheet_id = match range.sheet {
10365 crate::reference::SharedSheetLocator::Id(id) => id,
10366 _ => self.graph.default_sheet_id(),
10367 };
10368 regions.push_back(PreparationRegion {
10369 sheet: self.graph.sheet_name(sheet_id).to_string(),
10370 sheet_id,
10371 start_row: range
10372 .start_row
10373 .map_or(1, |bound| bound.index + 1),
10374 start_col: range
10375 .start_col
10376 .map_or(1, |bound| bound.index + 1),
10377 end_row: range.end_row.map_or(
10378 self.workbook_load_limits.max_sheet_rows,
10379 |bound| bound.index + 1,
10380 ),
10381 end_col: range.end_col.map_or(
10382 self.workbook_load_limits.max_sheet_cols,
10383 |bound| bound.index + 1,
10384 ),
10385 });
10386 }
10387 }
10388 NamedDefinition::Literal(_) => {}
10389 }
10390 }
10391 if let Some(table) = self.graph.table_by_vertex(vertex) {
10392 regions
10393 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
10394 }
10395 continue;
10396 }
10397 break;
10398 };
10399
10400 self.target_preparation_checkpoint(options.deadline, 1)?;
10401 if !visited_regions.insert(region.clone()) {
10402 continue;
10403 }
10404 if let PrepareScope::Sheets(sheets) = &mut scope
10405 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
10406 {
10407 sheets.push(region.sheet.clone());
10408 sheets.sort();
10409 }
10410 let package_match = self.staged_formula_index.package_for_region(
10411 ®ion.sheet,
10412 region.start_row,
10413 region.start_col,
10414 region.end_row,
10415 region.end_col,
10416 );
10417 let package_lease = match package_match {
10418 Some(Ok(lease)) => Some(lease),
10419 Some(Err(()))
10420 if region.start_row == 1
10421 && region.start_col == 1
10422 && region.end_row == self.workbook_load_limits.max_sheet_rows
10423 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
10424 {
10425 self.staged_formula_index
10426 .package_lease_for_sheet(®ion.sheet)
10427 }
10428 Some(Err(())) => {
10429 Self::widen_target_preparation(
10430 options.opaque_policy,
10431 &mut scope,
10432 &mut reasons,
10433 OpaqueReason::DeferredSourcePackage,
10434 )?;
10435 None
10436 }
10437 None => None,
10438 };
10439 let compatibility_before_package_replay = package_lease.is_some()
10440 && (authoritative_with_ordinary || has_unknown_package_sheet);
10441 let package_lease = if compatibility_before_package_replay {
10442 package_encountered = true;
10443 Self::widen_target_preparation(
10444 options.opaque_policy,
10445 &mut scope,
10446 &mut reasons,
10447 OpaqueReason::UnsupportedSourceSemantics,
10448 )?;
10449 None
10450 } else {
10451 package_lease
10452 };
10453 if let Some(package_lease) = package_lease
10454 && selected_package_sheets.insert(region.sheet.clone())
10455 {
10456 self.target_preparation_checkpoint(options.deadline, 1)?;
10457 let mut package = self.prepare_target_source_package(
10458 ®ion.sheet,
10459 package_lease,
10460 options.deadline,
10461 )?;
10462 for placement in &package.placements {
10463 self.target_preparation_checkpoint(options.deadline, 1)?;
10464 for dependency in &placement.fragment_dependency_proof().1.dependencies {
10465 self.target_preparation_checkpoint(options.deadline, 1)?;
10466 let (rows, cols) = dependency.read_region.axis_ranges();
10467 let (start_row, end_row) = rows.query_bounds();
10468 let (start_col, end_col) = cols.query_bounds();
10469 let dependency_sheet = dependency.read_region.sheet_id();
10470 regions.push_back(PreparationRegion {
10471 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10472 sheet_id: dependency_sheet,
10473 start_row: start_row.saturating_add(1),
10474 start_col: start_col.saturating_add(1),
10475 end_row: end_row
10476 .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
10477 .saturating_add(1),
10478 end_col: end_col
10479 .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
10480 .saturating_add(1),
10481 });
10482 }
10483 }
10484
10485 let mut final_fallback = BTreeMap::new();
10486 for record in package.fallback_records() {
10487 final_fallback.insert((record.row, record.col), record.clone());
10488 }
10489 let batch = self
10490 .formula_batch_from_exact_replay(®ion.sheet, final_fallback.into_values())?;
10491 for record in batch.formulas {
10492 self.target_preparation_checkpoint(options.deadline, 1)?;
10493 let ast = self
10494 .graph
10495 .data_store()
10496 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
10497 .ok_or_else(|| {
10498 ExcelError::new(ExcelErrorKind::Value)
10499 .with_message("target fallback AST is unavailable")
10500 })?;
10501 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10502 if let Some(reason) =
10503 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10504 {
10505 return Err(Self::preparation_stale(
10506 reason,
10507 "target fallback planning snapshot became stale during discovery",
10508 ));
10509 }
10510 let proven_sheet_local_dynamic =
10511 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10512 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10513 if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
10514 Self::widen_target_preparation_to_sheet(
10515 options.opaque_policy,
10516 &mut scope,
10517 &mut reasons,
10518 reason,
10519 ®ion.sheet,
10520 )?;
10521 } else {
10522 Self::widen_target_preparation(
10523 options.opaque_policy,
10524 &mut scope,
10525 &mut reasons,
10526 reason,
10527 )?;
10528 }
10529 }
10530 let placement = CellRef::new(
10531 package.sheet_id,
10532 Coord::from_excel(record.row, record.col, true, true),
10533 );
10534 let ingested = self
10535 .graph
10536 .ingest_pipeline(&snapshot)
10537 .enable_function_semantics()
10538 .ingest_formula(
10539 FormulaAstInput::RawArena(record.ast_id),
10540 placement,
10541 record.formula_text,
10542 )?;
10543 if ingested.dep_plan.dynamic {
10544 if proven_sheet_local_dynamic {
10545 Self::widen_target_preparation_to_sheet(
10546 options.opaque_policy,
10547 &mut scope,
10548 &mut reasons,
10549 OpaqueReason::DynamicReference,
10550 ®ion.sheet,
10551 )?;
10552 } else {
10553 Self::widen_target_preparation(
10554 options.opaque_policy,
10555 &mut scope,
10556 &mut reasons,
10557 OpaqueReason::DynamicReference,
10558 )?;
10559 }
10560 }
10561 for dep in &ingested.dep_plan.direct_cell_deps {
10562 self.target_preparation_checkpoint(options.deadline, 1)?;
10563 regions.push_back(PreparationRegion {
10564 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10565 sheet_id: dep.sheet_id,
10566 start_row: dep.coord.row().saturating_add(1),
10567 start_col: dep.coord.col().saturating_add(1),
10568 end_row: dep.coord.row().saturating_add(1),
10569 end_col: dep.coord.col().saturating_add(1),
10570 });
10571 }
10572 for range in &ingested.dep_plan.range_deps {
10573 self.target_preparation_checkpoint(options.deadline, 1)?;
10574 let dependency_sheet = match &range.sheet {
10575 crate::reference::SharedSheetLocator::Id(id) => *id,
10576 crate::reference::SharedSheetLocator::Current => package.sheet_id,
10577 crate::reference::SharedSheetLocator::Name(name) => {
10578 let Some(id) = self.graph.sheet_id(name) else {
10579 Self::widen_target_preparation(
10580 options.opaque_policy,
10581 &mut scope,
10582 &mut reasons,
10583 OpaqueReason::UnresolvedCrossSheetBinding,
10584 )?;
10585 continue;
10586 };
10587 id
10588 }
10589 };
10590 regions.push_back(PreparationRegion {
10591 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10592 sheet_id: dependency_sheet,
10593 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10594 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10595 end_row: range
10596 .end_row
10597 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10598 bound.index + 1
10599 }),
10600 end_col: range
10601 .end_col
10602 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10603 bound.index + 1
10604 }),
10605 });
10606 }
10607 for name in ingested
10608 .dep_plan
10609 .resolved_named_refs
10610 .iter()
10611 .chain(&ingested.dep_plan.named_refs)
10612 {
10613 self.target_preparation_checkpoint(options.deadline, 1)?;
10614 if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
10615 symbol_vertices.push_back(entry.vertex);
10616 } else if self.graph.resolve_source_scalar_entry(name).is_none()
10617 && self.graph.resolve_source_table_entry(name).is_none()
10618 {
10619 Self::widen_target_preparation(
10620 options.opaque_policy,
10621 &mut scope,
10622 &mut reasons,
10623 OpaqueReason::UnresolvedName,
10624 )?;
10625 }
10626 }
10627 for table in &ingested.dep_plan.table_refs {
10628 self.target_preparation_checkpoint(options.deadline, 1)?;
10629 if let Some(entry) = self.graph.resolve_table_entry(table) {
10630 symbol_vertices.push_back(entry.vertex);
10631 } else if self.graph.resolve_source_table_entry(table).is_none() {
10632 Self::widen_target_preparation(
10633 options.opaque_policy,
10634 &mut scope,
10635 &mut reasons,
10636 OpaqueReason::UnresolvedTable,
10637 )?;
10638 }
10639 }
10640 package.legacy.push((
10641 record.row,
10642 record.col,
10643 ingested.ast_id,
10644 ingested.dep_plan,
10645 ));
10646 }
10647 prepared_packages.push(package);
10648 }
10649 let leases = self.staged_formula_index.leases_in_region(
10650 ®ion.sheet,
10651 region.start_row,
10652 region.start_col,
10653 region.end_row,
10654 region.end_col,
10655 );
10656 for lease in leases {
10657 self.target_preparation_checkpoint(options.deadline, 1)?;
10658 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
10659 ExcelError::new(ExcelErrorKind::Ref)
10660 .with_message(format!("staged formula sheet not found: {}", region.sheet))
10661 })?;
10662 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
10663 if !selected.insert(key) {
10664 continue;
10665 }
10666 let text = self
10667 .staged_formulas
10668 .get(®ion.sheet)
10669 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
10670 .ok_or_else(|| {
10671 ExcelError::new(ExcelErrorKind::Value)
10672 .with_message("staged formula index is stale")
10673 })?
10674 .to_string();
10675 let formula = if text.starts_with('=') {
10676 text.clone()
10677 } else {
10678 format!("={text}")
10679 };
10680 self.target_preparation_checkpoint(options.deadline, 1)?;
10681 let ast = match formualizer_parse::parser::parse(&formula) {
10682 Ok(ast) => ast,
10683 Err(error) => {
10684 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
10685 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
10686 format!(
10687 "Formula parse error at {}!{}{}: {error}",
10688 region.sheet,
10689 col_letters_from_1based(lease.col)
10690 .unwrap_or_else(|_| "?".to_string()),
10691 lease.row
10692 ),
10693 ));
10694 }
10695 pending_diagnostics.push(FormulaParseDiagnostic {
10696 sheet: region.sheet.clone(),
10697 row: lease.row,
10698 col: lease.col,
10699 formula: formula.clone(),
10700 message: error.to_string(),
10701 policy: self.config.formula_parse_policy,
10702 });
10703 match self.config.formula_parse_policy {
10704 FormulaParsePolicy::KeepCachedValue => {
10705 selected_cells.push(
10706 formualizer_common::RangeAddress::new(
10707 region.sheet.clone(),
10708 lease.row,
10709 lease.col,
10710 lease.row,
10711 lease.col,
10712 )
10713 .expect("selected staged coordinates are valid"),
10714 );
10715 prepared.push(PreparedOrdinaryStagedFormula {
10716 sheet: region.sheet.clone(),
10717 sheet_id,
10718 lease,
10719 ast_id: None,
10720 plan: None,
10721 });
10722 continue;
10723 }
10724 FormulaParsePolicy::AsText => ASTNode::new(
10725 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
10726 None,
10727 ),
10728 FormulaParsePolicy::CoerceToError => ASTNode::new(
10729 ASTNodeType::Literal(LiteralValue::Error(
10730 ExcelError::new(ExcelErrorKind::Error)
10731 .with_message(format!("Malformed formula: {error}")),
10732 )),
10733 None,
10734 ),
10735 FormulaParsePolicy::Strict => unreachable!(),
10736 }
10737 }
10738 };
10739 self.target_preparation_checkpoint(options.deadline, 1)?;
10740 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10741 self.target_preparation_checkpoint(options.deadline, 1)?;
10742 if let Some(reason) =
10743 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10744 {
10745 return Err(Self::preparation_stale(
10746 reason,
10747 "target planning snapshot became stale during discovery",
10748 ));
10749 }
10750 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10751 if reason == OpaqueReason::DynamicReference
10752 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10753 {
10754 Self::widen_target_preparation_to_sheet(
10755 options.opaque_policy,
10756 &mut scope,
10757 &mut reasons,
10758 reason,
10759 ®ion.sheet,
10760 )?;
10761 } else {
10762 Self::widen_target_preparation(
10763 options.opaque_policy,
10764 &mut scope,
10765 &mut reasons,
10766 reason,
10767 )?;
10768 }
10769 }
10770 let proven_sheet_local_dynamic =
10771 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10772 let placement = CellRef::new(
10773 sheet_id,
10774 Coord::from_excel(lease.row, lease.col, true, true),
10775 );
10776 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
10777 FormulaAstInput::Tree(ast),
10778 placement,
10779 Some(Arc::from(formula)),
10780 )?;
10781 self.target_preparation_checkpoint(options.deadline, 1)?;
10782 if ingested.dep_plan.dynamic {
10783 if proven_sheet_local_dynamic {
10784 Self::widen_target_preparation_to_sheet(
10785 options.opaque_policy,
10786 &mut scope,
10787 &mut reasons,
10788 OpaqueReason::DynamicReference,
10789 ®ion.sheet,
10790 )?;
10791 } else {
10792 Self::widen_target_preparation(
10793 options.opaque_policy,
10794 &mut scope,
10795 &mut reasons,
10796 OpaqueReason::DynamicReference,
10797 )?;
10798 }
10799 }
10800 for dep in &ingested.dep_plan.direct_cell_deps {
10801 self.target_preparation_checkpoint(options.deadline, 1)?;
10802 regions.push_back(PreparationRegion {
10803 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10804 sheet_id: dep.sheet_id,
10805 start_row: dep.coord.row() + 1,
10806 start_col: dep.coord.col() + 1,
10807 end_row: dep.coord.row() + 1,
10808 end_col: dep.coord.col() + 1,
10809 });
10810 }
10811 for range in &ingested.dep_plan.range_deps {
10812 self.target_preparation_checkpoint(options.deadline, 1)?;
10813 let sheet_id = match range.sheet {
10814 crate::reference::SharedSheetLocator::Id(id) => id,
10815 crate::reference::SharedSheetLocator::Current => sheet_id,
10816 crate::reference::SharedSheetLocator::Name(ref name) => {
10817 let Some(id) = self.graph.sheet_id(name) else {
10818 Self::widen_target_preparation(
10819 options.opaque_policy,
10820 &mut scope,
10821 &mut reasons,
10822 OpaqueReason::UnresolvedCrossSheetBinding,
10823 )?;
10824 continue;
10825 };
10826 id
10827 }
10828 };
10829 regions.push_back(PreparationRegion {
10830 sheet: self.graph.sheet_name(sheet_id).to_string(),
10831 sheet_id,
10832 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10833 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10834 end_row: range
10835 .end_row
10836 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10837 bound.index + 1
10838 }),
10839 end_col: range
10840 .end_col
10841 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10842 bound.index + 1
10843 }),
10844 });
10845 }
10846 for name in ingested
10847 .dep_plan
10848 .resolved_named_refs
10849 .iter()
10850 .chain(&ingested.dep_plan.named_refs)
10851 {
10852 self.target_preparation_checkpoint(options.deadline, 1)?;
10853 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
10854 symbol_vertices.push_back(entry.vertex);
10855 } else if self.graph.resolve_source_scalar_entry(name).is_none()
10856 && self.graph.resolve_source_table_entry(name).is_none()
10857 {
10858 Self::widen_target_preparation(
10859 options.opaque_policy,
10860 &mut scope,
10861 &mut reasons,
10862 OpaqueReason::UnresolvedName,
10863 )?;
10864 }
10865 }
10866 for table in &ingested.dep_plan.table_refs {
10867 self.target_preparation_checkpoint(options.deadline, 1)?;
10868 if let Some(entry) = self.graph.resolve_table_entry(table) {
10869 symbol_vertices.push_back(entry.vertex);
10870 } else if self.graph.resolve_source_table_entry(table).is_none() {
10871 Self::widen_target_preparation(
10872 options.opaque_policy,
10873 &mut scope,
10874 &mut reasons,
10875 OpaqueReason::UnresolvedTable,
10876 )?;
10877 }
10878 }
10879 selected_cells.push(
10880 formualizer_common::RangeAddress::new(
10881 region.sheet.clone(),
10882 lease.row,
10883 lease.col,
10884 lease.row,
10885 lease.col,
10886 )
10887 .expect("selected staged coordinates are valid"),
10888 );
10889 prepared.push(PreparedOrdinaryStagedFormula {
10890 sheet: region.sheet.clone(),
10891 sheet_id,
10892 lease,
10893 ast_id: Some(ingested.ast_id),
10894 plan: Some(ingested.dep_plan),
10895 });
10896 }
10897
10898 if indexed_query_sheets.insert(region.sheet_id) {
10899 self.graph.prepare_sheet_index_for_query(region.sheet_id);
10900 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
10901 .saturating_mul(32);
10902 self.reserve_graph_source_scratch(bytes)?;
10903 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
10904 }
10905 let spill_anchors = self.graph.spill_anchors_in_region(
10906 region.sheet_id,
10907 region.start_row - 1,
10908 region.start_col - 1,
10909 region.end_row - 1,
10910 region.end_col - 1,
10911 );
10912 for anchor in spill_anchors {
10913 self.target_preparation_checkpoint(options.deadline, 1)?;
10914 symbol_vertices.push_back(anchor);
10915 }
10916 let vertices = self.graph.vertices_in_region(
10917 region.sheet_id,
10918 region.start_row - 1,
10919 region.end_row - 1,
10920 region.start_col - 1,
10921 region.end_col - 1,
10922 );
10923 for vertex in vertices {
10924 self.target_preparation_checkpoint(options.deadline, 1)?;
10925 symbol_vertices.push_back(vertex);
10926 }
10927 }
10928
10929 #[cfg(test)]
10930 self.target_preparation_fault(
10931 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
10932 )?;
10933
10934 if package_encountered
10935 || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
10936 && !prepared.is_empty())
10937 {
10938 Self::widen_target_preparation(
10939 options.opaque_policy,
10940 &mut scope,
10941 &mut reasons,
10942 OpaqueReason::UnsupportedSourceSemantics,
10943 )?;
10944 self.target_preparation_checkpoint(options.deadline, 0)?;
10945 let selected_count = self.staged_formula_count();
10946 let selected_packages = self
10947 .staged_formulas
10948 .values()
10949 .filter_map(|staged| staged.deferred_package.as_ref())
10950 .map(|package| package.families.len() + package.partitioned_families.len())
10951 .sum();
10952 self.formula_parse_diagnostics.truncate(diagnostics_len);
10953 self.last_formula_ingest_report = report_len;
10954 #[cfg(test)]
10955 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
10956 hook();
10957 }
10958 self.target_preparation_checkpoint(options.deadline, 0)?;
10959 #[cfg(test)]
10960 self.target_preparation_fault(
10961 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
10962 )?;
10963 let current_revisions = self.preparation_revisions();
10964 if let Some(reason) = Self::preparation_revision_stale_reason(
10965 &assumptions,
10966 ¤t_revisions,
10967 &planning_requests,
10968 true,
10969 ) {
10970 return Err(Self::preparation_stale(
10971 reason,
10972 "target compatibility preparation plan is stale",
10973 ));
10974 }
10975 #[cfg(test)]
10976 self.target_preparation_fault(
10977 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
10978 )?;
10979 let commit_work_before = self
10980 .active_resource_ledger
10981 .as_ref()
10982 .map_or(0, |ledger| ledger.snapshot().work_charged);
10983 let commit_started = crate::instant::FzInstant::now();
10984 self.build_graph_all_unobserved()?;
10985 let commit_window = commit_started.elapsed();
10986 let ledger_after = self
10987 .active_resource_ledger
10988 .as_ref()
10989 .map(|ledger| ledger.snapshot());
10990 let actual_commit_work = ledger_after
10991 .map_or(0, |snapshot| snapshot.work_charged)
10992 .saturating_sub(commit_work_before);
10993 let observed_scratch_bytes = ledger_after
10994 .map_or(0, |snapshot| snapshot.scratch_peak)
10995 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10996 let revisions = assumptions.clone();
10997 let report = PreparedTargetGraphReport {
10998 request_id: request_id.unwrap_or_default(),
10999 requested_targets: targets.len(),
11000 normalized_regions: visited_regions.len(),
11001 normalized_target_list: normalized,
11002 selected_staged_cells: selected_count,
11003 selected_source_families: selected_packages,
11004 retained_staged_cells: self.staged_formula_count(),
11005 selected_cells,
11006 retained_cells: Vec::new(),
11007 widened_scope: PrepareScope::Workbook,
11008 widening_reasons: reasons,
11009 revisions,
11010 commit_window,
11011 estimated_scratch_bytes: discovery_scratch_reserved
11012 .saturating_add((selected_count as u64).saturating_mul(256)),
11013 observed_scratch_bytes,
11014 estimated_commit_work: selected_count as u64,
11015 actual_commit_work,
11016 outcome: PreparationOutcome::CompatibilityPrepared,
11017 };
11018 self.observe_target_preparation_report(&report);
11019 return Ok(report);
11020 }
11021
11022 prepared.sort_by_key(|formula| formula.lease.insertion_order);
11023 for package in &prepared_packages {
11024 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11025 formualizer_common::RangeAddress::new(
11026 package.sheet.clone(),
11027 record.row,
11028 record.col,
11029 record.row,
11030 record.col,
11031 )
11032 .ok()
11033 }));
11034 }
11035 let (mut legacy_graph, mut planned_formula_count) =
11036 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11037 let placements = prepared_packages
11038 .iter_mut()
11039 .flat_map(|package| std::mem::take(&mut package.placements))
11040 .collect::<Vec<_>>();
11041 let formula_plane = if placements.is_empty() {
11042 None
11043 } else {
11044 match self.graph.formula_authority().prepare_formula_plane_append(
11045 placements,
11046 self.graph.data_store(),
11047 self.graph.sheet_reg(),
11048 ) {
11049 Ok(append) => Some(append),
11050 Err(error) => {
11051 let reason = format!("TargetFormulaPlaneAppend:{error}");
11052 for package in &mut prepared_packages {
11053 *package
11054 .source_report
11055 .fallback_reasons
11056 .entry(reason.clone())
11057 .or_default() += package.direct_families as u64;
11058 if self.config.formula_plane_mode
11059 == FormulaPlaneMode::AuthoritativeExperimental
11060 {
11061 self.materialize_target_package_direct_records(
11062 package,
11063 &assumptions,
11064 &mut planning_requests,
11065 options.deadline,
11066 )?;
11067 } else {
11068 package.direct_families = 0;
11069 package.direct_cells = 0;
11070 package.direct_fragments = 0;
11071 package.direct_complete_families = 0;
11072 package.direct_complete_cells = 0;
11073 package.direct_partition_families = 0;
11074 package.direct_partition_cells = 0;
11075 }
11076 }
11077 (legacy_graph, planned_formula_count) =
11078 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11079 None
11080 }
11081 }
11082 };
11083 let new_vertices = legacy_graph.new_vertex_count();
11084 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11085 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11086 })?;
11087 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11088 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11089 })?;
11090 let current = self.graph.baseline_stats();
11091 let final_vertices = current
11092 .graph_vertex_count
11093 .checked_add(new_vertices)
11094 .ok_or_else(|| {
11095 crate::engine::ResourceLedgerError::Exhausted(
11096 formualizer_common::ResourceExhaustionDetail {
11097 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11098 limit: u64::MAX,
11099 observed: u64::MAX,
11100 request_id,
11101 },
11102 )
11103 .into_excel_error()
11104 })?;
11105 let final_edges = current
11106 .graph_edge_count
11107 .checked_sub(removed_edges)
11108 .and_then(|count| count.checked_add(new_edges))
11109 .ok_or_else(|| {
11110 crate::engine::ResourceLedgerError::Exhausted(
11111 formualizer_common::ResourceExhaustionDetail {
11112 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11113 limit: u64::MAX,
11114 observed: u64::MAX,
11115 request_id,
11116 },
11117 )
11118 .into_excel_error()
11119 })?;
11120 #[cfg(test)]
11121 self.target_preparation_fault(
11122 crate::engine::target_preparation::TargetPreparationFault::Admission,
11123 )?;
11124 let resource = |reason, limit: u64, observed: u64| {
11125 crate::engine::ResourceLedgerError::Exhausted(
11126 formualizer_common::ResourceExhaustionDetail {
11127 reason,
11128 limit,
11129 observed,
11130 request_id,
11131 },
11132 )
11133 .into_excel_error()
11134 };
11135 let admission = crate::engine::resource_ledger::GraphAdmission {
11136 final_vertices,
11137 final_edges,
11138 materialization_cells: planned_formula_count as u64,
11139 added_vertices: new_vertices,
11140 added_edges: new_edges,
11141 };
11142 let materialized_bytes = admission
11143 .materialized_graph_bytes()
11144 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11145 if let Err(error) = self.preflight_graph_admission(admission) {
11146 if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11147 self.observe_target_admission_failure(detail.reason);
11148 }
11149 return Err(error);
11150 }
11151 let selected_package_records = prepared_packages
11152 .iter()
11153 .map(|package| package.replay_records.len() as u64)
11154 .sum::<u64>();
11155 let planned_working_bytes = (prepared.len() as u64)
11156 .saturating_add(selected_package_records)
11157 .saturating_mul(256)
11158 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11159 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11160 .saturating_add((selected.len() as u64).saturating_mul(96))
11161 .saturating_add(materialized_bytes);
11162 let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11163 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11164 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11165 self.observe_target_admission_failure(
11166 formualizer_common::ResourceExhaustionReason::ScratchMemory,
11167 );
11168 return Err(error);
11169 }
11170
11171 let estimated_commit_duration = std::time::Duration::from_nanos(
11172 (new_vertices as u64)
11173 .saturating_add(new_edges as u64)
11174 .saturating_add(prepared.len() as u64)
11175 .saturating_add(selected_package_records)
11176 .max(1)
11177 .saturating_mul(100),
11178 );
11179 if options.deadline.is_some_and(|deadline| {
11180 std::time::Instant::now()
11181 .checked_add(estimated_commit_duration)
11182 .is_none_or(|finish| finish > deadline)
11183 }) {
11184 self.observe_target_admission_failure(
11185 formualizer_common::ResourceExhaustionReason::Deadline,
11186 );
11187 return Err(resource(
11188 formualizer_common::ResourceExhaustionReason::Deadline,
11189 0,
11190 1,
11191 ));
11192 }
11193 #[cfg(test)]
11194 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11195 hook();
11196 }
11197 self.target_preparation_checkpoint(options.deadline, 0)?;
11198 #[cfg(test)]
11199 self.target_preparation_fault(
11200 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11201 )?;
11202 let current_revisions = self.preparation_revisions();
11203 let staged_leases_match = prepared.iter().all(|formula| {
11204 self.staged_formula_index
11205 .lease_matches(&formula.sheet, formula.lease)
11206 }) && prepared_packages.iter().all(|package| {
11207 self.staged_formula_index
11208 .package_lease_matches(&package.sheet, package.lease)
11209 });
11210 let stale_reason = Self::preparation_revision_stale_reason(
11211 &assumptions,
11212 ¤t_revisions,
11213 &planning_requests,
11214 staged_leases_match,
11215 );
11216 if let Some(reason) = stale_reason {
11217 return Err(Self::preparation_stale(
11218 reason,
11219 "target graph preparation plan is stale",
11220 ));
11221 }
11222 #[cfg(test)]
11223 self.target_preparation_fault(
11224 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11225 )?;
11226 self.graph
11227 .validate_prepared_legacy_graph_plan(&legacy_graph)
11228 .map_err(|error| {
11229 Self::preparation_stale(
11230 formualizer_common::PreparationStaleReason::Graph,
11231 format!("target graph preparation plan is stale: {error}"),
11232 )
11233 })?;
11234 if let Some(append) = formula_plane.as_ref() {
11235 self.graph
11236 .formula_authority()
11237 .validate_prepared_formula_plane_append(
11238 append,
11239 self.graph.data_store(),
11240 self.graph.sheet_reg(),
11241 )
11242 .map_err(|error| {
11243 Self::preparation_stale(
11244 formualizer_common::PreparationStaleReason::Authority,
11245 format!("target FormulaPlane preparation is stale: {error}"),
11246 )
11247 })?;
11248 }
11249 #[cfg(test)]
11250 self.target_preparation_fault(
11251 crate::engine::target_preparation::TargetPreparationFault::Reservation,
11252 )?;
11253 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
11254 self.formula_parse_diagnostics
11255 .try_reserve(pending_diagnostics.len())
11256 .map_err(|_| {
11257 resource(
11258 formualizer_common::ResourceExhaustionReason::Admission,
11259 pending_diagnostics.len() as u64,
11260 pending_diagnostics.len() as u64,
11261 )
11262 })?;
11263 self.target_preparation_checkpoint(options.deadline, 0)?;
11264 #[cfg(test)]
11265 self.target_preparation_fault(
11266 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
11267 )?;
11268
11269 let commit_started = crate::instant::FzInstant::now();
11270 let committed = self
11271 .graph
11272 .apply_prevalidated_legacy_graph_plan(legacy_graph);
11273 let plane_report =
11274 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11275 formula_plane.map(|append| {
11276 self.graph
11277 .formula_authority_mut()
11278 .apply_prevalidated_formula_plane_append(append)
11279 })
11280 } else {
11281 None
11282 };
11283 if let Some(report) = plane_report.as_ref() {
11284 self.graph.mark_formula_spans_dirty(
11285 report.spans.iter().copied(),
11286 WholeSpanDirtyReason::NewSpan,
11287 );
11288 }
11289 for formula in &prepared {
11290 let removed = self
11291 .staged_formulas
11292 .get_mut(&formula.sheet)
11293 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
11294 debug_assert!(removed.is_some());
11295 let index_removed = self.staged_formula_index.remove(
11296 &formula.sheet,
11297 formula.lease.row,
11298 formula.lease.col,
11299 );
11300 debug_assert!(index_removed);
11301 }
11302 for package in &prepared_packages {
11303 let removed = self
11304 .staged_formulas
11305 .get_mut(&package.sheet)
11306 .and_then(|staged| staged.deferred_package.take());
11307 debug_assert!(removed.is_some());
11308 self.staged_formula_index.set_package(&package.sheet, None);
11309 }
11310 let empty_sheets = self
11311 .staged_formulas
11312 .iter()
11313 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
11314 .collect::<Vec<_>>();
11315 for sheet in empty_sheets {
11316 self.staged_formulas.remove(&sheet);
11317 }
11318 if committed > 0 || plane_report.is_some() {
11319 self.mark_topology_edited();
11320 }
11321 self.formula_parse_diagnostics.extend(pending_diagnostics);
11322 if !prepared.is_empty() || !prepared_packages.is_empty() {
11323 let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
11324 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
11325 prepared_packages
11326 .iter()
11327 .map(|package| package.replay_records.len() as u64)
11328 .sum::<u64>(),
11329 );
11330 ingest_delta.graph_formula_cells_materialized = committed as u64;
11331 ingest_delta.graph_vertices_created = new_vertices as u64;
11332 ingest_delta.graph_edges_created = new_edges as u64;
11333 for package in &prepared_packages {
11334 let source = &package.source_report;
11335 ingest_delta.source_formula_events = ingest_delta
11336 .source_formula_events
11337 .saturating_add(source.source_formula_events);
11338 ingest_delta.source_formula_records_spooled = ingest_delta
11339 .source_formula_records_spooled
11340 .saturating_add(source.source_formula_records_spooled);
11341 ingest_delta.source_spool_encoded_bytes = ingest_delta
11342 .source_spool_encoded_bytes
11343 .saturating_add(source.source_spool_encoded_bytes);
11344 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
11345 .source_spool_peak_memory_bytes
11346 .max(source.source_spool_peak_memory_bytes);
11347 ingest_delta.source_spool_spilled_bytes = ingest_delta
11348 .source_spool_spilled_bytes
11349 .saturating_add(source.source_spool_spilled_bytes);
11350 ingest_delta.source_spool_spill_files = ingest_delta
11351 .source_spool_spill_files
11352 .saturating_add(source.source_spool_spill_files);
11353 ingest_delta.source_spool_replays = ingest_delta
11354 .source_spool_replays
11355 .saturating_add(source.source_spool_replays)
11356 .saturating_add(1);
11357 ingest_delta.source_families_seen = ingest_delta
11358 .source_families_seen
11359 .saturating_add(source.families_seen);
11360 ingest_delta.source_family_cells_seen = ingest_delta
11361 .source_family_cells_seen
11362 .saturating_add(source.family_cells_seen);
11363 ingest_delta.source_family_shadow_eligible = ingest_delta
11364 .source_family_shadow_eligible
11365 .saturating_add(source.source_clean_families);
11366 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
11367 .source_family_shadow_eligible_cells
11368 .saturating_add(source.source_clean_cells);
11369 ingest_delta.source_partitioned_families_seen = ingest_delta
11370 .source_partitioned_families_seen
11371 .saturating_add(source.source_fragmentable_families);
11372 ingest_delta.source_partition_holes = ingest_delta
11373 .source_partition_holes
11374 .saturating_add(source.source_hole_exclusions);
11375 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
11376 .source_partition_ordinary_exceptions
11377 .saturating_add(source.source_ordinary_exclusions);
11378 ingest_delta.source_partition_surviving_cells = ingest_delta
11379 .source_partition_surviving_cells
11380 .saturating_add(source.source_fragmentable_cells);
11381 for (reason, count) in &source.fallback_reasons {
11382 let total = ingest_delta
11383 .fallback_reasons
11384 .entry(reason.clone())
11385 .or_default();
11386 *total = total.saturating_add(*count);
11387 }
11388 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
11389 ingest_delta.source_family_fallback = ingest_delta
11390 .source_family_fallback
11391 .saturating_add(source.families_seen);
11392 ingest_delta.source_family_fallback_cells = ingest_delta
11393 .source_family_fallback_cells
11394 .saturating_add(source.family_cells_seen);
11395 } else {
11396 ingest_delta.shadow_candidate_cells = ingest_delta
11397 .shadow_candidate_cells
11398 .saturating_add(source.family_cells_seen);
11399 ingest_delta.shadow_accepted_span_cells = ingest_delta
11400 .shadow_accepted_span_cells
11401 .saturating_add(package.direct_cells);
11402 ingest_delta.shadow_fallback_cells =
11403 ingest_delta.shadow_fallback_cells.saturating_add(
11404 source
11405 .family_cells_seen
11406 .saturating_sub(package.direct_cells),
11407 );
11408 ingest_delta.source_anchor_parses = ingest_delta
11409 .source_anchor_parses
11410 .saturating_add(package.anchor_parses);
11411 ingest_delta.source_anchor_asts = ingest_delta
11412 .source_anchor_asts
11413 .saturating_add(package.anchor_asts);
11414 ingest_delta.source_anchor_analyses = ingest_delta
11415 .source_anchor_analyses
11416 .saturating_add(package.anchor_analyses);
11417 ingest_delta.source_compressed_families_prepared = ingest_delta
11418 .source_compressed_families_prepared
11419 .saturating_add(package.direct_complete_families);
11420 ingest_delta.source_compressed_cells_prepared = ingest_delta
11421 .source_compressed_cells_prepared
11422 .saturating_add(package.direct_complete_cells);
11423 ingest_delta.source_partitioned_families_prepared = ingest_delta
11424 .source_partitioned_families_prepared
11425 .saturating_add(package.direct_partition_families);
11426 ingest_delta.source_partition_fragments_prepared = ingest_delta
11427 .source_partition_fragments_prepared
11428 .saturating_add(package.direct_fragments);
11429 ingest_delta.source_partition_span_cells_prepared = ingest_delta
11430 .source_partition_span_cells_prepared
11431 .saturating_add(package.direct_partition_cells);
11432 ingest_delta.source_partition_analyses_reused = ingest_delta
11433 .source_partition_analyses_reused
11434 .saturating_add(
11435 package
11436 .direct_fragments
11437 .saturating_sub(package.direct_partition_families),
11438 );
11439 ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
11440 .graph_formula_vertices_avoided_shadow
11441 .saturating_add(package.direct_cells);
11442 ingest_delta.ast_roots_avoided_shadow =
11443 ingest_delta.ast_roots_avoided_shadow.saturating_add(
11444 package
11445 .direct_complete_cells
11446 .saturating_sub(package.direct_complete_families)
11447 .saturating_add(
11448 package
11449 .direct_partition_cells
11450 .saturating_sub(package.direct_fragments),
11451 ),
11452 );
11453 ingest_delta.edge_rows_avoided_shadow = ingest_delta
11454 .edge_rows_avoided_shadow
11455 .saturating_add(package.direct_cells);
11456 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11457 {
11458 ingest_delta.source_family_promoted = ingest_delta
11459 .source_family_promoted
11460 .saturating_add(package.direct_families as u64);
11461 ingest_delta.source_family_promoted_cells = ingest_delta
11462 .source_family_promoted_cells
11463 .saturating_add(package.direct_cells);
11464 let descendants = package
11465 .direct_complete_cells
11466 .saturating_sub(package.direct_complete_families)
11467 .saturating_add(
11468 package
11469 .direct_partition_cells
11470 .saturating_sub(package.direct_partition_families),
11471 );
11472 ingest_delta.source_descendant_strings_avoided = ingest_delta
11473 .source_descendant_strings_avoided
11474 .saturating_add(descendants);
11475 ingest_delta.source_descendant_events_avoided = ingest_delta
11476 .source_descendant_events_avoided
11477 .saturating_add(descendants);
11478 ingest_delta.source_descendant_analyses_avoided = ingest_delta
11479 .source_descendant_analyses_avoided
11480 .saturating_add(descendants);
11481 }
11482 ingest_delta.source_family_fallback =
11483 ingest_delta.source_family_fallback.saturating_add(
11484 source
11485 .families_seen
11486 .saturating_sub(package.direct_families as u64),
11487 );
11488 ingest_delta.source_family_fallback_cells =
11489 ingest_delta.source_family_fallback_cells.saturating_add(
11490 source
11491 .family_cells_seen
11492 .saturating_sub(package.direct_cells),
11493 );
11494 }
11495 }
11496 if let Some(report) = plane_report.as_ref() {
11497 ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
11498 ingest_delta.shadow_spans_created = report.spans.len() as u64;
11499 }
11500 self.record_formula_ingest_report(ingest_delta);
11501 }
11502 let commit_window = commit_started.elapsed();
11503 let retained_cells = self
11504 .staged_formula_index
11505 .all_leases()
11506 .into_iter()
11507 .filter_map(|(sheet, lease)| {
11508 formualizer_common::RangeAddress::new(
11509 sheet, lease.row, lease.col, lease.row, lease.col,
11510 )
11511 .ok()
11512 })
11513 .collect::<Vec<_>>();
11514 let observed_scratch_bytes = self
11515 .active_resource_ledger
11516 .as_ref()
11517 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
11518 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11519 let committed_spans = plane_report
11520 .as_ref()
11521 .map_or(0, |report| report.spans.len() as u64);
11522 let selected_source_families = prepared_packages
11523 .iter()
11524 .map(|package| package.lease.family_count)
11525 .sum::<usize>();
11526 let selected_staged_cells = prepared
11527 .len()
11528 .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
11529 let actual_commit_work = (new_vertices as u64)
11530 .saturating_add(new_edges as u64)
11531 .saturating_add(committed as u64)
11532 .saturating_add(committed_spans)
11533 .saturating_add(prepared.len() as u64)
11534 .saturating_add(prepared_packages.len() as u64);
11535 let report = PreparedTargetGraphReport {
11536 request_id: request_id.unwrap_or_default(),
11537 requested_targets: targets.len(),
11538 normalized_regions: visited_regions.len(),
11539 normalized_target_list: normalized,
11540 selected_staged_cells,
11541 selected_source_families,
11542 retained_staged_cells: self.staged_formula_count(),
11543 selected_cells,
11544 retained_cells,
11545 widened_scope: scope,
11546 widening_reasons: reasons,
11547 revisions: assumptions,
11548 commit_window,
11549 estimated_scratch_bytes: scratch_bytes,
11550 observed_scratch_bytes,
11551 estimated_commit_work: (new_vertices as u64)
11552 .saturating_add(new_edges as u64)
11553 .saturating_add(prepared.len() as u64)
11554 .saturating_add(selected_package_records),
11555 actual_commit_work,
11556 outcome: PreparationOutcome::Prepared,
11557 };
11558 self.observe_target_preparation_report(&report);
11559 Ok(report)
11560 }
11561
11562 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11564 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
11565 engine.build_graph_all_unobserved()
11566 })
11567 }
11568
11569 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11570 let selected = self.staged_formula_count();
11571 let started = crate::instant::FzInstant::now();
11572 self.resource_checkpoint(selected as u64)?;
11573 let scratch_bytes = (selected as u64).saturating_mul(256);
11574 let result = self.with_request_scratch(scratch_bytes, |engine| {
11575 let index_snapshot = engine.staged_formula_index.clone();
11576 let collected = std::mem::take(&mut engine.staged_formulas)
11577 .into_iter()
11578 .collect();
11579 engine.staged_formula_index.clear_all();
11580 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
11581 });
11582 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11583 result
11584 }
11585
11586 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
11588 &mut self,
11589 sheets: I,
11590 ) -> Result<(), formualizer_parse::ExcelError> {
11591 let mut sheets = sheets.into_iter();
11592 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
11593 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
11594 engine.with_request_scratch(name_scratch, |engine| {
11595 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
11598 engine.charge_bounded_work(names.len() as u64)?;
11599 engine.build_graph_for_sheet_names_unobserved(names)
11600 })
11601 })
11602 }
11603
11604 fn build_graph_for_sheet_names_unobserved(
11605 &mut self,
11606 sheets: Vec<String>,
11607 ) -> Result<(), formualizer_parse::ExcelError> {
11608 let started = crate::instant::FzInstant::now();
11609 let selected = sheets
11610 .iter()
11611 .filter_map(|sheet| self.staged_formulas.get(sheet))
11612 .map(StagedSheet::len)
11613 .sum::<usize>();
11614 self.resource_checkpoint(selected as u64)?;
11615 let scratch_bytes = (selected as u64).saturating_mul(256);
11616 self.reserve_request_scratch(scratch_bytes)?;
11617 let index_snapshot = self.staged_formula_index.clone();
11618 let mut collected = Vec::new();
11619 for sheet in sheets {
11620 if let Some(staged) = self.staged_formulas.remove(&sheet) {
11621 self.index_removed_staged_sheet(&sheet, &staged);
11622 collected.push((sheet, staged));
11623 }
11624 }
11625 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
11626 self.release_request_scratch(scratch_bytes);
11627 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11628 result
11629 }
11630
11631 fn build_graph_from_staged_batches(
11632 &mut self,
11633 collected: StagedFormulaBatches,
11634 share_parse_cache_across_sheets: bool,
11635 staged_index_snapshot: StagedFormulaIndex,
11636 ) -> Result<(), formualizer_parse::ExcelError> {
11637 if collected.is_empty() {
11638 return Ok(());
11639 }
11640 for (sheet, _) in &collected {
11641 let _ = self.add_sheet(sheet);
11642 }
11643
11644 let diagnostics_len = self.formula_parse_diagnostics.len();
11645 let mut collected = collected;
11646 let prepared = match self
11647 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
11648 {
11649 Ok(prepared) => prepared,
11650 Err(error) => {
11651 self.formula_parse_diagnostics.truncate(diagnostics_len);
11652 for (sheet, staged) in collected {
11653 self.restore_staged_sheet(sheet, staged);
11654 }
11655 self.staged_formula_index = staged_index_snapshot;
11656 return Err(error);
11657 }
11658 };
11659 let (ordinary, compressed, direct) = prepared;
11660
11661 if !ordinary.is_empty()
11664 && let Err(error) = self.ingest_formula_batches(ordinary)
11665 {
11666 self.formula_parse_diagnostics.truncate(diagnostics_len);
11667 for (sheet, staged) in collected {
11668 self.restore_staged_sheet(sheet, staged);
11669 }
11670 self.staged_formula_index = staged_index_snapshot;
11671 return Err(error);
11672 }
11673 if !compressed.is_empty() {
11674 let _ = self.ingest_compressed_formula_source_batches(compressed)?;
11675 }
11676 if !direct.is_empty() {
11677 let _ = self.finish_compressed_formula_sources(direct)?;
11678 }
11679 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
11680 Ok(())
11681 }
11682
11683 fn prepare_staged_formula_batches(
11684 &mut self,
11685 collected: &mut StagedFormulaBatches,
11686 share_parse_cache_across_sheets: bool,
11687 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
11688 let mut ordinary = Vec::new();
11689 let mut compressed = Vec::new();
11690 let mut direct = Vec::new();
11691 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
11692 rustc_hash::FxHashMap::default();
11693 cache.reserve(4096);
11694
11695 for (sheet, staged) in collected {
11696 if !share_parse_cache_across_sheets {
11697 cache.clear();
11698 }
11699 let mut entries: Vec<_> = staged
11700 .entries
11701 .iter()
11702 .cloned()
11703 .map(|(row, col, text)| (row, col, text, None))
11704 .collect();
11705 let mut deferred_source = None;
11706 let mut deferred_fallback = None;
11707 if let Some(package) = staged.deferred_package.as_mut() {
11708 if package.sheet_name != *sheet {
11709 return Err(ExcelError::new(ExcelErrorKind::Value)
11710 .with_message("deferred formula package sheet mismatch"));
11711 }
11712 let eligible: Vec<_> = package
11713 .families
11714 .iter()
11715 .filter(|family| !package.invalidated.contains(&family.source_id))
11716 .cloned()
11717 .collect();
11718 let eligible_partitions: Vec<_> = package
11719 .partitioned_families
11720 .iter()
11721 .filter(|family| !package.invalidated.contains(&family.source_id))
11722 .cloned()
11723 .collect();
11724 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11725 let mut preparation = self.prepare_source_formula_proposals(
11726 sheet,
11727 &eligible,
11728 &eligible_partitions,
11729 &package.partitioned_families,
11730 package.report.source_formula_records_spooled,
11731 Arc::clone(&package.replay),
11732 &package.suppressed,
11733 )?;
11734 let replay_records = std::mem::take(&mut preparation.eager_replay);
11735 let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
11736 let accepted: BTreeMap<_, _> = preparation
11737 .fragmented
11738 .iter()
11739 .map(|fragment| (fragment.source.source_id, &fragment.source))
11740 .collect();
11741 replay_records.into_iter().partition(|record| {
11742 record.partition_owner.is_some_and(|owner| {
11743 accepted.get(&owner).is_some_and(|source| {
11744 Self::exact_replay_record_is_prepared_partition_legacy(
11745 source, record,
11746 )
11747 })
11748 })
11749 })
11750 };
11751 preparation.eager_replay = fragment_legacy;
11752 entries.extend(ordered_fallback.into_iter().map(|record| {
11753 (
11754 record.row,
11755 record.col,
11756 record.text,
11757 Some((record.source_order, record.family, record.partition_owner)),
11758 )
11759 }));
11760 deferred_source = Some((package.report.clone(), preparation));
11761 } else {
11762 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
11763 for partition in &eligible_partitions {
11764 replay_disposition
11765 .register_partition(partition, false)
11766 .map_err(|reason| {
11767 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
11768 })?;
11769 }
11770 replay_disposition
11771 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
11772 let replayed = package
11773 .replay
11774 .lock()
11775 .map_err(|_| {
11776 ExcelError::new(ExcelErrorKind::Value)
11777 .with_message("deferred formula spool lock poisoned")
11778 })?
11779 .replay(&replay_disposition)
11780 .map_err(|message| {
11781 ExcelError::new(ExcelErrorKind::Value).with_message(message)
11782 })?;
11783 entries.extend(replayed.into_iter().map(|record| {
11784 (
11785 record.row,
11786 record.col,
11787 record.text,
11788 Some((record.source_order, record.family, record.partition_owner)),
11789 )
11790 }));
11791 let mut report = package.report.clone();
11792 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
11793 deferred_fallback =
11794 Some((report, package.families.clone(), eligible_partitions));
11795 }
11796 }
11797
11798 let mut formulas = Vec::new();
11799 let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
11800 for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
11801 let key = if txt.starts_with('=') {
11802 txt
11803 } else {
11804 format!("={txt}")
11805 };
11806 let ast_id = if let Some(cached) = cache.get(&key) {
11807 *cached
11808 } else {
11809 let parsed = match formualizer_parse::parser::parse(&key) {
11810 Ok(parsed) => Some(parsed),
11811 Err(error) => self.handle_formula_parse_error(
11812 sheet,
11813 row,
11814 col,
11815 &key,
11816 error.to_string(),
11817 )?,
11818 };
11819 let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
11820 cache.insert(key.clone(), ast_id);
11821 ast_id
11822 };
11823
11824 if let Some(ast_id) = ast_id {
11825 let mut formula = FormulaIngestRecord::new(
11826 row,
11827 col,
11828 ast_id,
11829 Some(Arc::<str>::from(key.clone())),
11830 );
11831 if let Some((order, family, owner)) = source_proof {
11832 formula = formula.with_source_proof(order, family, owner);
11833 } else if deferred_source.is_some() {
11834 formula = formula.with_source_proof(
11835 crate::engine::SourceFormulaOrder::new(
11836 staged_order_base.saturating_add(entry_index as u64),
11837 ),
11838 None,
11839 None,
11840 );
11841 }
11842 formulas.push(formula);
11843 }
11844 }
11845
11846 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
11847 if let Some((report, preparation)) = deferred_source {
11848 direct.push((batch, report, preparation));
11849 } else if let Some((report, families, partitions)) = deferred_fallback {
11850 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
11851 batch.sheet_name.clone(),
11852 report,
11853 families,
11854 partitions,
11855 );
11856 compressed.push((batch, source_batch));
11857 } else if !batch.is_empty() {
11858 ordinary.push(batch);
11859 }
11860 }
11861 Ok((ordinary, compressed, direct))
11862 }
11863
11864 pub fn begin_bulk_ingest_arrow(
11866 &mut self,
11867 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
11868 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
11869 }
11870
11871 pub fn begin_bulk_update_arrow(
11873 &mut self,
11874 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
11875 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
11876 }
11877
11878 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
11879 self.graph.sheet_id(sheet).ok_or(
11880 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11881 name: sheet.to_string(),
11882 reason: "Unknown sheet".to_string(),
11883 },
11884 )
11885 }
11886
11887 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
11888 if row_1based == 0 {
11889 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11890 }
11891 Ok(row_1based - 1)
11892 }
11893
11894 fn normalize_row_range_1based(
11895 start_row_1based: u32,
11896 end_row_1based: u32,
11897 ) -> Result<(u32, u32), crate::engine::EditorError> {
11898 if start_row_1based == 0 || end_row_1based == 0 {
11899 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11900 }
11901 if start_row_1based > end_row_1based {
11902 return Err(crate::engine::EditorError::TransactionFailed {
11903 reason: "Row range start is greater than end".to_string(),
11904 });
11905 }
11906 Ok((start_row_1based - 1, end_row_1based - 1))
11907 }
11908
11909 fn invalidate_row_visibility_mask_cache(&self) {
11910 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11911 cache.clear();
11912 }
11913 }
11914
11915 fn set_row_hidden_by_sheet_id(
11916 &mut self,
11917 sheet_id: SheetId,
11918 row0: u32,
11919 hidden: bool,
11920 source: RowVisibilitySource,
11921 ) -> bool {
11922 let changed = {
11923 let state = self.row_visibility.entry(sheet_id).or_default();
11924 state.set_row_hidden(row0, hidden, source)
11925 };
11926
11927 let remove_entry = self
11928 .row_visibility
11929 .get(&sheet_id)
11930 .map(|state| state.is_empty())
11931 .unwrap_or(false);
11932 if remove_entry {
11933 self.row_visibility.remove(&sheet_id);
11934 }
11935
11936 if changed {
11937 self.invalidate_row_visibility_mask_cache();
11938 }
11939
11940 changed
11941 }
11942
11943 fn set_rows_hidden_by_sheet_id(
11944 &mut self,
11945 sheet_id: SheetId,
11946 start_row0: u32,
11947 end_row0: u32,
11948 hidden: bool,
11949 source: RowVisibilitySource,
11950 ) -> bool {
11951 let changed = {
11952 let state = self.row_visibility.entry(sheet_id).or_default();
11953 state.set_rows_hidden(start_row0, end_row0, hidden, source)
11954 };
11955
11956 let remove_entry = self
11957 .row_visibility
11958 .get(&sheet_id)
11959 .map(|state| state.is_empty())
11960 .unwrap_or(false);
11961 if remove_entry {
11962 self.row_visibility.remove(&sheet_id);
11963 }
11964
11965 if changed {
11966 self.invalidate_row_visibility_mask_cache();
11967 }
11968
11969 changed
11970 }
11971
11972 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
11973 if count == 0 {
11974 return;
11975 }
11976 let mut changed = false;
11977 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
11978 changed = state.insert_rows(before0, count);
11979 state.is_empty()
11980 } else {
11981 false
11982 };
11983 if remove_entry {
11984 self.row_visibility.remove(&sheet_id);
11985 }
11986 if changed {
11987 self.invalidate_row_visibility_mask_cache();
11988 }
11989 }
11990
11991 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
11992 if count == 0 {
11993 return;
11994 }
11995 let mut changed = false;
11996 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
11997 changed = state.delete_rows(start0, count);
11998 state.is_empty()
11999 } else {
12000 false
12001 };
12002 if remove_entry {
12003 self.row_visibility.remove(&sheet_id);
12004 }
12005 if changed {
12006 self.invalidate_row_visibility_mask_cache();
12007 }
12008 }
12009
12010 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12011 if let crate::engine::ChangeEvent::SetRowVisibility {
12012 sheet_id,
12013 row0,
12014 source,
12015 old_hidden,
12016 ..
12017 } = event
12018 {
12019 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12020 }
12021 }
12022
12023 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12024 if let crate::engine::ChangeEvent::SetRowVisibility {
12025 sheet_id,
12026 row0,
12027 source,
12028 new_hidden,
12029 ..
12030 } = event
12031 {
12032 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12033 }
12034 }
12035
12036 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12037 for event in events.iter().rev() {
12038 self.apply_inverse_row_visibility_event(event);
12039 }
12040 }
12041
12042 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12043 for event in events {
12044 self.apply_forward_row_visibility_event(event);
12045 }
12046 }
12047
12048 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12049 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12050 sheet,
12051 row,
12052 col,
12053 old,
12054 ..
12055 } = event
12056 {
12057 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12058 }
12059 }
12060
12061 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12062 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12063 sheet,
12064 row,
12065 col,
12066 new,
12067 ..
12068 } = event
12069 {
12070 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12071 }
12072 }
12073
12074 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12077 match target {
12078 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12079 None => {
12080 self.clear_staged_formula_text(sheet, row, col);
12081 }
12082 }
12083 }
12084
12085 pub fn set_row_hidden(
12086 &mut self,
12087 sheet: &str,
12088 row_1based: u32,
12089 hidden: bool,
12090 source: RowVisibilitySource,
12091 ) -> Result<(), crate::engine::EditorError> {
12092 self.observe_function_semantic_epoch()
12093 .map_err(crate::engine::EditorError::Excel)?;
12094 self.observe_function_semantic_epoch()
12095 .map_err(crate::engine::EditorError::Excel)?;
12096 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12097 let row0 = Self::normalize_row_1based(row_1based)?;
12098 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12099 self.record_formula_plane_structural_change(StructuralScope::Region(
12100 Region::whole_row(sheet_id, row0),
12101 ));
12102 self.mark_data_edited();
12103 }
12104 Ok(())
12105 }
12106
12107 pub fn set_rows_hidden(
12108 &mut self,
12109 sheet: &str,
12110 start_row_1based: u32,
12111 end_row_1based: u32,
12112 hidden: bool,
12113 source: RowVisibilitySource,
12114 ) -> Result<(), crate::engine::EditorError> {
12115 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12116 let (start_row0, end_row0) =
12117 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12118 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12119 if start_row0 == end_row0 {
12120 self.record_formula_plane_structural_change(StructuralScope::Region(
12121 Region::whole_row(sheet_id, start_row0),
12122 ));
12123 } else {
12124 self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12125 }
12126 self.mark_data_edited();
12127 }
12128 Ok(())
12129 }
12130
12131 pub fn is_row_hidden(
12132 &self,
12133 sheet: &str,
12134 row_1based: u32,
12135 source: Option<RowVisibilitySource>,
12136 ) -> Option<bool> {
12137 let sheet_id = self.graph.sheet_id(sheet)?;
12138 let row0 = row_1based.checked_sub(1)?;
12139 Some(
12140 self.row_visibility
12141 .get(&sheet_id)
12142 .map(|state| state.is_row_hidden(row0, source))
12143 .unwrap_or(false),
12144 )
12145 }
12146
12147 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12148 let sheet_id = self.graph.sheet_id(sheet)?;
12149 Some(
12150 self.row_visibility
12151 .get(&sheet_id)
12152 .map(|state| state.version())
12153 .unwrap_or(0),
12154 )
12155 }
12156
12157 fn build_row_visibility_mask_for_view(
12158 &self,
12159 view: &RangeView<'_>,
12160 mode: VisibilityMaskMode,
12161 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12162 let sheet_rows = view.sheet().nrows as usize;
12163 if sheet_rows == 0 || view.start_row() >= sheet_rows {
12164 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12165 }
12166
12167 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12168 let start_row0 = view.start_row() as u32;
12169 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12170 let version = self
12171 .row_visibility
12172 .get(&sheet_id)
12173 .map(|state| state.version())
12174 .unwrap_or(0);
12175 let key = VisibilityMaskCacheKey {
12176 sheet_id,
12177 start_row0,
12178 end_row0,
12179 mode,
12180 version,
12181 };
12182
12183 if let Ok(cache) = self.row_visibility_mask_cache.read()
12184 && let Some(mask) = cache.get(&key)
12185 {
12186 #[cfg(test)]
12187 visibility_mask_test_hooks::inc_hit();
12188 return Some(mask.clone());
12189 }
12190
12191 #[cfg(test)]
12192 visibility_mask_test_hooks::inc_miss();
12193
12194 let state = self.row_visibility.get(&sheet_id);
12195 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12196 for row0 in start_row0..=end_row0 {
12197 let manual_hidden = state
12198 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12199 .unwrap_or(false);
12200 let filter_hidden = state
12201 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12202 .unwrap_or(false);
12203
12204 let include = match mode {
12205 VisibilityMaskMode::IncludeAll => true,
12206 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12207 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12208 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12209 !(manual_hidden || filter_hidden)
12210 }
12211 };
12212 out.push(include);
12213 }
12214
12215 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12216 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12217 const MAX_CACHE_ENTRIES: usize = 4096;
12218 if cache.len() >= MAX_CACHE_ENTRIES {
12219 cache.clear();
12220 #[cfg(test)]
12221 visibility_mask_test_hooks::inc_eviction();
12222 }
12223 cache.insert(key, mask.clone());
12224 }
12225
12226 Some(mask)
12227 }
12228
12229 fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
12230 match error {
12231 crate::engine::EditorError::Excel(error) => error,
12232 other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
12233 }
12234 }
12235
12236 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
12237 let changes =
12238 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
12239 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
12240 let provider_revision = self.resolver.planning_semantic_revision();
12241 let provider_changed = provider_revision != self.function_provider_revision_seen;
12242 if !global_changed && !provider_changed {
12243 return Ok(false);
12244 }
12245
12246 let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
12247 let sheets: BTreeSet<_> = self
12248 .graph
12249 .formula_authority()
12250 .plane
12251 .spans
12252 .active_spans()
12253 .filter_map(|span| {
12254 let Some(template) = self
12255 .graph
12256 .formula_authority()
12257 .plane
12258 .templates
12259 .get(span.template_id)
12260 else {
12261 return (provider_changed || global_changed && !changes.complete)
12262 .then_some(span.domain.sheet_id());
12263 };
12264 let Some(ast) = self
12265 .graph
12266 .data_store()
12267 .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
12268 else {
12269 return (provider_changed || global_changed && !changes.complete)
12270 .then_some(span.domain.sheet_id());
12271 };
12272 let global_affected = global_changed
12273 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
12274 let provider_affected = provider_changed && Self::ast_contains_function(&ast);
12275 (global_affected || provider_affected).then_some(span.domain.sheet_id())
12276 })
12277 .collect();
12278 let invalidated = !sheets.is_empty();
12279 for sheet_id in sheets {
12280 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
12281 .map_err(Self::editor_error_to_excel)?;
12282 }
12283 if global_changed && !changed.is_empty() || provider_changed {
12284 self.cached_static_schedule = None;
12285 }
12286 self.function_semantic_epoch_seen = changes.epoch;
12287 self.function_provider_revision_seen = provider_revision;
12288 Ok(invalidated)
12289 }
12290
12291 pub(crate) fn ast_uses_changed_function(
12292 ast: &ASTNode,
12293 changed: &BTreeSet<(String, String)>,
12294 ) -> bool {
12295 match &ast.node_type {
12296 ASTNodeType::Function { name, args } => {
12297 let normalized = name.to_uppercase();
12298 let mut spellings = vec![(String::new(), normalized.clone())];
12299 let mut stripped = normalized.as_str();
12300 loop {
12301 let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
12302 .iter()
12303 .find_map(|prefix| stripped.strip_prefix(prefix))
12304 else {
12305 break;
12306 };
12307 stripped = rest;
12308 spellings.push((String::new(), stripped.to_string()));
12309 }
12310 let resolved = crate::function_registry::resolve("", name);
12311 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
12312 || resolved.as_ref().is_some_and(|resolved| {
12313 changed.contains(&(
12314 resolved.namespace.clone(),
12315 resolved.canonical_name.clone(),
12316 ))
12317 });
12318 directly_changed
12319 || resolved.is_none()
12320 || args
12321 .iter()
12322 .any(|arg| Self::ast_uses_changed_function(arg, changed))
12323 }
12324 ASTNodeType::Call { callee, args } => {
12325 Self::ast_uses_changed_function(callee, changed)
12326 || args
12327 .iter()
12328 .any(|arg| Self::ast_uses_changed_function(arg, changed))
12329 }
12330 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
12331 ASTNodeType::BinaryOp { left, right, .. } => {
12332 Self::ast_uses_changed_function(left, changed)
12333 || Self::ast_uses_changed_function(right, changed)
12334 }
12335 ASTNodeType::Array(rows) => rows
12336 .iter()
12337 .flatten()
12338 .any(|node| Self::ast_uses_changed_function(node, changed)),
12339 _ => false,
12340 }
12341 }
12342
12343 fn ast_contains_function(ast: &ASTNode) -> bool {
12344 match &ast.node_type {
12345 ASTNodeType::Function { .. } => true,
12346 ASTNodeType::Call { callee, args } => {
12347 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
12348 }
12349 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
12350 ASTNodeType::BinaryOp { left, right, .. } => {
12351 Self::ast_contains_function(left) || Self::ast_contains_function(right)
12352 }
12353 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
12354 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
12355 }
12356 }
12357
12358 fn demote_span_containing_cell_for_write(
12359 &mut self,
12360 sheet_id: SheetId,
12361 row0: u32,
12362 col0: u32,
12363 ) -> Result<(), crate::engine::EditorError> {
12364 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12365 return Ok(());
12366 }
12367 let placement = PlacementCoord::new(sheet_id, row0, col0);
12368 let inside_active_span = self
12369 .graph
12370 .formula_authority()
12371 .plane
12372 .spans
12373 .find_at(placement)
12374 .is_some();
12375 if inside_active_span {
12376 self.demote_spans_preserving_computed_overlays(
12377 sheet_id,
12378 Region::point(sheet_id, row0, col0),
12379 )?;
12380 }
12381 Ok(())
12382 }
12383
12384 fn demote_spans_preserving_computed_overlays(
12385 &mut self,
12386 _sheet_id: SheetId,
12387 affected_region: Region,
12388 ) -> Result<(), crate::engine::EditorError> {
12389 self.demote_spans_for_structural_op_impl(None, affected_region, false)
12393 }
12394
12395 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
12396 Region::rows_from(sheet_id, start_row0)
12397 }
12398
12399 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
12400 Region::cols_from(sheet_id, start_col0)
12401 }
12402
12403 fn structural_dirty_region_for_domain(
12404 domain: &PlacementDomain,
12405 op: StructuralOp,
12406 ) -> Option<Region> {
12407 let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
12408 PlacementDomain::RowRun {
12409 sheet_id,
12410 row_start,
12411 row_end,
12412 col,
12413 } => (*sheet_id, *row_start, *row_end, *col, *col),
12414 PlacementDomain::ColRun {
12415 sheet_id,
12416 row,
12417 col_start,
12418 col_end,
12419 } => (*sheet_id, *row, *row, *col_start, *col_end),
12420 PlacementDomain::Rect {
12421 sheet_id,
12422 row_start,
12423 row_end,
12424 col_start,
12425 col_end,
12426 } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
12427 };
12428 match op {
12429 StructuralOp::InsertRows {
12430 sheet_id: edited,
12431 before,
12432 ..
12433 } if sheet_id == edited && row_end >= before => Some(Region::rect(
12434 sheet_id,
12435 row_start.max(before),
12436 row_end,
12437 col_start,
12438 col_end,
12439 )),
12440 StructuralOp::DeleteRows {
12441 sheet_id: edited,
12442 start,
12443 ..
12444 } if sheet_id == edited && row_end >= start => Some(Region::rect(
12445 sheet_id,
12446 row_start.max(start),
12447 row_end,
12448 col_start,
12449 col_end,
12450 )),
12451 StructuralOp::InsertColumns {
12452 sheet_id: edited,
12453 before,
12454 ..
12455 } if sheet_id == edited && col_end >= before => Some(Region::rect(
12456 sheet_id,
12457 row_start,
12458 row_end,
12459 col_start.max(before),
12460 col_end,
12461 )),
12462 StructuralOp::DeleteColumns {
12463 sheet_id: edited,
12464 start,
12465 ..
12466 } if sheet_id == edited && col_end >= start => Some(Region::rect(
12467 sheet_id,
12468 row_start,
12469 row_end,
12470 col_start.max(start),
12471 col_end,
12472 )),
12473 _ => None,
12474 }
12475 }
12476
12477 fn span_result_region_intersects_affected(
12478 span: &crate::formula_plane::runtime::FormulaSpan,
12479 affected_region: &Region,
12480 ) -> bool {
12481 Region::from_domain(span.result_region.domain()).intersects(affected_region)
12482 }
12483
12484 fn span_any_read_region_intersects_affected(
12485 plane: &FormulaPlane,
12486 span: &crate::formula_plane::runtime::FormulaSpan,
12487 affected_region: &Region,
12488 ) -> bool {
12489 span.read_summary_id
12490 .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
12491 .is_some_and(|summary| {
12492 summary
12493 .dependencies
12494 .iter()
12495 .any(|dependency| dependency.read_region.intersects(affected_region))
12496 })
12497 }
12498
12499 fn insert_formula_plane_dirty_coords_for_span(
12500 &self,
12501 span_ref: FormulaSpanRef,
12502 dirty: ProducerDirtyDomain,
12503 out: &mut FxHashSet<(SheetId, u32, u32)>,
12504 ) -> Result<(), crate::engine::EditorError> {
12505 let authority = self.graph.formula_authority();
12506 let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
12507 ExcelError::new(ExcelErrorKind::NImpl)
12508 .with_message("FormulaPlane dirty transfer referenced a stale span")
12509 })?;
12510 match dirty {
12511 ProducerDirtyDomain::Whole => {
12512 out.extend(
12513 span.domain
12514 .iter()
12515 .map(|coord| (coord.sheet_id, coord.row, coord.col)),
12516 );
12517 }
12518 ProducerDirtyDomain::Cells(cells) => {
12519 out.extend(cells.into_iter().filter_map(|key| {
12520 let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
12521 span.domain
12522 .contains(coord)
12523 .then_some((coord.sheet_id, coord.row, coord.col))
12524 }));
12525 }
12526 ProducerDirtyDomain::Regions(regions) => {
12527 out.extend(span.domain.iter().filter_map(|coord| {
12528 let key = crate::formula_plane::region_index::RegionKey::from(coord);
12529 regions
12530 .iter()
12531 .any(|region| region.contains_key(key))
12532 .then_some((coord.sheet_id, coord.row, coord.col))
12533 }));
12534 }
12535 }
12536 Ok(())
12537 }
12538
12539 fn compute_current_formula_plane_dirty_result_coords(
12540 &self,
12541 ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
12542 use crate::formula_plane::producer::compute_dirty_closure;
12543
12544 let authority = self.graph.formula_authority();
12545 let span_refs = authority.active_span_refs();
12546 let span_refs_by_id = span_refs
12547 .iter()
12548 .copied()
12549 .map(|span_ref| (span_ref.id, span_ref))
12550 .collect::<BTreeMap<_, _>>();
12551 let mut dirty_coords = FxHashSet::default();
12552
12553 for span_ref in self.graph.pending_formula_dirty_whole_spans() {
12554 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12555 self.insert_formula_plane_dirty_coords_for_span(
12556 span_ref,
12557 ProducerDirtyDomain::Whole,
12558 &mut dirty_coords,
12559 )?;
12560 }
12561 }
12562 for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
12563 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12564 self.insert_formula_plane_dirty_coords_for_span(
12565 span_ref,
12566 ProducerDirtyDomain::Regions(vec![region]),
12567 &mut dirty_coords,
12568 )?;
12569 }
12570 }
12571
12572 let pending_changed_regions = self
12573 .graph
12574 .pending_formula_dirty_regions()
12575 .collect::<Vec<_>>();
12576 if pending_changed_regions.is_empty() {
12577 return Ok(dirty_coords);
12578 }
12579
12580 let closure = compute_dirty_closure(
12581 &authority.consumer_reads,
12582 pending_changed_regions,
12583 |producer| authority.producer_results.producer_result_region(producer),
12584 );
12585 if closure.incomplete {
12586 for span_ref in span_refs {
12587 self.insert_formula_plane_dirty_coords_for_span(
12588 span_ref,
12589 ProducerDirtyDomain::Whole,
12590 &mut dirty_coords,
12591 )?;
12592 }
12593 return Ok(dirty_coords);
12594 }
12595 for work in closure.work {
12596 let FormulaProducerId::Span(span_id) = work.producer else {
12597 continue;
12598 };
12599 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12600 continue;
12601 };
12602 self.insert_formula_plane_dirty_coords_for_span(
12603 span_ref,
12604 work.dirty,
12605 &mut dirty_coords,
12606 )?;
12607 }
12608 for fallback in closure.fallbacks {
12609 let FormulaProducerId::Span(span_id) = fallback.consumer else {
12610 continue;
12611 };
12612 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12613 continue;
12614 };
12615 self.insert_formula_plane_dirty_coords_for_span(
12616 span_ref,
12617 ProducerDirtyDomain::Whole,
12618 &mut dirty_coords,
12619 )?;
12620 }
12621
12622 Ok(dirty_coords)
12623 }
12624
12625 fn demote_spans_for_structural_op(
12635 &mut self,
12636 op: StructuralOp,
12637 affected_region: Region,
12638 ) -> Result<(), crate::engine::EditorError> {
12639 if op.count() == 0 {
12640 return Ok(());
12641 }
12642 self.legacy_island_structural_summaries_trusted = false;
12645 self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
12646 }
12647
12648 fn indexed_structural_candidate_span_refs(
12649 &self,
12650 affected_region: Region,
12651 ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
12652 use crate::formula_plane::region_index::BoundedRegionQueryResult;
12653
12654 let authority = self.graph.formula_authority();
12655 if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
12656 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12657 .with_message("FormulaPlane structural candidate indexes are stale")
12658 .into());
12659 }
12660 let indexed_region = match affected_region.axis_ranges() {
12661 (
12662 crate::formula_plane::region_index::AxisRange::From(start),
12663 crate::formula_plane::region_index::AxisRange::All,
12664 ) => Region {
12665 sheet_id: affected_region.sheet_id(),
12666 rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12667 cols: crate::formula_plane::region_index::AxisRange::All,
12668 },
12669 (
12670 crate::formula_plane::region_index::AxisRange::All,
12671 crate::formula_plane::region_index::AxisRange::From(start),
12672 ) => Region {
12673 sheet_id: affected_region.sheet_id(),
12674 rows: crate::formula_plane::region_index::AxisRange::All,
12675 cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12676 },
12677 _ => affected_region,
12678 };
12679 let max_candidates = self.config.max_formula_plane_cache_candidates;
12680 let result_query = match authority
12681 .producer_results
12682 .query_bounded(indexed_region, max_candidates)
12683 {
12684 BoundedRegionQueryResult::Complete(result) => result,
12685 BoundedRegionQueryResult::Incomplete {
12686 observed_candidates,
12687 } => {
12688 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12689 .with_message(format!(
12690 "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
12691 ))
12692 .into());
12693 }
12694 };
12695 let remaining = max_candidates.saturating_sub(result_query.matches.len());
12696 let read_query = match authority
12697 .consumer_reads
12698 .query_changed_region_bounded(indexed_region, remaining)
12699 {
12700 BoundedRegionQueryResult::Complete(result) => result,
12701 BoundedRegionQueryResult::Incomplete { .. } => {
12702 return Err(ExcelError::new(ExcelErrorKind::NImpl)
12703 .with_message("FormulaPlane structural read candidate limit exceeded")
12704 .into());
12705 }
12706 };
12707
12708 let mut ids = BTreeSet::new();
12709 for matched in result_query.matches {
12710 if let FormulaProducerId::Span(id) = matched.value.producer {
12711 ids.insert(id);
12712 }
12713 }
12714 for matched in read_query.matches {
12715 if let FormulaProducerId::Span(id) = matched.value.consumer {
12716 ids.insert(id);
12717 }
12718 }
12719
12720 ids.into_iter()
12721 .map(|id| {
12722 authority.plane.spans.current_ref(id).ok_or_else(|| {
12723 crate::engine::EditorError::Excel(
12724 ExcelError::new(ExcelErrorKind::NImpl)
12725 .with_message("FormulaPlane structural index referenced a stale span"),
12726 )
12727 })
12728 })
12729 .collect()
12730 }
12731
12732 fn demote_spans_for_structural_op_impl(
12733 &mut self,
12734 op: Option<StructuralOp>,
12735 affected_region: Region,
12736 clear_computed_overlays: bool,
12737 ) -> Result<(), crate::engine::EditorError> {
12738 struct SpanPlan {
12739 span_ref: FormulaSpanRef,
12740 sheet_id: SheetId,
12741 ast: ASTNode,
12742 origin_row: u32,
12743 origin_col: u32,
12744 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12745 placements: Vec<(u32, u32)>,
12746 }
12747
12748 fn substitute_literal_slots_for_template_placement(
12749 ast: &ASTNode,
12750 binding: &[LiteralValue],
12751 ) -> ASTNode {
12752 fn clone_with_slots(
12753 ast: &ASTNode,
12754 binding: &[LiteralValue],
12755 next: &mut usize,
12756 in_array: bool,
12757 ) -> ASTNode {
12758 let node_type = match &ast.node_type {
12759 ASTNodeType::Literal(_) if !in_array => {
12760 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
12761 *next = next.saturating_add(1);
12762 ASTNodeType::Literal(value)
12763 }
12764 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
12765 ASTNodeType::Omitted => ASTNodeType::Omitted,
12766 ASTNodeType::Reference {
12767 original,
12768 reference,
12769 } => ASTNodeType::Reference {
12770 original: original.clone(),
12771 reference: reference.clone(),
12772 },
12773 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
12774 op: op.clone(),
12775 expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
12776 },
12777 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
12778 op: op.clone(),
12779 left: Box::new(clone_with_slots(left, binding, next, in_array)),
12780 right: Box::new(clone_with_slots(right, binding, next, in_array)),
12781 },
12782 ASTNodeType::Function { name, args } => ASTNodeType::Function {
12783 name: name.clone(),
12784 args: args
12785 .iter()
12786 .map(|arg| clone_with_slots(arg, binding, next, in_array))
12787 .collect(),
12788 },
12789 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
12790 callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
12791 args: args
12792 .iter()
12793 .map(|arg| clone_with_slots(arg, binding, next, in_array))
12794 .collect(),
12795 },
12796 ASTNodeType::Array(rows) => ASTNodeType::Array(
12797 rows.iter()
12798 .map(|row| {
12799 row.iter()
12800 .map(|cell| clone_with_slots(cell, binding, next, true))
12801 .collect()
12802 })
12803 .collect(),
12804 ),
12805 };
12806 ASTNode::new(node_type, ast.source_token.clone())
12807 }
12808 let mut next = 0usize;
12809 clone_with_slots(ast, binding, &mut next, false)
12810 }
12811
12812 let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
12813 self.formula_plane_structural_span_candidates = self
12814 .formula_plane_structural_span_candidates
12815 .saturating_add(span_refs.len() as u64);
12816 if span_refs.is_empty() {
12817 return Ok(());
12818 }
12819 let dirty_span_coords = if clear_computed_overlays {
12820 FxHashSet::default()
12821 } else {
12822 self.compute_current_formula_plane_dirty_result_coords()?
12823 };
12824
12825 struct SpanTemplateRewrite {
12832 adjusted_ast: ASTNode,
12833 exact_canonical_key: Arc<str>,
12834 parameterized_canonical_key: Arc<str>,
12835 formula_text: Option<Arc<str>>,
12836 canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
12842 value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
12843 }
12844
12845 struct ShiftPlan {
12846 span_ref: FormulaSpanRef,
12847 template_id: crate::formula_plane::ids::FormulaTemplateId,
12848 new_origin_row: u32,
12849 new_origin_col: u32,
12850 new_domain: crate::formula_plane::runtime::PlacementDomain,
12851 new_read_summary: Option<SpanReadSummary>,
12852 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12853 force_binding_residual_axes: bool,
12854 rewrite: Option<SpanTemplateRewrite>,
12855 dirty_region: Option<Region>,
12856 }
12857
12858 fn shift_operation_for(
12861 op: StructuralOp,
12862 ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
12863 use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
12864 match op {
12865 StructuralOp::InsertRows {
12866 sheet_id,
12867 before,
12868 count,
12869 } => ShiftOperation::InsertRows {
12870 sheet_id,
12871 before,
12872 count,
12873 },
12874 StructuralOp::DeleteRows {
12875 sheet_id,
12876 start,
12877 count,
12878 } => ShiftOperation::DeleteRows {
12879 sheet_id,
12880 start,
12881 count,
12882 },
12883 StructuralOp::InsertColumns {
12884 sheet_id,
12885 before,
12886 count,
12887 } => ShiftOperation::InsertColumns {
12888 sheet_id,
12889 before,
12890 count,
12891 },
12892 StructuralOp::DeleteColumns {
12893 sheet_id,
12894 start,
12895 count,
12896 } => ShiftOperation::DeleteColumns {
12897 sheet_id,
12898 start,
12899 count,
12900 },
12901 }
12902 }
12903
12904 struct SplitPlan {
12909 span_ref: FormulaSpanRef,
12910 sheet_id: SheetId,
12911 template_id: crate::formula_plane::ids::FormulaTemplateId,
12912 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12913 is_constant_result: bool,
12914 upper_domain: crate::formula_plane::runtime::PlacementDomain,
12915 upper_read_summary: Option<SpanReadSummary>,
12916 lower_new_origin_row: u32,
12917 lower_new_origin_col: u32,
12918 lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
12919 lower_new_read_summary: Option<SpanReadSummary>,
12920 lower_force_binding_residual_axes: bool,
12921 lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
12922 upper_dirty_region: Option<Region>,
12923 lower_dirty_region: Option<Region>,
12924 }
12925
12926 fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
12927 u32::try_from(i64::from(value).checked_add(delta)?).ok()
12928 }
12929
12930 fn shifted_read_summary(
12941 read_summary: &SpanReadSummary,
12942 new_result_region: Region,
12943 op: StructuralOp,
12944 row_delta: i64,
12945 col_delta: i64,
12946 origin_row_delta: i64,
12947 origin_col_delta: i64,
12948 ) -> Option<SpanReadSummary> {
12949 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
12950 for dependency in &read_summary.dependencies {
12951 let read_region = match op.classify_region(dependency.read_region) {
12952 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
12953 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
12954 dependency.read_region
12955 }
12956 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
12957 ..
12958 } => dependency
12959 .read_region
12960 .project_through_axis_shift(row_delta, col_delta)?,
12961 crate::formula_plane::structural_shift::AxisShiftCase::Straddles
12962 | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
12963 return None;
12964 }
12965 };
12966 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
12967 read_region,
12968 projection: dependency
12969 .projection
12970 .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
12971 });
12972 }
12973 Some(SpanReadSummary {
12974 result_region: new_result_region,
12975 dependencies,
12976 })
12977 }
12978
12979 fn compact_axis_through_delete(
12980 min: u32,
12981 max: u32,
12982 start: u32,
12983 count: u32,
12984 ) -> Option<(u32, u32)> {
12985 let end = start.saturating_add(count);
12986 if max < start || min >= end {
12987 return Some((min.saturating_sub(count), max.saturating_sub(count)));
12988 }
12989 let keeps_left = min < start;
12990 let keeps_right = max >= end;
12991 match (keeps_left, keeps_right) {
12992 (false, false) => None,
12993 (true, false) => Some((min, start.checked_sub(1)?)),
12994 (false, true) => Some((start, max.checked_sub(count)?)),
12995 (true, true) => Some((min, max.checked_sub(count)?)),
12996 }
12997 }
12998
12999 fn compact_domain_through_delete(
13000 domain: &PlacementDomain,
13001 op: StructuralOp,
13002 ) -> Option<PlacementDomain> {
13003 match (domain, op) {
13004 (
13005 PlacementDomain::RowRun {
13006 sheet_id,
13007 row_start,
13008 row_end,
13009 col,
13010 },
13011 StructuralOp::DeleteRows { start, count, .. },
13012 ) => {
13013 let (row_start, row_end) =
13014 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13015 Some(PlacementDomain::row_run(
13016 *sheet_id, row_start, row_end, *col,
13017 ))
13018 }
13019 (
13020 PlacementDomain::Rect {
13021 sheet_id,
13022 row_start,
13023 row_end,
13024 col_start,
13025 col_end,
13026 },
13027 StructuralOp::DeleteRows { start, count, .. },
13028 ) => {
13029 let (row_start, row_end) =
13030 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13031 Some(PlacementDomain::rect(
13032 *sheet_id, row_start, row_end, *col_start, *col_end,
13033 ))
13034 }
13035 (
13036 PlacementDomain::ColRun {
13037 sheet_id,
13038 row,
13039 col_start,
13040 col_end,
13041 },
13042 StructuralOp::DeleteColumns { start, count, .. },
13043 ) => {
13044 let (col_start, col_end) =
13045 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13046 Some(PlacementDomain::col_run(
13047 *sheet_id, *row, col_start, col_end,
13048 ))
13049 }
13050 (
13051 PlacementDomain::Rect {
13052 sheet_id,
13053 row_start,
13054 row_end,
13055 col_start,
13056 col_end,
13057 },
13058 StructuralOp::DeleteColumns { start, count, .. },
13059 ) => {
13060 let (col_start, col_end) =
13061 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13062 Some(PlacementDomain::rect(
13063 *sheet_id, *row_start, *row_end, col_start, col_end,
13064 ))
13065 }
13066 _ => None,
13067 }
13068 }
13069
13070 fn compact_axis_range_through_delete(
13071 axis: crate::formula_plane::region_index::AxisRange,
13072 start: u32,
13073 count: u32,
13074 ) -> Option<crate::formula_plane::region_index::AxisRange> {
13075 use crate::formula_plane::region_index::AxisRange;
13076 match axis {
13077 AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13078 .map(|(point, _)| AxisRange::Point(point)),
13079 AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13080 .map(|(min, max)| AxisRange::Span(min, max)),
13081 AxisRange::All => Some(AxisRange::All),
13082 AxisRange::From(_) | AxisRange::To(_) => None,
13083 }
13084 }
13085
13086 fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13087 let (rows, cols) = region.axis_ranges();
13088 match op {
13089 StructuralOp::DeleteRows {
13090 sheet_id,
13091 start,
13092 count,
13093 } if region.sheet_id() == sheet_id => Some(Region {
13094 sheet_id,
13095 rows: compact_axis_range_through_delete(rows, start, count)?,
13096 cols,
13097 }),
13098 StructuralOp::DeleteColumns {
13099 sheet_id,
13100 start,
13101 count,
13102 } if region.sheet_id() == sheet_id => Some(Region {
13103 sheet_id,
13104 rows,
13105 cols: compact_axis_range_through_delete(cols, start, count)?,
13106 }),
13107 _ => Some(region),
13108 }
13109 }
13110
13111 fn compact_read_summary_through_delete(
13112 read_summary: &SpanReadSummary,
13113 new_result_region: Region,
13114 op: StructuralOp,
13115 ) -> Option<SpanReadSummary> {
13116 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13117 for dependency in &read_summary.dependencies {
13118 let read_region = match op.classify_region(dependency.read_region) {
13119 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13120 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13121 dependency.read_region
13122 }
13123 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13124 ..
13125 } => {
13126 let (row_delta, col_delta) = op.axis_shift_delta();
13127 dependency
13128 .read_region
13129 .project_through_axis_shift(row_delta, col_delta)?
13130 }
13131 crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13132 compact_region_through_delete(dependency.read_region, op)?
13133 }
13134 crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13135 return None;
13136 }
13137 };
13138 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13139 read_region,
13140 projection: dependency.projection,
13141 });
13142 }
13143 Some(SpanReadSummary {
13144 result_region: new_result_region,
13145 dependencies,
13146 })
13147 }
13148
13149 fn rederive_read_summary_for_result(
13156 read_summary: &SpanReadSummary,
13157 result_region: Region,
13158 ) -> Option<SpanReadSummary> {
13159 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13160 for dependency in &read_summary.dependencies {
13161 let sheet_id = dependency.read_region.sheet_id();
13162 let read_regions = dependency
13163 .projection
13164 .read_regions_for_result(sheet_id, result_region)
13165 .ok()?;
13166 for read_region in read_regions {
13167 let dependency = crate::formula_plane::producer::SpanReadDependency {
13168 read_region,
13169 projection: dependency.projection,
13170 };
13171 if !dependencies.contains(&dependency) {
13172 dependencies.push(dependency);
13173 }
13174 }
13175 }
13176 Some(SpanReadSummary {
13177 result_region,
13178 dependencies,
13179 })
13180 }
13181
13182 fn plan_span_split(
13189 authority: &crate::formula_plane::authority::FormulaAuthority,
13190 span: &crate::formula_plane::runtime::FormulaSpan,
13191 span_ref: FormulaSpanRef,
13192 read_summary: Option<&SpanReadSummary>,
13193 op: StructuralOp,
13194 ) -> Option<SplitPlan> {
13195 use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13196
13197 if span.intrinsic_mask_id.is_some() {
13198 return None;
13200 }
13201 let binding_set = span
13202 .binding_set_id
13203 .and_then(|id| authority.plane.binding_sets.get(id));
13204 if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13205 return None;
13209 }
13210
13211 let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13212 let upper_result_region = Region::from_domain(&upper_domain);
13213 let lower_result_region = Region::from_domain(&lower_domain);
13214 let (upper_read_summary, lower_read_summary) = match read_summary {
13215 Some(summary) => (
13216 Some(rederive_read_summary_for_result(
13217 summary,
13218 upper_result_region,
13219 )?),
13220 Some(rederive_read_summary_for_result(
13221 summary,
13222 lower_result_region,
13223 )?),
13224 ),
13225 None => (None, None),
13226 };
13227
13228 let upper_span = crate::formula_plane::runtime::FormulaSpan {
13232 domain: upper_domain.clone(),
13233 result_region: ResultRegion::scalar_cells(upper_domain.clone()),
13234 ..span.clone()
13235 };
13236 if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
13237 != SpanShiftPlan::NoOp
13238 {
13239 return None;
13240 }
13241 let lower_span = crate::formula_plane::runtime::FormulaSpan {
13242 domain: lower_domain.clone(),
13243 result_region: ResultRegion::scalar_cells(lower_domain.clone()),
13244 ..span.clone()
13245 };
13246 let SpanShiftPlan::Shift {
13247 row_delta,
13248 col_delta,
13249 origin_row_delta,
13250 origin_col_delta,
13251 rewrite_absolute_reads,
13252 } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
13253 else {
13254 return None;
13255 };
13256 if rewrite_absolute_reads {
13257 return None;
13263 }
13264
13265 let lower_new_origin_row =
13269 checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
13270 let lower_new_origin_col =
13271 checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
13272 let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
13273 let lower_new_result_region = Region::from_domain(&lower_new_domain);
13274 let lower_new_read_summary = match lower_read_summary.as_ref() {
13275 Some(summary) => Some(shifted_read_summary(
13276 summary,
13277 lower_new_result_region,
13278 op,
13279 row_delta,
13280 col_delta,
13281 origin_row_delta,
13282 origin_col_delta,
13283 )?),
13284 None => None,
13285 };
13286
13287 let mut lower_overlay_refs = Vec::new();
13292 for overlay_ref in authority
13293 .plane
13294 .formula_overlay
13295 .refs_for_source_span(span_ref)
13296 {
13297 let entry = authority.plane.formula_overlay.get(overlay_ref)?;
13298 match op.classify_region(Region::from_domain(&entry.domain)) {
13299 AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
13300 AxisShiftCase::EntirelyAboveShift { .. } => {
13301 lower_overlay_refs.push(overlay_ref);
13302 }
13303 AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
13304 return None;
13305 }
13306 }
13307 }
13308
13309 let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
13310 !binding_set.value_ref_slots.is_empty()
13311 && (origin_row_delta != 0 || origin_col_delta != 0)
13312 });
13313
13314 Some(SplitPlan {
13315 span_ref,
13316 sheet_id: span.sheet_id,
13317 template_id: span.template_id,
13318 binding_set_id: span.binding_set_id,
13319 is_constant_result: span.is_constant_result,
13320 upper_domain,
13321 upper_read_summary,
13322 lower_new_origin_row,
13323 lower_new_origin_col,
13324 lower_new_domain,
13325 lower_new_read_summary,
13326 lower_force_binding_residual_axes,
13327 lower_overlay_refs,
13328 upper_dirty_region: None,
13329 lower_dirty_region: None,
13330 })
13331 }
13332
13333 fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
13339 let origin0 = origin.checked_sub(1)?;
13340 let end = start.saturating_add(count);
13341 if origin0 < start {
13342 Some(origin)
13343 } else if origin0 >= end {
13344 Some(origin - count)
13345 } else {
13346 None
13347 }
13348 }
13349
13350 fn origin_through_delete(
13357 origin_row: u32,
13358 origin_col: u32,
13359 op: StructuralOp,
13360 ) -> Option<(u32, u32)> {
13361 match op {
13362 StructuralOp::DeleteRows { start, count, .. } => Some((
13363 origin_coord_through_delete(origin_row, start, count)?,
13364 origin_col,
13365 )),
13366 StructuralOp::DeleteColumns { start, count, .. } => Some((
13367 origin_row,
13368 origin_coord_through_delete(origin_col, start, count)?,
13369 )),
13370 StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
13371 }
13372 }
13373
13374 let mut shift_plans = Vec::new();
13375 let mut split_plans = Vec::new();
13376 let mut remove_refs = Vec::new();
13377 let mut demote_refs = Vec::new();
13378 for span_ref in span_refs {
13379 let authority = self.graph.formula_authority();
13380 let Some(span) = authority.plane.spans.get(span_ref) else {
13381 continue;
13382 };
13383 let read_summary = span
13384 .read_summary_id
13385 .and_then(|id| authority.plane.span_read_summaries.get(id));
13386 let Some(op) = op else {
13387 let result_region_affected =
13392 Self::span_result_region_intersects_affected(span, &affected_region);
13393 let read_region_affected = Self::span_any_read_region_intersects_affected(
13394 &authority.plane,
13395 span,
13396 &affected_region,
13397 );
13398 if result_region_affected || read_region_affected {
13399 demote_refs.push(span_ref);
13400 }
13401 continue;
13402 };
13403 match classify_span_for_op(span, read_summary, op) {
13404 SpanShiftPlan::NoOp => {}
13405 SpanShiftPlan::Remove => {
13406 remove_refs.push(span_ref);
13407 }
13408 SpanShiftPlan::Demote {
13409 reason:
13410 crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
13411 } => {
13412 let binding_compaction_safe = span
13413 .binding_set_id
13414 .and_then(|id| authority.plane.binding_sets.get(id))
13415 .is_none_or(|binding_set| binding_set.is_single_literal_binding());
13416 let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
13421 !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
13422 summary, op,
13423 )
13424 });
13425 if binding_compaction_safe
13426 && absolute_reads_compaction_safe
13427 && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
13428 {
13429 let new_result_region = Region::from_domain(&new_domain);
13430 let new_read_summary = if let Some(summary) = read_summary {
13431 compact_read_summary_through_delete(summary, new_result_region, op)
13432 } else {
13433 None
13434 };
13435 let Some(template) = authority.plane.templates.get(span.template_id) else {
13436 return Err(ExcelError::new(ExcelErrorKind::Ref)
13437 .with_message(
13438 "FormulaPlane delete compaction found a span with a missing template",
13439 )
13440 .into());
13441 };
13442 let carried_origin =
13443 origin_through_delete(template.origin_row, template.origin_col, op);
13444 if (read_summary.is_none() || new_read_summary.is_some())
13445 && let Some((new_origin_row, new_origin_col)) = carried_origin
13446 {
13447 let force_binding_residual_axes = span
13448 .binding_set_id
13449 .and_then(|id| authority.plane.binding_sets.get(id))
13450 .is_some_and(|binding_set| {
13451 !binding_set.value_ref_slots.is_empty()
13452 && (new_origin_row != template.origin_row
13453 || new_origin_col != template.origin_col)
13454 });
13455 let dirty_region =
13456 Self::structural_dirty_region_for_domain(&new_domain, op);
13457 shift_plans.push(ShiftPlan {
13458 span_ref,
13459 template_id: span.template_id,
13460 new_origin_row,
13461 new_origin_col,
13462 new_domain,
13463 new_read_summary,
13464 binding_set_id: span.binding_set_id,
13465 force_binding_residual_axes,
13466 rewrite: None,
13467 dirty_region,
13468 });
13469 } else {
13470 demote_refs.push(span_ref);
13471 }
13472 } else {
13473 demote_refs.push(span_ref);
13474 }
13475 }
13476 SpanShiftPlan::Demote { .. } => {
13477 demote_refs.push(span_ref);
13478 }
13479 SpanShiftPlan::Split => {
13480 match plan_span_split(authority, span, span_ref, read_summary, op) {
13481 Some(mut plan) => {
13482 plan.upper_dirty_region =
13483 Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
13484 plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
13485 &plan.lower_new_domain,
13486 op,
13487 );
13488 split_plans.push(plan);
13489 }
13490 None => demote_refs.push(span_ref),
13493 }
13494 }
13495 SpanShiftPlan::Shift {
13496 row_delta,
13497 col_delta,
13498 origin_row_delta,
13499 origin_col_delta,
13500 rewrite_absolute_reads,
13501 } => {
13502 let Some(template) = authority.plane.templates.get(span.template_id) else {
13503 return Err(ExcelError::new(ExcelErrorKind::Ref)
13504 .with_message("FormulaPlane shift found a span with a missing template")
13505 .into());
13506 };
13507 let Some(new_origin_row) =
13508 checked_shift_u32(template.origin_row, origin_row_delta)
13509 else {
13510 return Err(ExcelError::new(ExcelErrorKind::Ref)
13511 .with_message("FormulaPlane shift overflowed template origin row")
13512 .into());
13513 };
13514 let Some(new_origin_col) =
13515 checked_shift_u32(template.origin_col, origin_col_delta)
13516 else {
13517 return Err(ExcelError::new(ExcelErrorKind::Ref)
13518 .with_message("FormulaPlane shift overflowed template origin column")
13519 .into());
13520 };
13521 let Some(new_domain) =
13522 span.domain.project_through_axis_shift(row_delta, col_delta)
13523 else {
13524 return Err(ExcelError::new(ExcelErrorKind::Ref)
13525 .with_message("FormulaPlane shift overflowed span domain")
13526 .into());
13527 };
13528 let new_result_region = Region::from_domain(&new_domain);
13529 let force_binding_residual_axes = span
13530 .binding_set_id
13531 .and_then(|id| authority.plane.binding_sets.get(id))
13532 .is_some_and(|binding_set| {
13533 !binding_set.value_ref_slots.is_empty()
13534 && (origin_row_delta != 0 || origin_col_delta != 0)
13535 });
13536 let rewrite_and_summary = if rewrite_absolute_reads {
13537 let frame_sound = read_summary.is_some_and(|summary| {
13558 crate::formula_plane::structural_shift::rewrite_frame_is_sound(
13559 summary,
13560 template.origin_row,
13561 template.origin_col,
13562 op,
13563 )
13564 });
13565 if !frame_sound {
13566 demote_refs.push(span_ref);
13567 continue;
13568 }
13569 let Some(ast) = self
13570 .graph
13571 .data_store()
13572 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
13573 else {
13574 demote_refs.push(span_ref);
13575 continue;
13576 };
13577 let shift_op = shift_operation_for(op);
13578 let adjuster =
13579 crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
13580 let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
13581 span.sheet_id,
13582 self.graph.sheet_reg(),
13583 );
13584 let Some(adjusted_ast) =
13585 adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
13586 else {
13587 demote_refs.push(span_ref);
13592 continue;
13593 };
13594 let canonical = crate::formula_plane::template_canonical::canonicalize_template(
13595 &adjusted_ast,
13596 new_origin_row,
13597 new_origin_col,
13598 );
13599 if !canonical.labels.is_authority_supported() {
13600 demote_refs.push(span_ref);
13601 continue;
13602 }
13603 let literal_slots_compatible = span
13608 .binding_set_id
13609 .and_then(|id| authority.plane.binding_sets.get(id))
13610 .is_none_or(|binding_set| {
13611 binding_set.literal_slots.as_ref()
13612 == canonical.literal_slot_descriptors.as_ref()
13613 });
13614 if !literal_slots_compatible {
13615 demote_refs.push(span_ref);
13616 continue;
13617 }
13618 let value_ref_slots = Arc::from(
13626 crate::formula_plane::placement::value_ref_slot_descriptors(
13627 &canonical.expr,
13628 )
13629 .into_boxed_slice(),
13630 );
13631 let dependency_summary =
13642 crate::formula_plane::dependency_summary::summarize_canonical_template(
13643 &canonical,
13644 );
13645 let rewritten_result_region =
13646 ResultRegion::scalar_cells(new_domain.clone());
13647 let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
13648 span.sheet_id,
13649 &rewritten_result_region,
13650 &dependency_summary,
13651 self.graph.sheet_reg(),
13652 ) else {
13653 demote_refs.push(span_ref);
13654 continue;
13655 };
13656 let formula_text = Some(Arc::<str>::from(
13657 formualizer_parse::pretty::canonical_formula(&adjusted_ast),
13658 ));
13659 Some((
13660 SpanTemplateRewrite {
13661 exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
13662 parameterized_canonical_key: Arc::<str>::from(
13663 canonical.parameterized_key.payload(),
13664 ),
13665 formula_text,
13666 canonical_expr: canonical.expr,
13667 value_ref_slots,
13668 adjusted_ast,
13669 },
13670 rederived_summary,
13671 ))
13672 } else {
13673 None
13674 };
13675 let (rewrite, new_read_summary) = match rewrite_and_summary {
13676 Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
13677 None => {
13678 let new_read_summary = if let Some(summary) = read_summary {
13679 Some(
13680 shifted_read_summary(
13681 summary,
13682 new_result_region,
13683 op,
13684 row_delta,
13685 col_delta,
13686 origin_row_delta,
13687 origin_col_delta,
13688 )
13689 .ok_or_else(|| {
13690 ExcelError::new(ExcelErrorKind::Ref).with_message(
13691 "FormulaPlane shift could not project read summary",
13692 )
13693 })?,
13694 )
13695 } else {
13696 None
13697 };
13698 (None, new_read_summary)
13699 }
13700 };
13701 let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
13702 shift_plans.push(ShiftPlan {
13703 span_ref,
13704 template_id: span.template_id,
13705 new_origin_row,
13706 new_origin_col,
13707 new_domain,
13708 new_read_summary,
13709 binding_set_id: span.binding_set_id,
13710 force_binding_residual_axes,
13711 rewrite,
13712 dirty_region,
13713 });
13714 }
13715 }
13716 }
13717 let span_geometry_changed =
13718 !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
13719 let mut span_dirty_deltas = Vec::new();
13720 if span_geometry_changed {
13721 let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
13725 for plan in shift_plans {
13726 let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
13727 let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
13728 let template_slot_map =
13729 crate::formula_plane::placement::build_template_slot_map(
13730 ast_id,
13731 self.graph.data_store(),
13732 &rewrite.canonical_expr,
13733 );
13734 (ast_id, template_slot_map)
13735 });
13736 prepared_shift_plans.push((plan, rewrite_prepared));
13737 }
13738 let authority = self.graph.formula_authority_mut();
13739 for span_ref in remove_refs {
13740 authority.plane.remove_overlays_for_source_span(span_ref);
13741 authority.plane.remove_span(span_ref);
13742 }
13743 for (plan, rewrite_prepared) in prepared_shift_plans {
13744 let dirty_region = plan.dirty_region;
13745 let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
13746 let Some((ast_id, template_slot_map)) = rewrite_prepared else {
13747 return Err(ExcelError::new(ExcelErrorKind::Ref)
13748 .with_message("FormulaPlane shift lost a rewritten template AST")
13749 .into());
13750 };
13751 let Some(template_id) = authority.plane.intern_rewritten_template(
13752 plan.template_id,
13753 ast_id,
13754 rewrite.exact_canonical_key,
13755 rewrite.parameterized_canonical_key,
13756 rewrite.formula_text,
13757 plan.new_origin_row,
13758 plan.new_origin_col,
13759 ) else {
13760 return Err(ExcelError::new(ExcelErrorKind::Ref)
13761 .with_message("FormulaPlane shift could not intern rewritten template")
13762 .into());
13763 };
13764 (
13765 template_id,
13766 Some((template_slot_map, rewrite.value_ref_slots)),
13767 )
13768 } else {
13769 let Some(template_id) = authority.plane.intern_shifted_template_origin(
13770 plan.template_id,
13771 plan.new_origin_row,
13772 plan.new_origin_col,
13773 ) else {
13774 return Err(ExcelError::new(ExcelErrorKind::Ref)
13775 .with_message("FormulaPlane shift could not clone template origin")
13776 .into());
13777 };
13778 (template_id, None)
13779 };
13780 if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
13781 && let Some(binding_set_id) = plan.binding_set_id
13782 {
13783 authority.plane.set_binding_template_slots(
13784 binding_set_id,
13785 template_slot_map,
13786 value_ref_slots,
13787 );
13788 }
13789 if let Some(binding_set_id) = plan.binding_set_id {
13790 let Some(template) = authority.plane.templates.get(template_id) else {
13791 return Err(ExcelError::new(ExcelErrorKind::Ref)
13792 .with_message("FormulaPlane shift could not find shifted template")
13793 .into());
13794 };
13795 let (ast_id, origin_row, origin_col) =
13796 (template.ast_id, template.origin_row, template.origin_col);
13797 authority.plane.set_binding_template_anchor(
13798 binding_set_id,
13799 ast_id,
13800 origin_row,
13801 origin_col,
13802 );
13803 }
13804 let read_summary_id = plan
13805 .new_read_summary
13806 .map(|summary| authority.plane.insert_span_read_summary(summary));
13807 let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
13808 if !authority.plane.replace_span_geometry(
13809 plan.span_ref,
13810 template_id,
13811 plan.new_domain,
13812 result_region,
13813 read_summary_id,
13814 ) {
13815 return Err(ExcelError::new(ExcelErrorKind::Ref)
13816 .with_message("FormulaPlane shift could not update span geometry")
13817 .into());
13818 }
13819 if let Some(region) = dirty_region {
13820 span_dirty_deltas.push((plan.span_ref, region));
13821 }
13822 if plan.force_binding_residual_axes
13823 && let Some(binding_set_id) = plan.binding_set_id
13824 {
13825 authority.plane.force_binding_residual_axes(binding_set_id);
13834 }
13835 }
13836 for plan in split_plans {
13837 let upper_dirty_region = plan.upper_dirty_region;
13838 let lower_dirty_region = plan.lower_dirty_region;
13839 let Some(split_relocation) = authority
13840 .plane
13841 .spans
13842 .get(plan.span_ref)
13843 .map(|span| span.ast_relocation)
13844 else {
13845 return Err(ExcelError::new(ExcelErrorKind::Ref)
13846 .with_message("FormulaPlane split lost its anchor AST state")
13847 .into());
13848 };
13849 let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
13852 plan.template_id,
13853 plan.lower_new_origin_row,
13854 plan.lower_new_origin_col,
13855 ) else {
13856 return Err(ExcelError::new(ExcelErrorKind::Ref)
13857 .with_message("FormulaPlane split could not clone template origin")
13858 .into());
13859 };
13860 let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
13861 let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
13862 return Err(ExcelError::new(ExcelErrorKind::Ref)
13863 .with_message(
13864 "FormulaPlane split found a span with a missing binding set",
13865 )
13866 .into());
13867 };
13868 let clone = source.clone();
13872 let new_binding_set_id = authority.plane.insert_binding_set(clone);
13873 let Some(template) = authority.plane.templates.get(lower_template_id) else {
13874 return Err(ExcelError::new(ExcelErrorKind::Ref)
13875 .with_message("FormulaPlane split could not find shifted template")
13876 .into());
13877 };
13878 let (ast_id, origin_row, origin_col) =
13879 (template.ast_id, template.origin_row, template.origin_col);
13880 authority.plane.set_binding_template_anchor(
13881 new_binding_set_id,
13882 ast_id,
13883 origin_row,
13884 origin_col,
13885 );
13886 Some(new_binding_set_id)
13887 } else {
13888 None
13889 };
13890 let lower_read_summary_id = plan
13891 .lower_new_read_summary
13892 .map(|summary| authority.plane.insert_span_read_summary(summary));
13893 let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
13894 let lower_ref = authority.plane.insert_span_with_ast_relocation(
13895 crate::formula_plane::runtime::NewFormulaSpan {
13896 sheet_id: plan.sheet_id,
13897 template_id: lower_template_id,
13898 domain: plan.lower_new_domain,
13899 result_region: lower_result_region,
13900 intrinsic_mask_id: None,
13901 read_summary_id: lower_read_summary_id,
13902 binding_set_id: lower_binding_set_id,
13903 is_constant_result: plan.is_constant_result,
13904 },
13905 crate::formula_plane::runtime::SpanAstRelocation {
13906 ast_id: split_relocation.ast_id,
13907 anchor_row: plan.lower_new_origin_row,
13908 anchor_col: plan.lower_new_origin_col,
13909 },
13910 );
13911 if let Some(region) = lower_dirty_region {
13912 span_dirty_deltas.push((lower_ref, region));
13913 }
13914 if let Some(binding_set_id) = lower_binding_set_id {
13915 authority
13916 .plane
13917 .set_binding_span_ref(binding_set_id, lower_ref);
13918 if plan.lower_force_binding_residual_axes {
13919 authority.plane.force_binding_residual_axes(binding_set_id);
13923 }
13924 }
13925 let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
13929 if !name_keys.is_empty() {
13930 authority
13931 .plane
13932 .register_span_name_dependents(lower_ref, &name_keys);
13933 }
13934 for overlay_ref in plan.lower_overlay_refs {
13935 if !authority
13936 .plane
13937 .set_overlay_source_span(overlay_ref, Some(lower_ref))
13938 {
13939 return Err(ExcelError::new(ExcelErrorKind::Ref)
13940 .with_message(
13941 "FormulaPlane split could not re-point overlay provenance",
13942 )
13943 .into());
13944 }
13945 }
13946 let upper_read_summary_id = plan
13949 .upper_read_summary
13950 .map(|summary| authority.plane.insert_span_read_summary(summary));
13951 let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
13952 if !authority.plane.replace_span_geometry_with_ast_relocation(
13953 plan.span_ref,
13954 plan.template_id,
13955 split_relocation,
13956 plan.upper_domain,
13957 upper_result_region,
13958 upper_read_summary_id,
13959 ) {
13960 return Err(ExcelError::new(ExcelErrorKind::Ref)
13961 .with_message("FormulaPlane split could not update span geometry")
13962 .into());
13963 }
13964 if let Some(region) = upper_dirty_region {
13965 span_dirty_deltas.push((plan.span_ref, region));
13966 }
13967 }
13968 authority.rebuild_indexes();
13969 }
13970 for (span_ref, region) in span_dirty_deltas {
13971 self.graph.mark_formula_span_region_dirty(span_ref, region);
13972 }
13973
13974 let mut span_plans = Vec::new();
13975 for span_ref in demote_refs {
13976 let authority = self.graph.formula_authority();
13977 let Some(span) = authority.plane.spans.get(span_ref) else {
13978 continue;
13979 };
13980 let relocation = span.ast_relocation;
13981 let ast = self
13982 .graph
13983 .data_store()
13984 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
13985 .ok_or_else(|| {
13986 ExcelError::new(ExcelErrorKind::Ref).with_message(
13987 "FormulaPlane demotion could not retrieve the span anchor AST",
13988 )
13989 })?;
13990 let placements = span
13991 .domain
13992 .iter()
13993 .map(|placement| (placement.row + 1, placement.col + 1))
13994 .collect();
13995 span_plans.push(SpanPlan {
13996 span_ref,
13997 sheet_id: span.sheet_id,
13998 ast,
13999 origin_row: relocation.anchor_row,
14000 origin_col: relocation.anchor_col,
14001 binding_set_id: span.binding_set_id,
14002 placements,
14003 });
14004 }
14005 if span_plans.is_empty() {
14006 return Ok(());
14007 }
14008
14009 let mut relocated = Vec::new();
14010 let mut placement_cells = Vec::new();
14011 for plan in &span_plans {
14012 for &(row, col) in &plan.placements {
14013 let row_delta = i64::from(row) - i64::from(plan.origin_row);
14014 let col_delta = i64::from(col) - i64::from(plan.origin_col);
14015 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14016 let authority = self.graph.formula_authority();
14017 if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14018 if binding_set.is_single_literal_binding() {
14019 plan.ast.clone()
14020 } else {
14021 let placement = crate::formula_plane::runtime::PlacementCoord::new(
14022 plan.sheet_id,
14023 row.saturating_sub(1),
14024 col.saturating_sub(1),
14025 );
14026 let binding =
14027 authority.plane.spans.get(plan.span_ref).and_then(|span| {
14028 binding_set
14029 .literal_bindings_for_placement(&span.domain, placement)
14030 });
14031 if let Some(binding) = binding {
14032 substitute_literal_slots_for_template_placement(
14033 &plan.ast,
14034 binding.as_ref(),
14035 )
14036 } else {
14037 plan.ast.clone()
14038 }
14039 }
14040 } else {
14041 plan.ast.clone()
14042 }
14043 } else {
14044 plan.ast.clone()
14045 };
14046 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14047 relocated.push((plan.sheet_id, row, col, ast));
14048 placement_cells.push((plan.sheet_id, row, col));
14049 }
14050 }
14051 let planned_by_sheet = {
14052 let mut pipeline = self.ingest_pipeline();
14053 let mut planned_by_sheet: BTreeMap<
14054 SheetId,
14055 Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14056 > = BTreeMap::new();
14057 for (formula_sheet_id, row, col, ast) in relocated {
14058 let placement =
14059 CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14060 let ingested =
14061 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14062 planned_by_sheet.entry(formula_sheet_id).or_default().push((
14063 row,
14064 col,
14065 ingested.ast_id,
14066 ingested.dep_plan,
14067 ));
14068 }
14069 planned_by_sheet
14070 };
14071 {
14072 let authority = self.graph.formula_authority_mut();
14073 for plan in &span_plans {
14074 authority
14075 .plane
14076 .remove_overlays_for_source_span(plan.span_ref);
14077 authority.plane.remove_span(plan.span_ref);
14078 }
14079 authority.rebuild_indexes();
14080 }
14081 if clear_computed_overlays {
14082 self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14089 }
14090 for (formula_sheet_id, planned) in planned_by_sheet {
14091 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14092 self.graph
14093 .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14094 }
14095 if !clear_computed_overlays {
14096 for (formula_sheet_id, row, col) in &placement_cells {
14097 let row0 = row.saturating_sub(1);
14098 let col0 = col.saturating_sub(1);
14099 if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14100 continue;
14101 }
14102 let cell =
14103 CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14104 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14105 self.graph.set_dirty(vertex_id, false);
14106 }
14107 }
14108 }
14109 Ok(())
14110 }
14111
14112 pub(crate) fn prepare_formula_span_demotion(
14116 &mut self,
14117 span_refs: &[FormulaSpanRef],
14118 ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14119 #[cfg(test)]
14120 let fault = self
14121 .formula_span_demotion_fault_for_test
14122 .take()
14123 .unwrap_or_default();
14124 #[cfg(not(test))]
14125 let fault = FormulaSpanDemotionFault::None;
14126
14127 if fault == FormulaSpanDemotionFault::AstPreparation {
14128 return Err(FormulaSpanDemotionError::Injected(fault));
14129 }
14130
14131 fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14132 fn visit(
14133 ast: &ASTNode,
14134 binding: &[LiteralValue],
14135 next: &mut usize,
14136 in_array: bool,
14137 ) -> ASTNode {
14138 let node_type = match &ast.node_type {
14139 ASTNodeType::Literal(_) if !in_array => {
14140 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14141 *next = next.saturating_add(1);
14142 ASTNodeType::Literal(value)
14143 }
14144 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14145 ASTNodeType::Omitted => ASTNodeType::Omitted,
14146 ASTNodeType::Reference {
14147 original,
14148 reference,
14149 } => ASTNodeType::Reference {
14150 original: original.clone(),
14151 reference: reference.clone(),
14152 },
14153 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14154 op: op.clone(),
14155 expr: Box::new(visit(expr, binding, next, in_array)),
14156 },
14157 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14158 op: op.clone(),
14159 left: Box::new(visit(left, binding, next, in_array)),
14160 right: Box::new(visit(right, binding, next, in_array)),
14161 },
14162 ASTNodeType::Function { name, args } => ASTNodeType::Function {
14163 name: name.clone(),
14164 args: args
14165 .iter()
14166 .map(|arg| visit(arg, binding, next, in_array))
14167 .collect(),
14168 },
14169 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14170 callee: Box::new(visit(callee, binding, next, in_array)),
14171 args: args
14172 .iter()
14173 .map(|arg| visit(arg, binding, next, in_array))
14174 .collect(),
14175 },
14176 ASTNodeType::Array(rows) => ASTNodeType::Array(
14177 rows.iter()
14178 .map(|row| {
14179 row.iter()
14180 .map(|cell| visit(cell, binding, next, true))
14181 .collect()
14182 })
14183 .collect(),
14184 ),
14185 };
14186 ASTNode::new(node_type, ast.source_token.clone())
14187 }
14188 let mut next = 0;
14189 visit(ast, binding, &mut next, false)
14190 }
14191
14192 struct SpanMaterialization {
14193 span_ref: FormulaSpanRef,
14194 sheet_id: SheetId,
14195 ast: ASTNode,
14196 origin_row: u32,
14197 origin_col: u32,
14198 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
14199 domain: PlacementDomain,
14200 }
14201
14202 let request_id = self
14203 .active_evaluation_resource_request
14204 .as_ref()
14205 .map(|request| request.request_id);
14206 let resource_error =
14207 |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
14208 FormulaSpanDemotionError::Resource(
14209 crate::engine::ResourceLedgerError::Exhausted(
14210 formualizer_common::ResourceExhaustionDetail {
14211 reason,
14212 limit,
14213 observed,
14214 request_id,
14215 },
14216 )
14217 .into_excel_error(),
14218 )
14219 };
14220 let authority = self.graph.formula_authority();
14221 let expected_plane_epoch = authority.plane.epoch().0;
14222 let expected_indexes_epoch = authority.indexes_epoch();
14223 let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
14224 let mut unique = Vec::new();
14225 unique.try_reserve_exact(span_refs.len()).map_err(|_| {
14226 resource_error(
14227 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14228 u64::MAX,
14229 span_refs.len() as u64,
14230 )
14231 })?;
14232 let mut plans = Vec::new();
14233 plans.try_reserve_exact(span_refs.len()).map_err(|_| {
14234 resource_error(
14235 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14236 u64::MAX,
14237 span_refs.len() as u64,
14238 )
14239 })?;
14240 let mut placement_count = 0usize;
14241 let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
14242 for &span_ref in span_refs {
14243 if unique.contains(&span_ref) {
14244 continue;
14245 }
14246 unique.push(span_ref);
14247 let span = authority
14248 .plane
14249 .spans
14250 .get(span_ref)
14251 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14252 let relocation = span.ast_relocation;
14253 let ast = self
14254 .graph
14255 .data_store()
14256 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14257 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14258 let count = usize::try_from(span.domain.cell_count())
14259 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
14260 placement_count = placement_count
14261 .checked_add(count)
14262 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14263 if u64::try_from(placement_count)
14264 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
14265 > self.workbook_load_limits.max_formula_plane_fallback_cells
14266 {
14267 return Err(resource_error(
14268 formualizer_common::ResourceExhaustionReason::MaterializationCells,
14269 self.workbook_load_limits.max_formula_plane_fallback_cells,
14270 placement_count as u64,
14271 ));
14272 }
14273 let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
14274 *sheet_count = sheet_count
14275 .checked_add(span.domain.cell_count())
14276 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14277 if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
14278 return Err(resource_error(
14279 formualizer_common::ResourceExhaustionReason::Admission,
14280 self.workbook_load_limits.max_sheet_logical_cells,
14281 *sheet_count,
14282 ));
14283 }
14284 let domain_in_bounds = match &span.domain {
14285 PlacementDomain::RowRun { row_end, col, .. } => {
14286 *row_end < self.workbook_load_limits.max_sheet_rows
14287 && *col < self.workbook_load_limits.max_sheet_cols
14288 }
14289 PlacementDomain::ColRun { row, col_end, .. } => {
14290 *row < self.workbook_load_limits.max_sheet_rows
14291 && *col_end < self.workbook_load_limits.max_sheet_cols
14292 }
14293 PlacementDomain::Rect {
14294 row_end, col_end, ..
14295 } => {
14296 *row_end < self.workbook_load_limits.max_sheet_rows
14297 && *col_end < self.workbook_load_limits.max_sheet_cols
14298 }
14299 };
14300 if !domain_in_bounds {
14301 return Err(resource_error(
14302 formualizer_common::ResourceExhaustionReason::Admission,
14303 u64::from(self.workbook_load_limits.max_sheet_rows)
14304 .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
14305 span.domain.cell_count(),
14306 ));
14307 }
14308 plans.push(SpanMaterialization {
14309 span_ref,
14310 sheet_id: span.sheet_id,
14311 ast,
14312 origin_row: relocation.anchor_row,
14313 origin_col: relocation.anchor_col,
14314 binding_set_id: span.binding_set_id,
14315 domain: span.domain.clone(),
14316 });
14317 }
14318 let mut relocated = Vec::new();
14319 relocated.try_reserve_exact(placement_count).map_err(|_| {
14320 resource_error(
14321 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14322 u64::MAX,
14323 placement_count as u64,
14324 )
14325 })?;
14326 for plan in &plans {
14327 for coord in plan.domain.iter() {
14328 let row = coord.row.saturating_add(1);
14329 let col = coord.col.saturating_add(1);
14330 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14331 let authority = self.graph.formula_authority();
14332 let binding_set = authority
14333 .plane
14334 .binding_sets
14335 .get(binding_set_id)
14336 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14337 if binding_set.is_single_literal_binding() {
14338 plan.ast.clone()
14339 } else {
14340 let placement = PlacementCoord::new(
14341 plan.sheet_id,
14342 row.saturating_sub(1),
14343 col.saturating_sub(1),
14344 );
14345 let span = authority
14346 .plane
14347 .spans
14348 .get(plan.span_ref)
14349 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14350 let binding = binding_set
14351 .literal_bindings_for_placement(&span.domain, placement)
14352 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14353 substitute_literal_slots(&plan.ast, binding.as_ref())
14354 }
14355 } else {
14356 plan.ast.clone()
14357 };
14358 let row_delta = i64::from(row) - i64::from(plan.origin_row);
14359 let col_delta = i64::from(col) - i64::from(plan.origin_col);
14360 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
14361 .map_err(FormulaSpanDemotionError::AstPreparation)?;
14362 relocated.push((plan.sheet_id, row, col, ast));
14363 }
14364 }
14365
14366 let mut analyzed = Vec::new();
14367 analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
14368 resource_error(
14369 formualizer_common::ResourceExhaustionReason::ScratchMemory,
14370 u64::MAX,
14371 relocated.len() as u64,
14372 )
14373 })?;
14374 {
14375 let mut pipeline = self.ingest_pipeline();
14376 for (sheet_id, row, col, ast) in relocated {
14377 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
14378 let ingested = pipeline
14379 .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
14380 .map_err(FormulaSpanDemotionError::AstPreparation)?;
14381 analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
14382 }
14383 }
14384
14385 if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
14386 return Err(FormulaSpanDemotionError::Injected(fault));
14387 }
14388 let legacy_graph = self
14389 .graph
14390 .prepare_legacy_graph_plan_multi_sheet(analyzed)
14391 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14392 self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
14393 .map_err(FormulaSpanDemotionError::Resource)?;
14394
14395 Ok(PreparedFormulaSpanDemotion {
14396 span_refs: unique,
14397 expected_plane_epoch,
14398 expected_indexes_epoch,
14399 expected_indexed_plane_epoch,
14400 placement_count,
14401 legacy_graph,
14402 fault,
14403 })
14404 }
14405
14406 pub(crate) fn validate_prepared_formula_span_demotion(
14407 &self,
14408 prepared: &PreparedFormulaSpanDemotion,
14409 ) -> Result<(), FormulaSpanDemotionError> {
14410 let authority = self.graph.formula_authority();
14411 if authority.plane.epoch().0 != prepared.expected_plane_epoch
14412 || authority.indexes_epoch() != prepared.expected_indexes_epoch
14413 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14414 || prepared
14415 .span_refs
14416 .iter()
14417 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14418 {
14419 return Err(FormulaSpanDemotionError::StaleAuthority);
14420 }
14421 self.graph
14422 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14423 .map_err(FormulaSpanDemotionError::LegacyGraph)
14424 }
14425
14426 pub(crate) fn commit_prepared_formula_span_demotion(
14432 &mut self,
14433 prepared: PreparedFormulaSpanDemotion,
14434 ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
14435 self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
14436 .map_err(FormulaSpanDemotionError::Resource)?;
14437 if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
14438 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14439 }
14440 self.graph
14441 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14442 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14443 if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
14444 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14445 }
14446 {
14447 let authority = self.graph.formula_authority();
14448 if authority.plane.epoch().0 != prepared.expected_plane_epoch
14449 || authority.indexes_epoch() != prepared.expected_indexes_epoch
14450 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14451 || prepared
14452 .span_refs
14453 .iter()
14454 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14455 {
14456 return Err(FormulaSpanDemotionError::StaleAuthority);
14457 }
14458 }
14459 if matches!(
14460 prepared.fault,
14461 FormulaSpanDemotionFault::AllocationReservation
14462 | FormulaSpanDemotionFault::BeforeFirstMutation
14463 ) {
14464 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14465 }
14466
14467 let PreparedFormulaSpanDemotion {
14468 span_refs,
14469 placement_count,
14470 legacy_graph,
14471 ..
14472 } = prepared;
14473 let _ = self
14474 .graph
14475 .apply_prevalidated_legacy_graph_plan(legacy_graph);
14476 let authority = self.graph.formula_authority_mut();
14477 for span_ref in &span_refs {
14478 authority.plane.remove_overlays_for_source_span(*span_ref);
14479 let _ = authority.plane.remove_span(*span_ref);
14480 }
14481 let _ = authority.rebuild_indexes();
14482 self.mark_topology_edited();
14483 Ok(FormulaSpanDemotionReport {
14484 spans_demoted: span_refs.len(),
14485 placements_materialized: placement_count,
14486 })
14487 }
14488
14489 #[cfg(test)]
14490 pub(crate) fn set_formula_span_demotion_fault_for_test(
14491 &mut self,
14492 fault: FormulaSpanDemotionFault,
14493 ) {
14494 self.formula_span_demotion_fault_for_test = Some(fault);
14495 }
14496
14497 #[cfg(test)]
14498 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
14499 self.force_non_cycle_schedule_fallback_for_test = true;
14500 }
14501
14502 #[cfg(test)]
14503 pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
14504 self.mixed_topology_index_builds_for_test
14505 }
14506
14507 fn collect_cyclic_span_refs(
14512 &self,
14513 schedule: &MixedSchedule,
14514 span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
14515 ) -> Vec<FormulaSpanRef> {
14516 let mut refs = Vec::new();
14517 for fallback in &schedule.fallbacks {
14518 if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
14519 continue;
14520 }
14521 if let FormulaProducerId::Span(span_id) = fallback.producer
14522 && let Some(span_ref) = span_refs_by_id.get(&span_id)
14523 && !refs.contains(span_ref)
14524 {
14525 refs.push(*span_ref);
14526 }
14527 }
14528 refs
14529 }
14530
14531 fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
14539 let materialization_started = crate::instant::FzInstant::now();
14540 let prepared = self
14541 .prepare_formula_span_demotion(span_refs)
14542 .map_err(|error| {
14543 if let FormulaSpanDemotionError::Resource(error) = error {
14544 error
14545 } else {
14546 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14547 "FormulaPlane cycle-member span demotion preparation failed: {error}"
14548 ))
14549 }
14550 })?;
14551 let report = self
14552 .commit_prepared_formula_span_demotion(prepared)
14553 .map_err(|error| {
14554 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14555 "FormulaPlane cycle-member span demotion commit failed: {error}"
14556 ))
14557 })?;
14558 self.observe_materialization(
14559 report.placements_materialized,
14560 true,
14561 materialization_started.elapsed(),
14562 );
14563 self.formula_plane_cycle_member_span_demotions = self
14564 .formula_plane_cycle_member_span_demotions
14565 .saturating_add(report.spans_demoted as u64);
14566 Self::record_shadow_fallback_reason(
14567 &mut self.formula_ingest_report_total,
14568 PlacementFallbackReason::CycleMember,
14569 report.spans_demoted as u64,
14570 );
14571 Ok(())
14572 }
14573
14574 fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
14589 let dirty = self.graph.get_evaluation_vertices();
14590 if dirty.is_empty() {
14591 return Ok(0);
14592 }
14593 let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
14594 let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
14595 let mut cycle_errors = 0usize;
14596 let mut stamped_vertices: Vec<VertexId> = Vec::new();
14597 for &unit in &schedule.units {
14598 let ScheduleUnit::Cycle(i) = unit else {
14599 continue;
14600 };
14601 let members = schedule.unit_cycle(i);
14602 let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
14603 if stamped > 0 {
14604 cycle_errors += 1;
14605 }
14606 stamped_vertices.extend(members.iter().copied());
14607 }
14608 if !stamped_vertices.is_empty() {
14613 self.resource_checkpoint(0)?;
14614 self.graph.clear_dirty_flags(&stamped_vertices);
14615 }
14616 Ok(cycle_errors)
14617 }
14618
14619 fn materialize_deferred_sheet_before_structural_edit(
14620 &mut self,
14621 sheet: &str,
14622 ) -> Result<(), crate::engine::EditorError> {
14623 if self.staged_formulas.contains_key(sheet) {
14624 self.build_graph_for_sheets([sheet])?;
14625 }
14626 Ok(())
14627 }
14628
14629 pub fn insert_rows(
14631 &mut self,
14632 sheet: &str,
14633 before: u32,
14634 count: u32,
14635 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14636 {
14637 if count == 0 {
14638 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14639 }
14640 self.observe_function_semantic_epoch()
14641 .map_err(crate::engine::EditorError::Excel)?;
14642 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14643 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14644 let sheet_id = self.ensure_known_sheet_id(sheet)?;
14645 let before0 = before.saturating_sub(1);
14646 let affected_region = Self::structural_row_region(sheet_id, before0);
14647 let op = StructuralOp::InsertRows {
14648 sheet_id,
14649 before: before0,
14650 count,
14651 };
14652 self.demote_spans_for_structural_op(op, affected_region)?;
14653 let summary = {
14654 let mut editor = VertexEditor::new(&mut self.graph);
14655 editor.insert_rows(sheet_id, before0, count)?
14656 };
14657 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14658 let before0 = before0 as usize;
14659 asheet.insert_rows(before0, count as usize);
14660 }
14661 self.mark_moved_formula_vertices_dirty(&summary);
14662 self.clear_computed_overlay_after_row(sheet, before0 as usize);
14663 self.shift_row_visibility_insert(sheet_id, before0, count);
14664 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14665 self.mark_topology_edited();
14666 Ok(summary)
14667 }
14668
14669 pub fn delete_rows(
14671 &mut self,
14672 sheet: &str,
14673 start: u32,
14674 count: u32,
14675 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14676 {
14677 if count == 0 {
14678 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14679 }
14680 self.observe_function_semantic_epoch()
14681 .map_err(crate::engine::EditorError::Excel)?;
14682 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14683 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14684 let sheet_id = self.ensure_known_sheet_id(sheet)?;
14685 let start0 = start.saturating_sub(1);
14686 let affected_region = Self::structural_row_region(sheet_id, start0);
14687 let op = StructuralOp::DeleteRows {
14688 sheet_id,
14689 start: start0,
14690 count,
14691 };
14692 self.demote_spans_for_structural_op(op, affected_region)?;
14693 let summary = {
14694 let mut editor = VertexEditor::new(&mut self.graph);
14695 editor.delete_rows(sheet_id, start0, count)?
14696 };
14697 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14698 let start0 = start0 as usize;
14699 asheet.delete_rows(start0, count as usize);
14700 }
14701 self.mark_moved_formula_vertices_dirty(&summary);
14702 self.clear_computed_overlay_after_row(sheet, start0 as usize);
14703 self.shift_row_visibility_delete(sheet_id, start0, count);
14704 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14705 self.mark_topology_edited();
14706 Ok(summary)
14707 }
14708
14709 pub fn insert_columns(
14711 &mut self,
14712 sheet: &str,
14713 before: u32,
14714 count: u32,
14715 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14716 {
14717 if count == 0 {
14718 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14719 }
14720 self.observe_function_semantic_epoch()
14721 .map_err(crate::engine::EditorError::Excel)?;
14722 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14723 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14724 let sheet_id = self.graph.sheet_id(sheet).ok_or(
14725 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14726 name: sheet.to_string(),
14727 reason: "Unknown sheet".to_string(),
14728 },
14729 )?;
14730 let before0 = before.saturating_sub(1);
14731 let affected_region = Self::structural_col_region(sheet_id, before0);
14732 let op = StructuralOp::InsertColumns {
14733 sheet_id,
14734 before: before0,
14735 count,
14736 };
14737 self.demote_spans_for_structural_op(op, affected_region)?;
14738 let summary = {
14739 let mut editor = VertexEditor::new(&mut self.graph);
14740 editor.insert_columns(sheet_id, before0, count)?
14741 };
14742 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14743 let before0 = before0 as usize;
14744 asheet.insert_columns(before0, count as usize);
14745 }
14746 self.mark_moved_formula_vertices_dirty(&summary);
14747 self.clear_computed_overlay_after_col(sheet, before0 as usize);
14748 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14749 self.mark_topology_edited();
14750 Ok(summary)
14751 }
14752
14753 pub fn delete_columns(
14755 &mut self,
14756 sheet: &str,
14757 start: u32,
14758 count: u32,
14759 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14760 {
14761 if count == 0 {
14762 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14763 }
14764 self.observe_function_semantic_epoch()
14765 .map_err(crate::engine::EditorError::Excel)?;
14766 use crate::engine::graph::editor::vertex_editor::VertexEditor;
14767 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14768 let sheet_id = self.graph.sheet_id(sheet).ok_or(
14769 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14770 name: sheet.to_string(),
14771 reason: "Unknown sheet".to_string(),
14772 },
14773 )?;
14774 let start0 = start.saturating_sub(1);
14775 let affected_region = Self::structural_col_region(sheet_id, start0);
14776 let op = StructuralOp::DeleteColumns {
14777 sheet_id,
14778 start: start0,
14779 count,
14780 };
14781 self.demote_spans_for_structural_op(op, affected_region)?;
14782 let summary = {
14783 let mut editor = VertexEditor::new(&mut self.graph);
14784 editor.delete_columns(sheet_id, start0, count)?
14785 };
14786 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14787 let start0 = start0 as usize;
14788 asheet.delete_columns(start0, count as usize);
14789 }
14790 self.mark_moved_formula_vertices_dirty(&summary);
14791 self.clear_computed_overlay_after_col(sheet, start0 as usize);
14792 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14793 self.mark_topology_edited();
14794 Ok(summary)
14795 }
14796 fn arrow_used_row_bounds(
14798 &self,
14799 sheet: &str,
14800 start_col: u32,
14801 end_col: u32,
14802 ) -> Option<(u32, u32)> {
14803 let a = self.sheet_store().sheet(sheet)?;
14804 if a.columns.is_empty() {
14805 return None;
14806 }
14807 let sc0 = start_col.saturating_sub(1) as usize;
14808 let ec0 = end_col.saturating_sub(1) as usize;
14809 let col_hi = a.columns.len().saturating_sub(1);
14810 if sc0 > col_hi {
14811 return None;
14812 }
14813 let ec0 = ec0.min(col_hi);
14814 let snap = self.data_snapshot_id();
14816 let mut min_r0: Option<usize> = None;
14817 for ci in sc0..=ec0 {
14818 let sheet_id = self.graph.sheet_id(sheet)?;
14819 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
14820 g.as_ref()
14821 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14822 }) {
14823 let mv = mv as usize;
14824 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
14825 continue;
14826 }
14827 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14829 if let Ok(mut g) = self.row_bounds_cache.write() {
14830 g.get_or_insert_with(|| RowBoundsCache::new(snap))
14831 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
14832 }
14833 if let Some(m) = min_c {
14834 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
14835 }
14836 }
14837 min_r0?;
14838 let mut max_r0: Option<usize> = None;
14839 for ci in sc0..=ec0 {
14840 let sheet_id = self.graph.sheet_id(sheet)?;
14841 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
14842 g.as_ref()
14843 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14844 }) {
14845 let mv = mv as usize;
14846 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
14847 continue;
14848 }
14849 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14850 if let Ok(mut g) = self.row_bounds_cache.write() {
14851 g.get_or_insert_with(|| RowBoundsCache::new(snap))
14852 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
14853 }
14854 if let Some(m) = max_c {
14855 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
14856 }
14857 }
14858 match (min_r0, max_r0) {
14859 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
14860 _ => None,
14861 }
14862 }
14863
14864 fn scan_column_used_bounds(
14865 a: &crate::arrow_store::ArrowSheet,
14866 ci: usize,
14867 ) -> (Option<u32>, Option<u32>) {
14868 let col = &a.columns[ci];
14869
14870 let mut min_r0: Option<u32> = None;
14872 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
14873 let tags = chunk.type_tag.values();
14874 for (off, &t) in tags.iter().enumerate() {
14875 let overlay_non_empty = chunk
14876 .overlay
14877 .get(off)
14878 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14879 .unwrap_or(false)
14880 || chunk
14881 .computed_overlay
14882 .get(off)
14883 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14884 .unwrap_or(false);
14885 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14886 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14887 break;
14888 };
14889 let row0 = chunk_start + off;
14890 min_r0 = Some(row0 as u32);
14891 break;
14892 }
14893 }
14894 if min_r0.is_some() {
14895 break;
14896 }
14897 }
14898 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
14899 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14900 sparse_idxs.sort_unstable();
14901 for chunk_idx in sparse_idxs {
14902 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14903 continue;
14904 };
14905 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14906 continue;
14907 };
14908 let tags = chunk.type_tag.values();
14909 for (off, &t) in tags.iter().enumerate() {
14910 let overlay_non_empty = chunk
14911 .overlay
14912 .get(off)
14913 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14914 .unwrap_or(false)
14915 || chunk
14916 .computed_overlay
14917 .get(off)
14918 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14919 .unwrap_or(false);
14920 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14921 let row0 = chunk_start + off;
14922 min_r0 = Some(row0 as u32);
14923 break;
14924 }
14925 }
14926 if min_r0.is_some() {
14927 break;
14928 }
14929 }
14930 }
14931
14932 let mut max_r0: Option<u32> = None;
14934 if !col.sparse_chunks.is_empty() {
14935 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14936 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
14937 for chunk_idx in sparse_idxs {
14938 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14939 continue;
14940 };
14941 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14942 continue;
14943 };
14944 let tags = chunk.type_tag.values();
14945 for (rev_idx, &t) in tags.iter().enumerate().rev() {
14946 let overlay_non_empty = chunk
14947 .overlay
14948 .get(rev_idx)
14949 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14950 .unwrap_or(false)
14951 || chunk
14952 .computed_overlay
14953 .get(rev_idx)
14954 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14955 .unwrap_or(false);
14956 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14957 let row0 = chunk_start + rev_idx;
14958 max_r0 = Some(row0 as u32);
14959 break;
14960 }
14961 }
14962 if max_r0.is_some() {
14963 break;
14964 }
14965 }
14966 }
14967 if max_r0.is_none() {
14968 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
14969 let tags = chunk.type_tag.values();
14970 for (rev_idx, &t) in tags.iter().enumerate().rev() {
14971 let overlay_non_empty = chunk
14972 .overlay
14973 .get(rev_idx)
14974 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14975 .unwrap_or(false)
14976 || chunk
14977 .computed_overlay
14978 .get(rev_idx)
14979 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14980 .unwrap_or(false);
14981 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14982 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14983 break;
14984 };
14985 let row0 = chunk_start + rev_idx;
14986 max_r0 = Some(row0 as u32);
14987 break;
14988 }
14989 }
14990 if max_r0.is_some() {
14991 break;
14992 }
14993 }
14994 }
14995
14996 (min_r0, max_r0)
14997 }
14998
14999 fn arrow_used_col_bounds(
15001 &self,
15002 sheet: &str,
15003 start_row: u32,
15004 end_row: u32,
15005 ) -> Option<(u32, u32)> {
15006 let a = self.sheet_store().sheet(sheet)?;
15007 if a.columns.is_empty() {
15008 return None;
15009 }
15010 let sr0 = start_row.saturating_sub(1) as usize;
15011 let er0 = end_row.saturating_sub(1) as usize;
15012 if sr0 > er0 {
15013 return None;
15014 }
15015 let mut min_c0: Option<usize> = None;
15018 let mut max_c0: Option<usize> = None;
15019 for (ci, col) in a.columns.iter().enumerate() {
15021 let mut any_in_range = false;
15022
15023 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15024 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15025 return false;
15026 };
15027 let chunk_len = chunk.type_tag.len();
15028 if chunk_len == 0 {
15029 return false;
15030 }
15031 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15032 if sr0 > chunk_end || er0 < chunk_start {
15034 return false;
15035 }
15036 let start_off = sr0.max(chunk_start) - chunk_start;
15037 let end_off = er0.min(chunk_end) - chunk_start;
15038 let tags = chunk.type_tag.values();
15039 for off in start_off..=end_off {
15040 let overlay_non_empty = chunk
15041 .overlay
15042 .get(off)
15043 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15044 .unwrap_or(false)
15045 || chunk
15046 .computed_overlay
15047 .get(off)
15048 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15049 .unwrap_or(false);
15050 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15051 return true;
15052 }
15053 }
15054 false
15055 };
15056
15057 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15058 if scan_chunk(chunk_idx, chunk) {
15059 any_in_range = true;
15060 break;
15061 }
15062 }
15063
15064 if !any_in_range && !col.sparse_chunks.is_empty() {
15065 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15066 if scan_chunk(chunk_idx, chunk) {
15067 any_in_range = true;
15068 break;
15069 }
15070 }
15071 }
15072
15073 if any_in_range {
15074 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
15075 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
15076 }
15077 }
15078 match (min_c0, max_c0) {
15079 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15080 _ => None,
15081 }
15082 }
15083
15084 fn formula_row_bounds_for_columns(
15085 &self,
15086 sheet: &str,
15087 start_col: u32,
15088 end_col: u32,
15089 ) -> Option<(u32, u32)> {
15090 let sheet_id = self.graph.sheet_id(sheet)?;
15091 let sc0 = start_col.saturating_sub(1);
15092 let ec0 = end_col.saturating_sub(1);
15093 let mut min_r0: Option<u32> = None;
15094 let mut max_r0: Option<u32> = None;
15095
15096 if let Some(index) = self.graph.sheet_index(sheet_id) {
15097 for vid in index.vertices_in_col_range(sc0, ec0) {
15098 if !matches!(
15099 self.graph.get_vertex_kind(vid),
15100 VertexKind::FormulaScalar | VertexKind::FormulaArray
15101 ) {
15102 continue;
15103 }
15104 let row0 = self.graph.vertex_coord(vid).row();
15105 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15106 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15107 }
15108 } else {
15109 for vid in self.graph.vertices_in_sheet(sheet_id) {
15110 if !matches!(
15111 self.graph.get_vertex_kind(vid),
15112 VertexKind::FormulaScalar | VertexKind::FormulaArray
15113 ) {
15114 continue;
15115 }
15116 let coord = self.graph.vertex_coord(vid);
15117 let col0 = coord.col();
15118 if col0 < sc0 || col0 > ec0 {
15119 continue;
15120 }
15121 let row0 = coord.row();
15122 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15123 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15124 }
15125 }
15126
15127 match (min_r0, max_r0) {
15128 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15129 _ => None,
15130 }
15131 }
15132
15133 fn formula_col_bounds_for_rows(
15134 &self,
15135 sheet: &str,
15136 start_row: u32,
15137 end_row: u32,
15138 ) -> Option<(u32, u32)> {
15139 let sheet_id = self.graph.sheet_id(sheet)?;
15140 let sr0 = start_row.saturating_sub(1);
15141 let er0 = end_row.saturating_sub(1);
15142 let mut min_c0: Option<u32> = None;
15143 let mut max_c0: Option<u32> = None;
15144
15145 if let Some(index) = self.graph.sheet_index(sheet_id) {
15146 for vid in index.vertices_in_row_range(sr0, er0) {
15147 if !matches!(
15148 self.graph.get_vertex_kind(vid),
15149 VertexKind::FormulaScalar | VertexKind::FormulaArray
15150 ) {
15151 continue;
15152 }
15153 let col0 = self.graph.vertex_coord(vid).col();
15154 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15155 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15156 }
15157 } else {
15158 for vid in self.graph.vertices_in_sheet(sheet_id) {
15159 if !matches!(
15160 self.graph.get_vertex_kind(vid),
15161 VertexKind::FormulaScalar | VertexKind::FormulaArray
15162 ) {
15163 continue;
15164 }
15165 let coord = self.graph.vertex_coord(vid);
15166 let row0 = coord.row();
15167 if row0 < sr0 || row0 > er0 {
15168 continue;
15169 }
15170 let col0 = coord.col();
15171 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15172 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15173 }
15174 }
15175
15176 match (min_c0, max_c0) {
15177 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15178 _ => None,
15179 }
15180 }
15181
15182 fn union_used_bounds(
15183 first: Option<(u32, u32)>,
15184 second: Option<(u32, u32)>,
15185 ) -> Option<(u32, u32)> {
15186 match (first, second) {
15187 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
15188 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
15189 (None, None) => None,
15190 }
15191 }
15192
15193 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
15196 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15197 return;
15198 }
15199 if self.arrow_sheets.sheet(sheet).is_none() {
15200 self.arrow_sheets
15201 .sheets
15202 .push(crate::arrow_store::ArrowSheet {
15203 name: std::sync::Arc::<str>::from(sheet),
15204 date_system: self.config.date_system,
15205 columns: Vec::new(),
15206 nrows: 0,
15207 chunk_starts: Vec::new(),
15208 chunk_rows: 32 * 1024,
15209 });
15210 }
15211
15212 let row0 = row.saturating_sub(1) as usize;
15213 let col0 = col.saturating_sub(1) as usize;
15214
15215 let asheet = self
15216 .arrow_sheets
15217 .sheet_mut(sheet)
15218 .expect("ArrowSheet must exist");
15219
15220 let cur_cols = asheet.columns.len();
15221 if col0 >= cur_cols {
15222 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15223 }
15224
15225 if row0 >= asheet.nrows as usize {
15226 if asheet.columns.is_empty() {
15227 asheet.insert_columns(0, 1);
15228 }
15229 asheet.ensure_row_capacity(row0 + 1);
15230 }
15231 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
15232 let ov =
15233 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
15234 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
15235 let _ = ch.overlay.set(in_off, ov);
15236 ch.computed_overlay.remove(in_off)
15241 } else {
15242 return;
15243 };
15244 let abs_threshold = 1024usize;
15246 let frac_den = 50usize;
15247 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
15248 if freed > 0 {
15249 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
15250 }
15251 self.adjust_computed_overlay_bytes(computed_delta);
15252 }
15253 }
15254
15255 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
15260 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15261 return;
15262 }
15263 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15264 return;
15265 };
15266 let row0 = row.saturating_sub(1) as usize;
15267 let col0 = col.saturating_sub(1) as usize;
15268 if row0 >= asheet.nrows as usize {
15269 return;
15270 }
15271 if col0 >= asheet.columns.len() {
15272 return;
15273 }
15274 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15275 return;
15276 };
15277 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
15278 let _ = ch.overlay.remove(in_off);
15279 }
15280 }
15281
15282 fn clear_computed_overlay_col_row_range(
15283 &mut self,
15284 sheet: &str,
15285 col0: usize,
15286 start_row0: usize,
15287 end_row0_exclusive: usize,
15288 ) {
15289 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15290 return;
15291 }
15292 if start_row0 >= end_row0_exclusive {
15293 return;
15294 }
15295
15296 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15297 return;
15298 };
15299 if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
15300 return;
15301 }
15302 let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
15303 if start_row0 >= end_row0_exclusive {
15304 return;
15305 }
15306
15307 let starts = asheet.chunk_starts.clone();
15308 let nrows = asheet.nrows as usize;
15309 let mut delta = 0isize;
15310 let Some(col) = asheet.columns.get_mut(col0) else {
15311 return;
15312 };
15313 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15314 let Some(&chunk_start) = starts.get(chunk_idx) else {
15315 continue;
15316 };
15317 let chunk_end = starts
15318 .get(chunk_idx + 1)
15319 .copied()
15320 .unwrap_or(nrows)
15321 .min(chunk_start.saturating_add(ch.len()));
15322 let clear_start = start_row0.max(chunk_start);
15323 let clear_end = end_row0_exclusive.min(chunk_end);
15324 if clear_start >= clear_end {
15325 continue;
15326 }
15327 if clear_start == chunk_start && clear_end == chunk_end {
15328 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15329 } else {
15330 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15331 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15332 delta = delta.saturating_add(
15333 ch.computed_overlay
15334 .remove_range(start_in_chunk..end_in_chunk),
15335 );
15336 }
15337 }
15338 for (chunk_idx, ch) in &mut col.sparse_chunks {
15339 let Some(&chunk_start) = starts.get(*chunk_idx) else {
15340 continue;
15341 };
15342 let chunk_end = starts
15343 .get(*chunk_idx + 1)
15344 .copied()
15345 .unwrap_or(nrows)
15346 .min(chunk_start.saturating_add(ch.len()));
15347 let clear_start = start_row0.max(chunk_start);
15348 let clear_end = end_row0_exclusive.min(chunk_end);
15349 if clear_start >= clear_end {
15350 continue;
15351 }
15352 if clear_start == chunk_start && clear_end == chunk_end {
15353 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15354 } else {
15355 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15356 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15357 delta = delta.saturating_add(
15358 ch.computed_overlay
15359 .remove_range(start_in_chunk..end_in_chunk),
15360 );
15361 }
15362 }
15363 self.adjust_computed_overlay_bytes(delta);
15364 }
15365
15366 fn clear_computed_overlay_cells_in_region(
15367 &mut self,
15368 cells: &[(SheetId, u32, u32)],
15369 affected_region: &Region,
15370 ) {
15371 let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
15372 for (formula_sheet_id, row, col) in cells {
15373 let row0 = row.saturating_sub(1);
15374 let col0 = col.saturating_sub(1);
15375 let placement_region = Region::point(*formula_sheet_id, row0, col0);
15376 if placement_region.intersects(affected_region) {
15377 by_col
15378 .entry((*formula_sheet_id, col0))
15379 .or_default()
15380 .push(row0);
15381 }
15382 }
15383
15384 for ((formula_sheet_id, col0), mut rows) in by_col {
15385 rows.sort_unstable();
15386 rows.dedup();
15387 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
15388 let mut start = rows[0];
15389 let mut prev = rows[0];
15390 for row in rows.into_iter().skip(1) {
15391 if row == prev.saturating_add(1) {
15392 prev = row;
15393 continue;
15394 }
15395 self.clear_computed_overlay_col_row_range(
15396 &sheet_name,
15397 col0 as usize,
15398 start as usize,
15399 prev.saturating_add(1) as usize,
15400 );
15401 start = row;
15402 prev = row;
15403 }
15404 self.clear_computed_overlay_col_row_range(
15405 &sheet_name,
15406 col0 as usize,
15407 start as usize,
15408 prev.saturating_add(1) as usize,
15409 );
15410 }
15411 }
15412
15413 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
15414 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15415 return;
15416 }
15417
15418 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15419 return;
15420 };
15421 if start_row0 >= asheet.nrows as usize {
15422 return;
15423 }
15424
15425 let starts = asheet.chunk_starts.clone();
15426 let nrows = asheet.nrows as usize;
15427 let mut delta = 0isize;
15428 for col in &mut asheet.columns {
15429 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15430 let Some(&chunk_start) = starts.get(chunk_idx) else {
15431 continue;
15432 };
15433 let chunk_end = starts
15434 .get(chunk_idx + 1)
15435 .copied()
15436 .unwrap_or(nrows)
15437 .min(chunk_start.saturating_add(ch.len()));
15438 if chunk_end <= start_row0 {
15439 continue;
15440 }
15441 if chunk_start >= start_row0 {
15442 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15443 } else {
15444 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15445 delta = delta
15446 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15447 }
15448 }
15449
15450 for (chunk_idx, ch) in &mut col.sparse_chunks {
15451 let Some(&chunk_start) = starts.get(*chunk_idx) else {
15452 continue;
15453 };
15454 let chunk_end = starts
15455 .get(*chunk_idx + 1)
15456 .copied()
15457 .unwrap_or(nrows)
15458 .min(chunk_start.saturating_add(ch.len()));
15459 if chunk_end <= start_row0 {
15460 continue;
15461 }
15462 if chunk_start >= start_row0 {
15463 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15464 } else {
15465 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15466 delta = delta
15467 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15468 }
15469 }
15470 }
15471 self.adjust_computed_overlay_bytes(delta);
15472 }
15473
15474 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
15475 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15476 return;
15477 }
15478
15479 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15480 return;
15481 };
15482 if start_col0 >= asheet.columns.len() {
15483 return;
15484 }
15485
15486 let mut delta = 0isize;
15487 for col in asheet.columns.iter_mut().skip(start_col0) {
15488 for ch in &mut col.chunks {
15489 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15490 }
15491 for ch in col.sparse_chunks.values_mut() {
15492 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15493 }
15494 }
15495 self.adjust_computed_overlay_bytes(delta);
15496 }
15497
15498 #[inline]
15499 fn literal_to_overlay_value(
15500 value: &LiteralValue,
15501 date_system: crate::engine::DateSystem,
15502 ) -> crate::arrow_store::OverlayValue {
15503 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
15504 }
15505
15506 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
15507 self.arrow_sheets
15508 .sheet(sheet)
15509 .map(|sheet| sheet.date_system)
15510 .unwrap_or(self.config.date_system)
15511 }
15512
15513 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15516 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15517 return None;
15518 }
15519 let asheet = self.arrow_sheets.sheet(sheet)?;
15520 let row0 = row.saturating_sub(1) as usize;
15521 let col0 = col.saturating_sub(1) as usize;
15522 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15523 return None;
15524 }
15525 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15526 let ch = asheet.columns[col0].chunk(ch_idx)?;
15527 ch.overlay
15528 .get_scalar(in_off)
15529 .map(|ov| ov.to_literal_for(asheet.date_system))
15530 }
15531
15532 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15535 if !(self.config.arrow_storage_enabled
15536 && self.config.delta_overlay_enabled
15537 && self.config.write_formula_overlay_enabled)
15538 {
15539 return None;
15540 }
15541 let asheet = self.arrow_sheets.sheet(sheet)?;
15542 let row0 = row.saturating_sub(1) as usize;
15543 let col0 = col.saturating_sub(1) as usize;
15544 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15545 return None;
15546 }
15547 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15548 let ch = asheet.columns[col0].chunk(ch_idx)?;
15549 ch.computed_overlay
15550 .get_scalar(in_off)
15551 .map(|ov| ov.to_literal_for(asheet.date_system))
15552 }
15553
15554 fn set_delta_overlay_cell_raw(
15555 &mut self,
15556 sheet: &str,
15557 row: u32,
15558 col: u32,
15559 value: Option<LiteralValue>,
15560 ) {
15561 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15562 return;
15563 }
15564
15565 self.ensure_arrow_sheet(sheet);
15566 let date_system = self.arrow_sheet_date_system(sheet);
15567 let ov_opt = value
15568 .as_ref()
15569 .map(|value| Self::literal_to_overlay_value(value, date_system));
15570 let row0 = row.saturating_sub(1) as usize;
15571 let col0 = col.saturating_sub(1) as usize;
15572 let asheet = self
15573 .arrow_sheets
15574 .sheet_mut(sheet)
15575 .expect("ArrowSheet must exist");
15576
15577 let cur_cols = asheet.columns.len();
15578 if col0 >= cur_cols {
15579 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15580 }
15581 if row0 >= asheet.nrows as usize {
15582 if asheet.columns.is_empty() {
15583 asheet.insert_columns(0, 1);
15584 }
15585 asheet.ensure_row_capacity(row0 + 1);
15586 }
15587
15588 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15589 return;
15590 };
15591 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15592 return;
15593 };
15594
15595 if let Some(ov) = ov_opt {
15596 let _ = ch.overlay.set(in_off, ov);
15597 } else {
15598 let _ = ch.overlay.remove(in_off);
15599 }
15600 }
15601
15602 fn set_computed_overlay_cell_raw(
15603 &mut self,
15604 sheet: &str,
15605 row: u32,
15606 col: u32,
15607 value: Option<LiteralValue>,
15608 ) {
15609 if !(self.config.arrow_storage_enabled
15610 && self.config.delta_overlay_enabled
15611 && self.config.write_formula_overlay_enabled)
15612 {
15613 return;
15614 }
15615
15616 self.ensure_arrow_sheet(sheet);
15617 let date_system = self.arrow_sheet_date_system(sheet);
15618 let ov_opt = value
15619 .as_ref()
15620 .map(|value| Self::literal_to_overlay_value(value, date_system));
15621 let row0 = row.saturating_sub(1) as usize;
15622 let col0 = col.saturating_sub(1) as usize;
15623 let asheet = self
15624 .arrow_sheets
15625 .sheet_mut(sheet)
15626 .expect("ArrowSheet must exist");
15627
15628 let cur_cols = asheet.columns.len();
15629 if col0 >= cur_cols {
15630 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15631 }
15632 if row0 >= asheet.nrows as usize {
15633 if asheet.columns.is_empty() {
15634 asheet.insert_columns(0, 1);
15635 }
15636 asheet.ensure_row_capacity(row0 + 1);
15637 }
15638
15639 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15640 return;
15641 };
15642 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15643 return;
15644 };
15645
15646 let delta = if let Some(ov) = ov_opt {
15647 ch.computed_overlay.set(in_off, ov)
15648 } else {
15649 ch.computed_overlay.remove(in_off)
15650 };
15651 self.adjust_computed_overlay_bytes(delta);
15652 }
15653
15654 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
15655 use crate::engine::ArrowOp;
15656
15657 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
15658 Box::new(batch.ops.iter().rev())
15659 } else {
15660 Box::new(batch.ops.iter())
15661 };
15662
15663 for op in iter {
15664 match op {
15665 ArrowOp::SetDeltaCell {
15666 sheet_id,
15667 row0,
15668 col0,
15669 old,
15670 new,
15671 } => {
15672 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15673 let v = if undo { old.clone() } else { new.clone() };
15674 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15675 }
15676 ArrowOp::SetComputedCell {
15677 sheet_id,
15678 row0,
15679 col0,
15680 old,
15681 new,
15682 } => {
15683 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15684 let v = if undo { old.clone() } else { new.clone() };
15685 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15686 }
15687 ArrowOp::RestoreComputedRect {
15688 sheet_id,
15689 sr0,
15690 sc0,
15691 er0,
15692 ec0,
15693 old,
15694 new,
15695 } => {
15696 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15697 let vals = if undo { old } else { new };
15698 let height = (*er0).saturating_sub(*sr0) as usize + 1;
15699 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
15700 for r in 0..height {
15701 for c in 0..width {
15702 let v = vals
15703 .get(r)
15704 .and_then(|row| row.get(c))
15705 .cloned()
15706 .unwrap_or(LiteralValue::Empty);
15707 self.set_computed_overlay_cell_raw(
15708 &sheet,
15709 *sr0 + 1 + r as u32,
15710 *sc0 + 1 + c as u32,
15711 Some(v),
15712 );
15713 }
15714 }
15715 }
15716 ArrowOp::InsertRows {
15717 sheet_id,
15718 before0,
15719 count,
15720 } => {
15721 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15722 self.ensure_arrow_sheet(&sheet);
15723 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15724 if undo {
15725 asheet.delete_rows(*before0 as usize, *count as usize);
15726 } else {
15727 asheet.insert_rows(*before0 as usize, *count as usize);
15728 }
15729 }
15730 }
15731 ArrowOp::InsertCols {
15732 sheet_id,
15733 before0,
15734 count,
15735 } => {
15736 let sheet = self.graph.sheet_name(*sheet_id).to_string();
15737 self.ensure_arrow_sheet(&sheet);
15738 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15739 if undo {
15740 asheet.delete_columns(*before0 as usize, *count as usize);
15741 } else {
15742 asheet.insert_columns(*before0 as usize, *count as usize);
15743 }
15744 }
15745 }
15746 }
15747 }
15748 }
15749
15750 fn record_spill_ops_into_arrow_undo(
15751 &mut self,
15752 undo: &mut crate::engine::ArrowUndoBatch,
15753 events: &[crate::engine::ChangeEvent],
15754 ) {
15755 use crate::engine::ChangeEvent;
15756 use formualizer_common::LiteralValue;
15757
15758 #[allow(clippy::type_complexity)]
15759 let rect_from_snapshot =
15760 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
15761 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
15762 if snap.target_cells.is_empty() {
15763 return None;
15764 }
15765 let sheet_id = snap.target_cells[0].sheet_id;
15766 let sr0 = snap.target_cells[0].coord.row();
15767 let sc0 = snap.target_cells[0].coord.col();
15768 if snap.values.is_empty() || snap.values[0].is_empty() {
15769 return None;
15770 }
15771 let h = snap.values.len() as u32;
15772 let w = snap.values[0].len() as u32;
15773 let er0 = sr0.saturating_add(h.saturating_sub(1));
15774 let ec0 = sc0.saturating_add(w.saturating_sub(1));
15775 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
15776 };
15777
15778 for ev in events {
15779 match ev {
15780 ChangeEvent::SpillCommitted { old, new, .. } => {
15781 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
15782 let old_vals = if let Some(old_snap) = old {
15783 rect_from_snapshot(old_snap)
15784 .map(|(_, _, _, _, _, v)| v)
15785 .unwrap_or_else(|| {
15786 vec![
15787 vec![LiteralValue::Empty; new_vals[0].len()];
15788 new_vals.len()
15789 ]
15790 })
15791 } else {
15792 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
15793 };
15794 undo.record_restore_computed_rect(
15795 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15796 );
15797 }
15798 }
15799 ChangeEvent::SpillCleared { old, .. } => {
15800 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
15801 let new_vals =
15802 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
15803 undo.record_restore_computed_rect(
15804 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15805 );
15806 }
15807 }
15808 _ => {}
15809 }
15810 }
15811 }
15812
15813 fn mirror_value_to_computed_overlay(
15818 &mut self,
15819 sheet: &str,
15820 row: u32,
15821 col: u32,
15822 value: &LiteralValue,
15823 ) {
15824 if !(self.config.arrow_storage_enabled
15825 && self.config.delta_overlay_enabled
15826 && self.config.write_formula_overlay_enabled)
15827 {
15828 return;
15829 }
15830 if self.computed_overlay_mirroring_disabled {
15831 return;
15832 }
15833
15834 let date_system = self.arrow_sheet_date_system(sheet);
15835 let ov = Self::literal_to_overlay_value(value, date_system);
15836 self.write_computed_overlay_value_0based(
15837 sheet,
15838 row.saturating_sub(1),
15839 col.saturating_sub(1),
15840 ov,
15841 );
15842 }
15843
15844 fn write_computed_overlay_value_0based(
15845 &mut self,
15846 sheet: &str,
15847 row0: u32,
15848 col0: u32,
15849 value: OverlayValue,
15850 ) {
15851 if !(self.config.arrow_storage_enabled
15852 && self.config.delta_overlay_enabled
15853 && self.config.write_formula_overlay_enabled)
15854 {
15855 return;
15856 }
15857 if self.computed_overlay_mirroring_disabled {
15858 return;
15859 }
15860
15861 self.ensure_arrow_sheet(sheet);
15862
15863 let row0 = row0 as usize;
15864 let col0 = col0 as usize;
15865 let asheet = self
15866 .arrow_sheets
15867 .sheet_mut(sheet)
15868 .expect("ArrowSheet must exist");
15869
15870 let cur_cols = asheet.columns.len();
15871 if col0 >= cur_cols {
15872 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15873 }
15874
15875 if row0 >= asheet.nrows as usize {
15876 if asheet.columns.is_empty() {
15877 asheet.insert_columns(0, 1);
15878 }
15879 asheet.ensure_row_capacity(row0 + 1);
15880 }
15881
15882 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15883 return;
15884 };
15885 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15886 return;
15887 };
15888
15889 let delta = ch.computed_overlay.set_scalar(in_off, value);
15890 self.adjust_computed_overlay_bytes(delta);
15891
15892 if let Some(cap) = self.config.max_overlay_memory_bytes
15893 && self.computed_overlay_bytes_estimate > cap
15894 {
15895 self.disable_computed_overlay_mirroring_due_to_budget(cap);
15896 }
15897 }
15898
15899 pub(crate) fn plan_computed_write_coalescing(
15900 &self,
15901 buffer: &ComputedWriteBuffer,
15902 ) -> ComputedWriteCoalescingPlan {
15903 self.plan_computed_write_coalescing_from_writes(buffer.writes().iter().cloned())
15904 }
15905
15906 fn plan_owned_computed_write_coalescing(
15907 &self,
15908 writes: Vec<ComputedWrite>,
15909 ) -> ComputedWriteCoalescingPlan {
15910 self.plan_computed_write_coalescing_from_writes(writes)
15911 }
15912
15913 fn plan_computed_write_coalescing_from_writes(
15914 &self,
15915 writes: impl IntoIterator<Item = ComputedWrite>,
15916 ) -> ComputedWriteCoalescingPlan {
15917 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
15918 BTreeMap::new();
15919 let mut input_cells = 0usize;
15920
15921 for write in writes {
15922 match write {
15923 ComputedWrite::Cell {
15924 seq,
15925 sheet_id,
15926 row0,
15927 col0,
15928 value,
15929 } => {
15930 input_cells = input_cells.saturating_add(1);
15931 self.push_computed_write_plan_entry(
15932 &mut groups,
15933 seq,
15934 sheet_id,
15935 row0,
15936 col0,
15937 value,
15938 );
15939 }
15940 ComputedWrite::Rect {
15941 seq,
15942 sheet_id,
15943 sr0,
15944 sc0,
15945 values,
15946 } => {
15947 for (r_off, row) in values.into_iter().enumerate() {
15948 for (c_off, value) in row.into_iter().enumerate() {
15949 input_cells = input_cells.saturating_add(1);
15950 self.push_computed_write_plan_entry(
15951 &mut groups,
15952 seq,
15953 sheet_id,
15954 sr0.saturating_add(r_off as u32),
15955 sc0.saturating_add(c_off as u32),
15956 value,
15957 );
15958 }
15959 }
15960 }
15961 }
15962 }
15963
15964 let mut plan = ComputedWriteCoalescingPlan {
15965 chunks: Vec::with_capacity(groups.len()),
15966 input_cells,
15967 coalesced_cells: 0,
15968 overwritten_cells: 0,
15969 };
15970 for (key, entries) in groups {
15971 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(key, entries);
15972 plan.coalesced_cells = plan
15973 .coalesced_cells
15974 .saturating_add(chunk_plan.entries.len());
15975 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
15976 plan.chunks.push(chunk_plan);
15977 }
15978 debug_assert_eq!(
15979 plan.input_cells,
15980 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
15981 );
15982 plan
15983 }
15984
15985 fn push_computed_write_plan_entry(
15986 &self,
15987 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
15988 seq: u64,
15989 sheet_id: SheetId,
15990 row0: u32,
15991 col0: u32,
15992 value: OverlayValue,
15993 ) {
15994 let (chunk_idx, chunk_start_row0, row_in_chunk) =
15995 self.locate_computed_write_chunk(sheet_id, row0);
15996 let key = ComputedWriteChunkKey {
15997 sheet_id,
15998 col0,
15999 chunk_idx,
16000 chunk_start_row0,
16001 };
16002 groups
16003 .entry(key)
16004 .or_default()
16005 .push(ComputedWriteChunkEntryPlan {
16006 row_in_chunk,
16007 seq,
16008 value,
16009 });
16010 }
16011
16012 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
16013 let sheet_name = self.graph.sheet_name(sheet_id);
16014 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
16015 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
16016 }
16017 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
16018 }
16019
16020 fn locate_row_in_sheet_for_computed_write_plan(
16021 sheet: &crate::arrow_store::ArrowSheet,
16022 row0: usize,
16023 ) -> (usize, u32, usize) {
16024 if row0 < sheet.nrows as usize
16025 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
16026 {
16027 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
16028 return (chunk_idx, chunk_start as u32, row_in_chunk);
16029 }
16030
16031 let chunk_rows = sheet.chunk_rows.max(1);
16032 if sheet.chunk_starts.is_empty() {
16033 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
16034 }
16035
16036 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
16037 let mut chunk_start = sheet.chunk_starts[chunk_idx];
16038 while chunk_start.saturating_add(chunk_rows) <= row0 {
16039 chunk_idx = chunk_idx.saturating_add(1);
16040 chunk_start = chunk_start.saturating_add(chunk_rows);
16041 }
16042 (
16043 chunk_idx,
16044 chunk_start as u32,
16045 row0.saturating_sub(chunk_start),
16046 )
16047 }
16048
16049 fn locate_row_in_empty_sheet_for_computed_write_plan(
16050 row0: usize,
16051 chunk_rows: usize,
16052 ) -> (usize, u32, usize) {
16053 let chunk_rows = chunk_rows.max(1);
16054 let chunk_idx = row0 / chunk_rows;
16055 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
16056 (
16057 chunk_idx,
16058 chunk_start as u32,
16059 row0.saturating_sub(chunk_start),
16060 )
16061 }
16062
16063 #[cfg(test)]
16064 pub(crate) fn debug_plan_computed_write_coalescing(
16065 &self,
16066 buffer: &ComputedWriteBuffer,
16067 ) -> ComputedWriteCoalescingPlan {
16068 self.plan_computed_write_coalescing(buffer)
16069 }
16070
16071 fn record_computed_write_buffer_delta(
16072 &self,
16073 buffer: &ComputedWriteBuffer,
16074 delta: &mut DeltaCollector,
16075 ) {
16076 if delta.mode == DeltaMode::Off {
16077 return;
16078 }
16079 let mut final_values = BTreeMap::new();
16080 for write in buffer.writes() {
16081 match write {
16082 ComputedWrite::Cell {
16083 sheet_id,
16084 row0,
16085 col0,
16086 value,
16087 ..
16088 } => {
16089 final_values.insert((*sheet_id, *row0, *col0), value.clone());
16090 }
16091 ComputedWrite::Rect {
16092 sheet_id,
16093 sr0,
16094 sc0,
16095 values,
16096 ..
16097 } => {
16098 for (row_offset, row) in values.iter().enumerate() {
16099 for (col_offset, value) in row.iter().enumerate() {
16100 final_values.insert(
16101 (
16102 *sheet_id,
16103 sr0.saturating_add(row_offset as u32),
16104 sc0.saturating_add(col_offset as u32),
16105 ),
16106 value.clone(),
16107 );
16108 }
16109 }
16110 }
16111 }
16112 }
16113 for ((sheet_id, row0, col0), value) in final_values {
16114 let old = self
16115 .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
16116 .unwrap_or(LiteralValue::Empty);
16117 if old != value.to_literal_for(self.config.date_system) {
16118 delta.record_cell(sheet_id, row0, col0);
16119 }
16120 }
16121 }
16122
16123 pub(crate) fn flush_computed_write_buffer(
16124 &mut self,
16125 buffer: &mut ComputedWriteBuffer,
16126 ) -> Result<(), ExcelError> {
16127 if buffer.is_empty() {
16128 return Ok(());
16129 }
16130
16131 self.resource_checkpoint(0)?;
16136 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
16137 let plan = self.plan_owned_computed_write_coalescing(buffer.take_writes());
16138 self.flush_computed_write_plan(plan);
16139 self.observe_evaluation_commit_window(commit_started);
16140
16141 Ok(())
16142 }
16143
16144 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
16145 for chunk in plan.chunks {
16146 self.flush_computed_write_chunk_plan(chunk);
16147 }
16148 }
16149
16150 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
16151 match &chunk.shape {
16152 ComputedWriteChunkPlanShape::Point => {
16153 self.flush_computed_write_chunk_plan_as_points(chunk);
16154 }
16155 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
16156 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
16157 }
16158 ComputedWriteChunkPlanShape::DenseRange { .. } => {
16159 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16160 }
16161 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
16162 if Self::should_emit_computed_run_fragment(*len, *runs) {
16163 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
16164 } else {
16165 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16166 }
16167 }
16168 }
16169 }
16170
16171 #[inline]
16172 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
16173 runs <= len / 2
16174 }
16175
16176 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
16177 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
16178 for entry in chunk.entries {
16179 let row0 = chunk
16180 .chunk_start_row0
16181 .saturating_add(entry.row_in_chunk as u32);
16182 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
16183 }
16184 }
16185
16186 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
16187 &mut self,
16188 chunk: ComputedWriteChunkPlan,
16189 ) {
16190 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
16191 let sheet_id = chunk.sheet_id;
16192 let col0 = chunk.col0;
16193 let chunk_idx = chunk.chunk_idx;
16194 let chunk_start_row0 = chunk.chunk_start_row0;
16195 let items: Vec<(usize, OverlayValue)> = chunk
16196 .entries
16197 .into_iter()
16198 .map(|entry| (entry.row_in_chunk, entry.value))
16199 .collect();
16200 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
16201 items,
16202 point_estimate,
16203 ) {
16204 Some(Ok(fragment)) => {
16205 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
16206 }
16207 Some(Err(cells)) => {
16208 self.flush_computed_overlay_cells_as_points(
16209 sheet_id,
16210 col0,
16211 chunk_start_row0,
16212 cells,
16213 );
16214 }
16215 None => {}
16216 }
16217 }
16218
16219 #[inline]
16220 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
16221 chunk
16222 .entries
16223 .iter()
16224 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
16225 .fold(0usize, usize::saturating_add)
16226 }
16227
16228 fn flush_computed_overlay_cells_as_points(
16229 &mut self,
16230 sheet_id: SheetId,
16231 col0: u32,
16232 chunk_start_row0: u32,
16233 cells: Vec<(usize, OverlayValue)>,
16234 ) {
16235 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16236 for (row_in_chunk, value) in cells {
16237 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
16238 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
16239 }
16240 }
16241
16242 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16243 if chunk.entries.is_empty() {
16244 return;
16245 }
16246 let start = chunk.entries[0].row_in_chunk;
16247 let values: Vec<OverlayValue> =
16248 chunk.entries.into_iter().map(|entry| entry.value).collect();
16249 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
16250 self.apply_computed_overlay_fragment(
16251 chunk.sheet_id,
16252 chunk.col0,
16253 chunk.chunk_idx,
16254 fragment,
16255 );
16256 }
16257 }
16258
16259 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16260 if chunk.entries.is_empty() {
16261 return;
16262 }
16263 let start = chunk.entries[0].row_in_chunk;
16264 let values: Vec<OverlayValue> =
16265 chunk.entries.into_iter().map(|entry| entry.value).collect();
16266 if let Some(fragment) = OverlayFragment::run_range(start, values) {
16267 self.apply_computed_overlay_fragment(
16268 chunk.sheet_id,
16269 chunk.col0,
16270 chunk.chunk_idx,
16271 fragment,
16272 );
16273 }
16274 }
16275
16276 fn apply_computed_overlay_fragment(
16277 &mut self,
16278 sheet_id: SheetId,
16279 col0: u32,
16280 chunk_idx: usize,
16281 fragment: OverlayFragment,
16282 ) {
16283 if !(self.config.arrow_storage_enabled
16284 && self.config.delta_overlay_enabled
16285 && self.config.write_formula_overlay_enabled)
16286 {
16287 return;
16288 }
16289 if self.computed_overlay_mirroring_disabled {
16290 return;
16291 }
16292
16293 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16294 self.ensure_arrow_sheet(&sheet_name);
16295
16296 let col0 = col0 as usize;
16297 let asheet = self
16298 .arrow_sheets
16299 .sheet_mut(&sheet_name)
16300 .expect("ArrowSheet must exist");
16301
16302 let cur_cols = asheet.columns.len();
16303 if col0 >= cur_cols {
16304 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16305 }
16306
16307 let start_row0 = asheet
16308 .chunk_starts
16309 .get(chunk_idx)
16310 .copied()
16311 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
16312 let required_rows =
16313 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
16314 if required_rows > asheet.nrows as usize {
16315 if asheet.columns.is_empty() {
16316 asheet.insert_columns(0, 1);
16317 }
16318 asheet.ensure_row_capacity(required_rows);
16319 }
16320
16321 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
16322 return;
16323 };
16324 let delta = ch.computed_overlay.apply_fragment(fragment);
16325 self.adjust_computed_overlay_bytes(delta);
16326
16327 if let Some(cap) = self.config.max_overlay_memory_bytes
16328 && self.computed_overlay_bytes_estimate > cap
16329 {
16330 self.disable_computed_overlay_mirroring_due_to_budget(cap);
16331 }
16332 }
16333
16334 #[inline]
16335 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
16336 if delta >= 0 {
16337 self.computed_overlay_bytes_estimate = self
16338 .computed_overlay_bytes_estimate
16339 .saturating_add(delta as usize);
16340 } else {
16341 self.computed_overlay_bytes_estimate = self
16342 .computed_overlay_bytes_estimate
16343 .saturating_sub((-delta) as usize);
16344 }
16345 }
16346
16347 fn clear_all_computed_overlays(&mut self) {
16348 let mut freed_total = 0usize;
16349 for sh in self.arrow_sheets.sheets.iter_mut() {
16350 for col in sh.columns.iter_mut() {
16351 for ch in col.chunks.iter_mut() {
16352 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16353 }
16354 for ch in col.sparse_chunks.values_mut() {
16355 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16356 }
16357 }
16358 }
16359 self.computed_overlay_bytes_estimate = self
16360 .computed_overlay_bytes_estimate
16361 .saturating_sub(freed_total);
16362 }
16363
16364 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
16365 self.compact_all_computed_overlays();
16368 }
16369
16370 fn compact_all_computed_overlays(&mut self) {
16373 let mut freed_total = 0usize;
16374 for sheet in self.arrow_sheets.sheets.iter_mut() {
16375 for col_idx in 0..sheet.columns.len() {
16376 let num_dense = sheet.columns[col_idx].chunks.len();
16378 for ch_idx in 0..num_dense {
16379 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
16380 }
16381 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
16383 .sparse_chunks
16384 .keys()
16385 .copied()
16386 .collect();
16387 for ch_idx in sparse_keys {
16388 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
16389 }
16390 }
16391 }
16392 self.computed_overlay_bytes_estimate = self
16393 .computed_overlay_bytes_estimate
16394 .saturating_sub(freed_total);
16395 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16396 }
16397
16398 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
16399 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
16400 }
16401
16402 fn record_vertex_value_to_overlay(
16403 &mut self,
16404 vertex_id: VertexId,
16405 value: &LiteralValue,
16406 computed_writes: Option<&mut ComputedWriteBuffer>,
16407 ) -> Result<(), ExcelError> {
16408 if !(self.config.arrow_storage_enabled
16409 && self.config.delta_overlay_enabled
16410 && self.config.write_formula_overlay_enabled)
16411 {
16412 return Ok(());
16413 }
16414 if self.computed_overlay_mirroring_disabled {
16415 return Ok(());
16416 }
16417 if !matches!(
16418 self.graph.get_vertex_kind(vertex_id),
16419 VertexKind::FormulaScalar | VertexKind::FormulaArray
16420 ) {
16421 return Ok(());
16422 }
16423 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
16424 return Ok(());
16425 };
16426 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
16427 let date_system = self.arrow_sheet_date_system(&sheet_name);
16428 let ov = Self::literal_to_overlay_value(value, date_system);
16429 if let Some(buffer) = computed_writes {
16430 buffer.push_cell(cell.sheet_id, cell.coord.row(), cell.coord.col(), ov);
16431 if self.should_flush_computed_write_buffer(buffer) {
16432 self.flush_computed_write_buffer(buffer)?;
16433 }
16434 } else {
16435 self.write_computed_overlay_value_0based(
16436 &sheet_name,
16437 cell.coord.row(),
16438 cell.coord.col(),
16439 ov,
16440 );
16441 }
16442 Ok(())
16443 }
16444
16445 #[inline]
16446 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
16447 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
16448 if cap == 0 {
16449 return false;
16450 }
16451 self.computed_overlay_bytes_estimate
16452 .saturating_add(buffer.estimated_bytes())
16453 > cap
16454 })
16455 }
16456
16457 pub fn overlay_memory_usage(&self) -> usize {
16459 self.computed_overlay_bytes_estimate
16460 }
16461
16462 #[cfg(test)]
16463 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
16464 self.overlay_compactions
16465 }
16466
16467 #[cfg(test)]
16468 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
16469 let mut total = 0usize;
16470 for sheet in &self.arrow_sheets.sheets {
16471 for column in &sheet.columns {
16472 for chunk in &column.chunks {
16473 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16474 }
16475 for chunk in column.sparse_chunks.values() {
16476 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16477 }
16478 }
16479 }
16480 self.computed_overlay_bytes_estimate = total;
16481 total
16482 }
16483
16484 fn resolve_sheet_locator_for_write(
16485 &mut self,
16486 loc: formualizer_common::SheetLocator<'_>,
16487 current_sheet: &str,
16488 ) -> Result<SheetId, ExcelError> {
16489 Ok(match loc {
16490 formualizer_common::SheetLocator::Id(id) => id,
16491 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
16492 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
16493 })
16494 }
16495
16496 fn resolve_sheet_locator_for_read(
16497 &self,
16498 loc: formualizer_common::SheetLocator<'_>,
16499 current_sheet: &str,
16500 ) -> Result<SheetId, ExcelError> {
16501 match loc {
16502 formualizer_common::SheetLocator::Id(id) => Ok(id),
16503 formualizer_common::SheetLocator::Name(name) => self
16504 .graph
16505 .sheet_id(name.as_ref())
16506 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16507 formualizer_common::SheetLocator::Current => self
16508 .graph
16509 .sheet_id(current_sheet)
16510 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16511 }
16512 }
16513
16514 pub fn set_cell_value(
16516 &mut self,
16517 sheet: &str,
16518 row: u32,
16519 col: u32,
16520 value: LiteralValue,
16521 ) -> Result<(), ExcelError> {
16522 self.observe_function_semantic_epoch()?;
16523 let sheet_existed = self.graph.sheet_id(sheet).is_some();
16524 let sheet_id = self.graph.sheet_id_mut(sheet);
16525 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16526 let replaced_formula =
16527 self.graph
16528 .get_vertex_id_for_address(&cell_ref)
16529 .is_some_and(|vertex| {
16530 matches!(
16531 self.graph.get_vertex_kind(*vertex),
16532 VertexKind::FormulaScalar | VertexKind::FormulaArray
16533 )
16534 });
16535 self.demote_span_containing_cell_for_write(
16536 sheet_id,
16537 row.saturating_sub(1),
16538 col.saturating_sub(1),
16539 )
16540 .map_err(Self::editor_error_to_excel)?;
16541 self.graph.set_cell_value(sheet, row, col, value.clone())?;
16542 self.record_formula_plane_changed_cell(sheet, row, col);
16543 if !sheet_existed || replaced_formula {
16544 self.mark_topology_edited();
16545 }
16546 self.mirror_value_to_overlay(sheet, row, col, &value);
16548 self.snapshot_id
16550 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16551 self.has_edited = true;
16552 Ok(())
16553 }
16554
16555 fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
16560 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16561 return;
16562 }
16563 let sheet_id = self.graph.sheet_id_mut(sheet);
16564 self.record_formula_plane_structural_change(StructuralScope::Cell {
16565 sheet: sheet_id,
16566 row: row.saturating_sub(1),
16567 col: col.saturating_sub(1),
16568 });
16569 }
16570
16571 fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
16572 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16573 return;
16574 }
16575
16576 match event {
16577 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16578 self.record_formula_plane_structural_change(StructuralScope::Cell {
16579 sheet: addr.sheet_id,
16580 row: addr.coord.row(),
16581 col: addr.coord.col(),
16582 });
16583 }
16584 ChangeEvent::SpillCommitted { new, .. } => {
16585 if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
16586 self.record_formula_plane_structural_change(scope);
16587 }
16588 }
16589 ChangeEvent::SpillCleared { old, .. } => {
16590 if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
16591 self.record_formula_plane_structural_change(scope);
16592 }
16593 }
16594 ChangeEvent::DefineName { .. }
16595 | ChangeEvent::UpdateName { .. }
16596 | ChangeEvent::DeleteName { .. }
16597 | ChangeEvent::NamedRangeAdjusted { .. } => {
16598 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
16602 }
16603 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
16604 }
16607 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
16608 self.record_formula_plane_structural_change(StructuralScope::Region(
16609 Region::whole_row(*sheet_id, *row0),
16610 ));
16611 }
16612 ChangeEvent::AddVertex { .. }
16613 | ChangeEvent::RemoveVertex { .. }
16614 | ChangeEvent::EdgeAdded { .. }
16615 | ChangeEvent::EdgeRemoved { .. }
16616 | ChangeEvent::CompoundStart { .. }
16617 | ChangeEvent::CompoundEnd { .. }
16618 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
16619 }
16620 }
16621
16622 fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
16623 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16624 return;
16625 }
16626
16627 match scope {
16628 StructuralScope::Cell { sheet, row, col } => {
16629 self.graph
16630 .mark_formula_region_dirty(Region::point(sheet, row, col));
16631 }
16632 StructuralScope::Region(region) => {
16633 self.graph.mark_formula_region_dirty(region);
16634 }
16635 StructuralScope::Sheet(sheet_id) => {
16636 self.graph
16637 .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
16638 }
16639 StructuralScope::RemovedSheet(sheet_id) => {
16640 let removed_refs = {
16641 let authority = self.graph.formula_authority();
16642 authority
16643 .active_span_refs()
16644 .into_iter()
16645 .filter(|span_ref| {
16646 authority
16647 .plane
16648 .spans
16649 .get(*span_ref)
16650 .map(|span| span.sheet_id == sheet_id)
16651 .unwrap_or(false)
16652 })
16653 .collect::<Vec<_>>()
16654 };
16655
16656 {
16657 let authority = self.graph.formula_authority_mut();
16658 for span_ref in removed_refs {
16659 authority.plane.remove_span(span_ref);
16660 }
16661 let _ = authority.rebuild_indexes();
16662 }
16663 self.graph
16664 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16665 }
16666 StructuralScope::OpaqueGlobal => {
16667 let _ = self.graph.formula_authority_mut().rebuild_indexes();
16668 self.graph
16669 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16670 }
16671 StructuralScope::AllSheets => {
16672 let _ = self.graph.formula_authority_mut().rebuild_indexes();
16676 }
16677 }
16678 }
16679
16680 fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
16681 let first = cells.first()?;
16682 let sheet_id = first.sheet_id;
16683 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
16684 return Some(StructuralScope::OpaqueGlobal);
16685 }
16686 let mut row_start = first.coord.row();
16687 let mut row_end = row_start;
16688 let mut col_start = first.coord.col();
16689 let mut col_end = col_start;
16690 for cell in cells.iter().skip(1) {
16691 row_start = row_start.min(cell.coord.row());
16692 row_end = row_end.max(cell.coord.row());
16693 col_start = col_start.min(cell.coord.col());
16694 col_end = col_end.max(cell.coord.col());
16695 }
16696 Some(StructuralScope::Region(Region::rect(
16697 sheet_id, row_start, row_end, col_start, col_end,
16698 )))
16699 }
16700
16701 pub fn set_cell_value_ref(
16702 &mut self,
16703 cell: formualizer_common::SheetCellRef<'_>,
16704 current_sheet: &str,
16705 value: LiteralValue,
16706 ) -> Result<(), ExcelError> {
16707 let owned = cell.into_owned();
16708 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16709 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16710 self.set_cell_value(
16711 &sheet_name,
16712 owned.coord.row() + 1,
16713 owned.coord.col() + 1,
16714 value,
16715 )
16716 }
16717
16718 pub fn set_cell_formula_ref(
16719 &mut self,
16720 cell: formualizer_common::SheetCellRef<'_>,
16721 current_sheet: &str,
16722 ast: ASTNode,
16723 ) -> Result<(), ExcelError> {
16724 let owned = cell.into_owned();
16725 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16726 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16727 self.set_cell_formula(
16728 &sheet_name,
16729 owned.coord.row() + 1,
16730 owned.coord.col() + 1,
16731 ast,
16732 )
16733 }
16734
16735 pub fn get_cell_value_ref(
16736 &self,
16737 cell: formualizer_common::SheetCellRef<'_>,
16738 current_sheet: &str,
16739 ) -> Result<Option<LiteralValue>, ExcelError> {
16740 let owned = cell.into_owned();
16741 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
16742 let sheet_name = self.graph.sheet_name(sheet_id);
16743 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
16744 }
16745
16746 pub fn resolve_range_view_sheet_ref<'c>(
16747 &'c self,
16748 r: &formualizer_common::SheetRef<'_>,
16749 current_sheet: &str,
16750 ) -> Result<RangeView<'c>, ExcelError> {
16751 use formualizer_common::SheetLocator;
16752
16753 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
16754 match loc {
16755 SheetLocator::Current => Ok(None),
16756 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
16757 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
16758 }
16759 };
16760
16761 let rt = match r {
16762 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
16763 sheet: sheet_to_opt_name(cell.sheet.clone())?,
16764 row: cell.coord.row() + 1,
16765 col: cell.coord.col() + 1,
16766 row_abs: cell.coord.row_abs(),
16767 col_abs: cell.coord.col_abs(),
16768 },
16769 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
16770 sheet: sheet_to_opt_name(range.sheet.clone())?,
16771 start_row: range.start_row.map(|b| b.index + 1),
16772 start_col: range.start_col.map(|b| b.index + 1),
16773 end_row: range.end_row.map(|b| b.index + 1),
16774 end_col: range.end_col.map(|b| b.index + 1),
16775 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
16776 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
16777 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
16778 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
16779 },
16780 };
16781
16782 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
16783 }
16784
16785 pub fn set_cell_formula(
16787 &mut self,
16788 sheet: &str,
16789 row: u32,
16790 col: u32,
16791 ast: ASTNode,
16792 ) -> Result<(), ExcelError> {
16793 self.observe_function_semantic_epoch()?;
16794 let sheet_id = self.graph.sheet_id_mut(sheet);
16795 self.demote_span_containing_cell_for_write(
16796 sheet_id,
16797 row.saturating_sub(1),
16798 col.saturating_sub(1),
16799 )
16800 .map_err(Self::editor_error_to_excel)?;
16801 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16802 let ingested = {
16803 let mut pipeline = self.ingest_pipeline();
16804 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
16805 };
16806 self.graph.set_cell_formula_with_plan(
16807 sheet,
16808 row,
16809 col,
16810 ingested.ast_id,
16811 &ingested.dep_plan,
16812 ingested.dep_plan.volatile,
16813 ingested.dep_plan.dynamic,
16814 )?;
16815 self.record_formula_plane_changed_cell(sheet, row, col);
16816
16817 self.clear_delta_overlay_cell(sheet, row, col);
16822
16823 self.mark_topology_edited();
16825 Ok(())
16826 }
16827
16828 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
16830 where
16831 I: IntoIterator<Item = (u32, u32, ASTNode)>,
16832 {
16833 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
16834 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
16835 let sheet_id = self.graph.sheet_id_mut(sheet);
16836 let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
16837 let placement =
16838 PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
16839 self.graph
16840 .formula_authority()
16841 .plane
16842 .spans
16843 .find_at(placement)
16844 .is_some()
16845 });
16846 if writes_inside_active_span {
16847 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
16848 .map_err(Self::editor_error_to_excel)?;
16849 }
16850 let ingested = {
16851 let mut pipeline = self.ingest_pipeline();
16852 let inputs = collected.into_iter().map(|(row, col, ast)| {
16853 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16854 (FormulaAstInput::Tree(ast), placement, None)
16855 });
16856 pipeline.ingest_batch(inputs)?
16857 };
16858 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
16859 .into_iter()
16860 .map(|formula| {
16861 (
16862 formula.placement.coord.row() + 1,
16863 formula.placement.coord.col() + 1,
16864 formula.ast_id,
16865 formula.dep_plan,
16866 )
16867 })
16868 .collect();
16869 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
16870 for (row, col) in edited_cells {
16871 self.record_formula_plane_changed_cell(sheet, row, col);
16872 }
16873 if n > 0 {
16875 self.mark_topology_edited();
16876 }
16877 Ok(n)
16878 }
16879
16880 #[inline]
16881 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
16882 match v {
16883 LiteralValue::Empty => None,
16884 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
16885 other => Some(other),
16886 }
16887 }
16888
16889 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16891 self.read_cell_value(sheet, row, col)
16892 .and_then(Self::normalize_public_cell_read)
16893 }
16894
16895 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16897 let asheet = self.sheet_store().sheet(sheet)?;
16898 let r0 = row.saturating_sub(1) as usize;
16899 let c0 = col.saturating_sub(1) as usize;
16900 let v = asheet.get_cell_value(r0, c0);
16901 if matches!(v, LiteralValue::Empty) {
16902 None
16903 } else {
16904 Some(v)
16905 }
16906 }
16907
16908 pub(crate) fn read_range_values(
16910 &self,
16911 sheet: &str,
16912 sr: u32,
16913 sc: u32,
16914 er: u32,
16915 ec: u32,
16916 ) -> RangeView<'_> {
16917 let Some(asheet) = self.sheet_store().sheet(sheet) else {
16918 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
16919 };
16920 if er < sr || ec < sc {
16921 return asheet.range_view(1, 1, 0, 0);
16922 }
16923 let sr0 = sr.saturating_sub(1) as usize;
16924 let sc0 = sc.saturating_sub(1) as usize;
16925 let er0 = er.saturating_sub(1) as usize;
16926 let ec0 = ec.saturating_sub(1) as usize;
16927 asheet.range_view(sr0, sc0, er0, ec0)
16928 }
16929
16930 pub(crate) fn formula_plane_span_ast_at(
16931 &self,
16932 span_ref: FormulaSpanRef,
16933 placement: PlacementCoord,
16934 ) -> Option<ASTNode> {
16935 let authority = self.graph.formula_authority();
16936 let span = authority.plane.spans.get(span_ref)?;
16937 let relocation = span.ast_relocation;
16938 let mut ast = self
16939 .graph
16940 .data_store()
16941 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
16942 if let Some(binding_set_id) = span.binding_set_id {
16943 let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
16944 if !binding_set.is_single_literal_binding() {
16945 let binding =
16946 binding_set.literal_bindings_for_placement(&span.domain, placement)?;
16947 ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
16948 }
16949 }
16950 let row = placement.row.checked_add(1)?;
16951 let col = placement.col.checked_add(1)?;
16952 let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
16953 let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
16954 relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
16955 }
16956
16957 pub fn get_cell(
16959 &self,
16960 sheet: &str,
16961 row: u32,
16962 col: u32,
16963 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
16964 let v = self.get_cell_value(sheet, row, col);
16965 let sheet_id = self.graph.sheet_id(sheet)?;
16966 let coord = Coord::from_excel(row, col, true, true);
16967 let cell = CellRef::new(sheet_id, coord);
16968 let placement =
16969 crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
16970 let handle = self
16971 .graph
16972 .formula_authority()
16973 .plane
16974 .resolve_formula_at(placement, None);
16975 match handle.resolution {
16976 crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
16977 let ast = self.formula_plane_span_ast_at(span, placement);
16981 return Some((ast, v));
16982 }
16983 crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
16984 let ast = self
16985 .graph
16986 .formula_authority()
16987 .plane
16988 .formula_overlay
16989 .get(overlay_ref)
16990 .and_then(|overlay| match overlay.kind {
16991 crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
16992 template_id,
16993 ) => self
16994 .graph
16995 .formula_authority()
16996 .plane
16997 .templates
16998 .get(template_id)
16999 .and_then(|template| {
17000 self.graph
17001 .data_store()
17002 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
17003 }),
17004 _ => None,
17005 });
17006 return Some((ast, v));
17007 }
17008 _ => {}
17009 }
17010
17011 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
17012 let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
17013 self.graph
17014 .data_store()
17015 .retrieve_ast(ast_id, self.graph.sheet_reg())
17016 });
17017 Some((ast, v))
17018 } else if v.is_some() {
17019 Some((None, v))
17020 } else {
17021 None
17022 }
17023 }
17024
17025 pub fn begin_batch(&mut self) {
17027 self.graph.begin_batch();
17028 }
17029
17030 pub fn end_batch(&mut self) {
17032 self.graph.end_batch();
17033 }
17034
17035 pub fn begin_deferred_dirty(&mut self) {
17044 self.graph.begin_deferred_dirty();
17045 }
17046
17047 pub fn end_deferred_dirty(&mut self) {
17050 let _ = self.graph.end_deferred_dirty();
17051 }
17052
17053 pub fn dirty_propagation_visits(&self) -> u64 {
17057 self.graph.dirty_propagation_visits()
17058 }
17059
17060 #[inline]
17063 fn record_cell_if_changed(
17064 delta: &mut DeltaCollector,
17065 cell: &CellRef,
17066 old: &LiteralValue,
17067 new: &LiteralValue,
17068 ) {
17069 if old != new {
17070 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17071 }
17072 }
17073
17074 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17075 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
17076 engine.observe_function_semantic_epoch()?;
17077 engine.resource_checkpoint(1)?;
17079 let is_formula = engine.graph.vertex_exists(vertex_id)
17080 && matches!(
17081 engine.graph.get_vertex_kind(vertex_id),
17082 VertexKind::FormulaScalar | VertexKind::FormulaArray
17083 );
17084 if is_formula {
17085 engine.begin_evaluation_request();
17086 engine.graph.flush_pending_edge_deltas();
17087 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
17088 vertex_id,
17089 )];
17090 if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17091 && engine.graph.formula_authority().active_span_count() > 0
17092 {
17093 engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
17094 } else {
17095 engine.evaluate_legacy_target_roots(&roots, None)?;
17096 }
17097 } else if engine.config.formula_plane_mode
17098 == FormulaPlaneMode::AuthoritativeExperimental
17099 && engine.graph.formula_authority().active_span_count() > 0
17100 {
17101 engine.begin_evaluation_request();
17102 engine.graph.flush_pending_edge_deltas();
17103 engine.evaluate_authoritative_formula_plane(None, None)?;
17104 }
17105 engine.evaluate_vertex_impl(vertex_id, None)
17106 })
17107 }
17108
17109 fn evaluate_vertex_impl(
17110 &mut self,
17111 vertex_id: VertexId,
17112 delta: Option<&mut DeltaCollector>,
17113 ) -> Result<LiteralValue, ExcelError> {
17114 if !self.graph.vertex_exists(vertex_id) {
17118 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17119 .with_message(format!("Vertex not found: {vertex_id:?}")));
17120 }
17121 if self.active_resource_ledger.is_some()
17122 && matches!(
17123 self.graph.get_vertex_kind(vertex_id),
17124 VertexKind::FormulaScalar | VertexKind::FormulaArray
17125 )
17126 {
17127 let value = self
17128 .evaluate_vertex_immutable(vertex_id)
17129 .unwrap_or_else(LiteralValue::Error);
17130 let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
17131 self.resource_checkpoint(0)?;
17133 let mut delta = delta;
17134 for effect in &effects {
17135 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
17136 }
17137 return Ok(value);
17138 }
17139
17140 let mut delta = delta;
17141
17142 let kind = self.graph.get_vertex_kind(vertex_id);
17144 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17145
17146 let ast_id = match kind {
17147 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17148 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
17149 ast_id
17150 } else {
17151 return Ok(LiteralValue::Number(0.0));
17152 }
17153 }
17154 VertexKind::Empty | VertexKind::Cell => {
17155 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17156 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17157 let row = cell_ref.coord.row() + 1;
17158 let col = cell_ref.coord.col() + 1;
17159 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17160 return Ok(v);
17161 }
17162 }
17163 return Ok(LiteralValue::Number(0.0));
17164 }
17165 VertexKind::NamedScalar => {
17166 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
17167 return Ok(value);
17168 }
17169 VertexKind::NamedArray => {
17170 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
17171 return Ok(value);
17172 }
17173 VertexKind::InfiniteRange
17174 | VertexKind::Range
17175 | VertexKind::External
17176 | VertexKind::Table => {
17177 return Ok(LiteralValue::Number(0.0));
17179 }
17180 };
17181
17182 let sheet_name = self.graph.sheet_name(sheet_id);
17184 let cell_ref = self
17185 .graph
17186 .get_cell_ref(vertex_id)
17187 .expect("cell ref for vertex");
17188 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17189
17190 let result =
17191 interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
17192
17193 match result {
17195 Ok(cv) => {
17196 let result_literal =
17197 crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
17198 match result_literal {
17199 LiteralValue::Array(rows) => {
17200 self.graph
17202 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
17203 let anchor = self
17205 .graph
17206 .get_cell_ref(vertex_id)
17207 .expect("cell ref for vertex");
17208 let sheet_id = anchor.sheet_id;
17209 let h = rows.len() as u32;
17210 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
17211
17212 let spill_cells = (h as u64).saturating_mul(w as u64);
17214 if spill_cells > self.config.spill.max_spill_cells as u64 {
17215 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17216 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17217 .with_message("SpillTooLarge")
17218 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17219 expected_rows: h,
17220 expected_cols: w,
17221 });
17222 let spill_val = LiteralValue::Error(spill_err.clone());
17223 if let Some(d) = delta.as_deref_mut() {
17224 let old = self
17225 .read_cell_value(
17226 self.graph.sheet_name(anchor.sheet_id),
17227 anchor.coord.row() + 1,
17228 anchor.coord.col() + 1,
17229 )
17230 .unwrap_or(LiteralValue::Empty);
17231 if old != spill_val {
17232 d.record_cell(
17233 anchor.sheet_id,
17234 anchor.coord.row(),
17235 anchor.coord.col(),
17236 );
17237 }
17238 }
17239 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17240 if self.config.arrow_storage_enabled
17241 && self.config.delta_overlay_enabled
17242 && self.config.write_formula_overlay_enabled
17243 {
17244 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17245 self.mirror_value_to_computed_overlay(
17246 &sheet_name,
17247 anchor.coord.row() + 1,
17248 anchor.coord.col() + 1,
17249 &spill_val,
17250 );
17251 }
17252 return Ok(spill_val);
17253 }
17254 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
17258 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
17259 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
17260 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17261 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17262 .with_message("Spill exceeds sheet bounds")
17263 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17264 expected_rows: h,
17265 expected_cols: w,
17266 });
17267 let spill_val = LiteralValue::Error(spill_err.clone());
17268 if let Some(d) = delta.as_deref_mut() {
17269 let old = self
17270 .read_cell_value(
17271 self.graph.sheet_name(anchor.sheet_id),
17272 anchor.coord.row() + 1,
17273 anchor.coord.col() + 1,
17274 )
17275 .unwrap_or(LiteralValue::Empty);
17276 if old != spill_val {
17277 d.record_cell(
17278 anchor.sheet_id,
17279 anchor.coord.row(),
17280 anchor.coord.col(),
17281 );
17282 }
17283 }
17284 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17285 if self.config.arrow_storage_enabled
17286 && self.config.delta_overlay_enabled
17287 && self.config.write_formula_overlay_enabled
17288 {
17289 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17290 self.mirror_value_to_computed_overlay(
17291 &sheet_name,
17292 anchor.coord.row() + 1,
17293 anchor.coord.col() + 1,
17294 &spill_val,
17295 );
17296 }
17297 return Ok(spill_val);
17298 }
17299 let mut targets = Vec::new();
17300 for r in 0..h {
17301 for c in 0..w {
17302 targets.push(self.graph.make_cell_ref_internal(
17303 sheet_id,
17304 anchor.coord.row() + r,
17305 anchor.coord.col() + c,
17306 ));
17307 }
17308 }
17309
17310 match self.spill_mgr.reserve(
17312 vertex_id,
17313 anchor,
17314 SpillShape { rows: h, cols: w },
17315 SpillMeta {
17316 epoch: self.recalc_epoch,
17317 config: self.config.spill,
17318 },
17319 ) {
17320 Ok(()) => {
17321 if let Err(e) = self.commit_spill_and_mirror(
17325 vertex_id,
17326 &targets,
17327 rows.clone(),
17328 delta.as_deref_mut(),
17329 None,
17330 ) {
17331 self.clear_spill_projection_and_mirror(
17333 vertex_id,
17334 delta.as_deref_mut(),
17335 );
17336 if let Some(d) = delta.as_deref_mut() {
17337 let old = self
17338 .read_cell_value(
17339 self.graph.sheet_name(anchor.sheet_id),
17340 anchor.coord.row() + 1,
17341 anchor.coord.col() + 1,
17342 )
17343 .unwrap_or(LiteralValue::Empty);
17344 let new = LiteralValue::Error(e.clone());
17345 if old != new {
17346 d.record_cell(
17347 anchor.sheet_id,
17348 anchor.coord.row(),
17349 anchor.coord.col(),
17350 );
17351 }
17352 }
17353 let err_val = LiteralValue::Error(e.clone());
17354 self.graph.update_vertex_value(vertex_id, err_val.clone());
17355 if self.config.arrow_storage_enabled
17356 && self.config.delta_overlay_enabled
17357 && self.config.write_formula_overlay_enabled
17358 {
17359 let sheet_name =
17360 self.graph.sheet_name(anchor.sheet_id).to_string();
17361 self.mirror_value_to_computed_overlay(
17362 &sheet_name,
17363 anchor.coord.row() + 1,
17364 anchor.coord.col() + 1,
17365 &err_val,
17366 );
17367 }
17368 return Ok(err_val);
17369 }
17370 let top_left = rows
17372 .first()
17373 .and_then(|r| r.first())
17374 .cloned()
17375 .unwrap_or(LiteralValue::Empty);
17376 self.graph.update_vertex_value(vertex_id, top_left.clone());
17377 Ok(top_left)
17378 }
17379 Err(e) => {
17380 self.clear_spill_projection_and_mirror(
17381 vertex_id,
17382 delta.as_deref_mut(),
17383 );
17384 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17385 .with_message(
17386 e.message.unwrap_or_else(|| "Spill blocked".to_string()),
17387 )
17388 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17389 expected_rows: h,
17390 expected_cols: w,
17391 });
17392 let spill_val = LiteralValue::Error(spill_err.clone());
17393 if let Some(d) = delta.as_deref_mut() {
17394 let old = self
17395 .read_cell_value(
17396 self.graph.sheet_name(anchor.sheet_id),
17397 anchor.coord.row() + 1,
17398 anchor.coord.col() + 1,
17399 )
17400 .unwrap_or(LiteralValue::Empty);
17401 if old != spill_val {
17402 d.record_cell(
17403 anchor.sheet_id,
17404 anchor.coord.row(),
17405 anchor.coord.col(),
17406 );
17407 }
17408 }
17409 self.graph.update_vertex_value(vertex_id, spill_val.clone());
17410 if self.config.arrow_storage_enabled
17411 && self.config.delta_overlay_enabled
17412 && self.config.write_formula_overlay_enabled
17413 {
17414 let sheet_name =
17415 self.graph.sheet_name(anchor.sheet_id).to_string();
17416 self.mirror_value_to_computed_overlay(
17417 &sheet_name,
17418 anchor.coord.row() + 1,
17419 anchor.coord.col() + 1,
17420 &spill_val,
17421 );
17422 }
17423 Ok(spill_val)
17424 }
17425 }
17426 }
17427 other => {
17428 let spill_cells = self
17430 .graph
17431 .spill_cells_for_anchor(vertex_id)
17432 .map(|cells| cells.to_vec())
17433 .unwrap_or_default();
17434 if let Some(d) = delta.as_deref_mut()
17435 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17436 {
17437 if spill_cells.is_empty() {
17438 let old = self
17439 .read_cell_value(
17440 self.graph.sheet_name(anchor.sheet_id),
17441 anchor.coord.row() + 1,
17442 anchor.coord.col() + 1,
17443 )
17444 .unwrap_or(LiteralValue::Empty);
17445 if old != other {
17446 d.record_cell(
17447 anchor.sheet_id,
17448 anchor.coord.row(),
17449 anchor.coord.col(),
17450 );
17451 }
17452 } else {
17453 for cell in spill_cells.iter() {
17454 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17455 let old = self
17456 .get_cell_value(
17457 sheet_name,
17458 cell.coord.row() + 1,
17459 cell.coord.col() + 1,
17460 )
17461 .unwrap_or(LiteralValue::Empty);
17462 let new = if cell.sheet_id == anchor.sheet_id
17463 && cell.coord.row() == anchor.coord.row()
17464 && cell.coord.col() == anchor.coord.col()
17465 {
17466 other.clone()
17467 } else {
17468 LiteralValue::Empty
17469 };
17470 Self::record_cell_if_changed(d, cell, &old, &new);
17471 }
17472 }
17473 }
17474 self.graph.clear_spill_region(vertex_id);
17475 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17476 self.record_formula_plane_structural_change(scope);
17477 }
17478 if self.config.arrow_storage_enabled
17479 && self.config.delta_overlay_enabled
17480 && self.config.write_formula_overlay_enabled
17481 {
17482 let empty = LiteralValue::Empty;
17483 for cell in spill_cells.iter() {
17484 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17485 self.mirror_value_to_computed_overlay(
17486 &sheet_name,
17487 cell.coord.row() + 1,
17488 cell.coord.col() + 1,
17489 &empty,
17490 );
17491 }
17492 }
17493 self.graph.update_vertex_value(vertex_id, other.clone());
17494 if self.config.arrow_storage_enabled
17496 && self.config.delta_overlay_enabled
17497 && self.config.write_formula_overlay_enabled
17498 {
17499 let anchor = self
17500 .graph
17501 .get_cell_ref(vertex_id)
17502 .expect("cell ref for vertex");
17503 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17504 self.mirror_value_to_computed_overlay(
17505 &sheet_name,
17506 anchor.coord.row() + 1,
17507 anchor.coord.col() + 1,
17508 &other,
17509 );
17510 }
17511 Ok(other)
17512 }
17513 }
17514 }
17515 Err(e) => {
17516 let spill_cells = self
17519 .graph
17520 .spill_cells_for_anchor(vertex_id)
17521 .map(|cells| cells.to_vec())
17522 .unwrap_or_default();
17523 let err_val = LiteralValue::Error(e.clone());
17524 if let Some(d) = delta
17525 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17526 {
17527 if spill_cells.is_empty() {
17528 let old = self
17529 .read_cell_value(
17530 self.graph.sheet_name(anchor.sheet_id),
17531 anchor.coord.row() + 1,
17532 anchor.coord.col() + 1,
17533 )
17534 .unwrap_or(LiteralValue::Empty);
17535 if old != err_val {
17536 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
17537 }
17538 } else {
17539 for cell in spill_cells.iter() {
17540 let sheet_name = self.graph.sheet_name(cell.sheet_id);
17541 let old = self
17542 .get_cell_value(
17543 sheet_name,
17544 cell.coord.row() + 1,
17545 cell.coord.col() + 1,
17546 )
17547 .unwrap_or(LiteralValue::Empty);
17548 let new = if cell.sheet_id == anchor.sheet_id
17549 && cell.coord.row() == anchor.coord.row()
17550 && cell.coord.col() == anchor.coord.col()
17551 {
17552 err_val.clone()
17553 } else {
17554 LiteralValue::Empty
17555 };
17556 Self::record_cell_if_changed(d, cell, &old, &new);
17557 }
17558 }
17559 }
17560 self.graph.clear_spill_region(vertex_id);
17561 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17562 self.record_formula_plane_structural_change(scope);
17563 }
17564 if self.config.arrow_storage_enabled
17565 && self.config.delta_overlay_enabled
17566 && self.config.write_formula_overlay_enabled
17567 {
17568 let empty = LiteralValue::Empty;
17569 for cell in spill_cells.iter() {
17570 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17571 self.mirror_value_to_computed_overlay(
17572 &sheet_name,
17573 cell.coord.row() + 1,
17574 cell.coord.col() + 1,
17575 &empty,
17576 );
17577 }
17578 }
17579 self.graph.update_vertex_value(vertex_id, err_val.clone());
17580 if self.config.arrow_storage_enabled
17581 && self.config.delta_overlay_enabled
17582 && self.config.write_formula_overlay_enabled
17583 {
17584 let anchor = self
17585 .graph
17586 .get_cell_ref(vertex_id)
17587 .expect("cell ref for vertex");
17588 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17589 self.mirror_value_to_computed_overlay(
17590 &sheet_name,
17591 anchor.coord.row() + 1,
17592 anchor.coord.col() + 1,
17593 &err_val,
17594 );
17595 }
17596 Ok(err_val)
17597 }
17598 }
17599 }
17600
17601 fn evaluate_named_scalar(
17602 &mut self,
17603 vertex_id: VertexId,
17604 sheet_id: SheetId,
17605 ) -> Result<LiteralValue, ExcelError> {
17606 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17607 ExcelError::new(ExcelErrorKind::Name)
17608 .with_message("Named range metadata missing".to_string())
17609 })?;
17610
17611 match &named_range.definition {
17612 NamedDefinition::Cell(cell_ref) => {
17613 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17614 let row = cell_ref.coord.row() + 1;
17615 let col = cell_ref.coord.col() + 1;
17616
17617 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
17618 && matches!(
17619 self.graph.get_vertex_kind(dep_vertex),
17620 VertexKind::FormulaScalar | VertexKind::FormulaArray
17621 )
17622 {
17623 let value = self.evaluate_vertex(dep_vertex)?;
17625 self.graph.update_vertex_value(vertex_id, value.clone());
17626 Ok(value)
17627 } else {
17628 let value = self
17629 .get_cell_value(sheet_name, row, col)
17630 .unwrap_or(LiteralValue::Empty);
17631 self.graph.update_vertex_value(vertex_id, value.clone());
17632 Ok(value)
17633 }
17634 }
17635 NamedDefinition::Literal(v) => {
17636 let out = v.clone();
17637 self.graph.update_vertex_value(vertex_id, out.clone());
17638 Ok(out)
17639 }
17640 NamedDefinition::Formula { ast, .. } => {
17641 let context_sheet = match named_range.scope {
17642 NameScope::Sheet(id) => id,
17643 NameScope::Workbook => sheet_id,
17644 };
17645 let sheet_name = self.graph.sheet_name(context_sheet);
17646 let cell_ref = self
17647 .graph
17648 .get_cell_ref(vertex_id)
17649 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17650 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17651 match interpreter.evaluate_ast(ast) {
17652 Ok(cv) => {
17653 let value = cv.into_literal();
17654 match value {
17655 LiteralValue::Array(_) => {
17656 let err = ExcelError::new(ExcelErrorKind::NImpl)
17657 .with_message("Array result in scalar named range".to_string());
17658 let err_val = LiteralValue::Error(err.clone());
17659 self.graph.update_vertex_value(vertex_id, err_val.clone());
17660 Ok(err_val)
17661 }
17662 other => {
17663 self.graph.update_vertex_value(vertex_id, other.clone());
17664 Ok(other)
17665 }
17666 }
17667 }
17668 Err(err) => {
17669 let err_val = LiteralValue::Error(err.clone());
17670 self.graph.update_vertex_value(vertex_id, err_val.clone());
17671 Ok(err_val)
17672 }
17673 }
17674 }
17675 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17676 .with_message("Range-valued name evaluated as scalar".to_string())),
17677 }
17678 }
17679
17680 fn evaluate_named_array(
17681 &mut self,
17682 vertex_id: VertexId,
17683 sheet_id: SheetId,
17684 ) -> Result<LiteralValue, ExcelError> {
17685 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17686 ExcelError::new(ExcelErrorKind::Name)
17687 .with_message("Named range metadata missing".to_string())
17688 })?;
17689
17690 let out = match &named_range.definition {
17691 NamedDefinition::Range(range_ref) => {
17692 if range_ref.start.sheet_id != range_ref.end.sheet_id {
17693 return Err(ExcelError::new(ExcelErrorKind::Ref)
17694 .with_message("Named range cannot span sheets".to_string()));
17695 }
17696
17697 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17698 let sr0 = range_ref.start.coord.row();
17699 let sc0 = range_ref.start.coord.col();
17700 let er0 = range_ref.end.coord.row();
17701 let ec0 = range_ref.end.coord.col();
17702 if sr0 > er0 || sc0 > ec0 {
17703 return Err(ExcelError::new(ExcelErrorKind::Ref)
17704 .with_message("Invalid named range bounds".to_string()));
17705 }
17706
17707 let h = (er0 - sr0 + 1) as usize;
17708 let w = (ec0 - sc0 + 1) as usize;
17709 let cell_count = (h as u64).saturating_mul(w as u64);
17710 if cell_count > self.config.spill.max_spill_cells as u64 {
17711 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17712 "Named range too large to materialize as an array".to_string(),
17713 ));
17714 }
17715
17716 let mut rows = Vec::with_capacity(h);
17717 for r0 in sr0..=er0 {
17718 let mut row = Vec::with_capacity(w);
17719 for c0 in sc0..=ec0 {
17720 let v = self
17721 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
17722 .unwrap_or(LiteralValue::Empty);
17723 row.push(v);
17724 }
17725 rows.push(row);
17726 }
17727 LiteralValue::Array(rows)
17728 }
17729 NamedDefinition::Cell(cell_ref) => {
17730 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17731 let row = cell_ref.coord.row() + 1;
17732 let col = cell_ref.coord.col() + 1;
17733 let v = self
17734 .get_cell_value(sheet_name, row, col)
17735 .unwrap_or(LiteralValue::Empty);
17736 LiteralValue::Array(vec![vec![v]])
17737 }
17738 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
17739 NamedDefinition::Formula { ast, .. } => {
17740 let context_sheet = match named_range.scope {
17741 NameScope::Sheet(id) => id,
17742 NameScope::Workbook => sheet_id,
17743 };
17744 let sheet_name = self.graph.sheet_name(context_sheet);
17745 let cell_ref = self
17746 .graph
17747 .get_cell_ref(vertex_id)
17748 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17749 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17750 match interpreter.evaluate_ast(ast) {
17751 Ok(cv) => {
17752 let v = cv.into_literal();
17753 match v {
17754 LiteralValue::Array(_) => v,
17755 other => LiteralValue::Array(vec![vec![other]]),
17756 }
17757 }
17758 Err(err) => LiteralValue::Error(err),
17759 }
17760 }
17761 };
17762
17763 self.graph.update_vertex_value(vertex_id, out.clone());
17764 Ok(out)
17765 }
17766
17767 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
17768 crate::engine::ResourceLedgerError::Exhausted(
17769 formualizer_common::ResourceExhaustionDetail {
17770 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
17771 limit: limit as u64,
17772 observed: limit.saturating_add(1) as u64,
17773 request_id: self
17774 .active_evaluation_resource_request
17775 .as_ref()
17776 .map(|stats| stats.request_id),
17777 },
17778 )
17779 .into_excel_error()
17780 .with_message(format!("{context} did not converge after {limit} replans"))
17781 }
17782
17783 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
17784 matches!(
17785 &error.extra,
17786 formualizer_common::ExcelErrorExtra::PreparationStale {
17787 reason: formualizer_common::PreparationStaleReason::Semantic
17788 | formualizer_common::PreparationStaleReason::Provider
17789 }
17790 )
17791 }
17792
17793 fn prepare_graph_for_routed_evaluation(
17794 &mut self,
17795 targets: &[crate::engine::EvaluationTarget],
17796 options: &crate::engine::TargetEvalOptions<'_>,
17797 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
17798 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
17799 let mut retries = 0usize;
17800 loop {
17801 match self.prepare_graph_for_targets_unobserved(targets, options) {
17802 Err(error)
17803 if Self::transient_target_preparation_stale(&error)
17804 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
17805 {
17806 retries = retries.saturating_add(1);
17807 }
17808 result => return result,
17809 }
17810 }
17811 }
17812
17813 fn evaluate_mixed_targets(
17814 &mut self,
17815 targets: &[crate::engine::EvaluationTarget],
17816 delta: Option<&mut DeltaCollector>,
17817 ) -> Result<EvalResult, ExcelError> {
17818 let _source_cache = self.source_cache_session();
17819 let cancel = self.active_cancel_flag.clone();
17820 let options = crate::engine::TargetEvalOptions {
17821 request_id: self
17822 .active_evaluation_resource_request
17823 .as_ref()
17824 .map(|stats| stats.request_id),
17825 cancel,
17826 deadline: None,
17827 budgets: None,
17828 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
17829 };
17830 self.prepare_and_execute_target_recipe(targets, &options, delta)
17831 }
17832
17833 fn prepare_and_execute_target_recipe(
17834 &mut self,
17835 targets: &[crate::engine::EvaluationTarget],
17836 options: &crate::engine::TargetEvalOptions<'_>,
17837 delta: Option<&mut DeltaCollector>,
17838 ) -> Result<EvalResult, ExcelError> {
17839 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
17840 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
17841 }
17842
17843 fn execute_prepared_target_recipe(
17844 &mut self,
17845 targets: &[crate::engine::EvaluationTarget],
17846 scope: &crate::engine::PrepareScope,
17847 delta: Option<&mut DeltaCollector>,
17848 ) -> Result<EvalResult, ExcelError> {
17849 if matches!(scope, crate::engine::PrepareScope::Workbook)
17850 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
17851 {
17852 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
17853 }
17854 let mut roots = self.resolve_target_producers(targets)?;
17855 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
17856 let request_id = self
17857 .active_evaluation_resource_request
17858 .as_ref()
17859 .map(|request| request.request_id);
17860 let root_count = roots.len();
17861 let mut widened_roots =
17862 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
17863 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
17864 let sheet_ids = sheets
17865 .iter()
17866 .filter_map(|sheet| self.graph.sheet_id(sheet))
17867 .collect::<FxHashSet<_>>();
17868 for vertex in self.graph.formula_vertices() {
17869 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
17870 widened_roots
17871 .push(crate::engine::target_preparation::TargetProducer::Legacy(
17872 vertex,
17873 ))
17874 .map_err(|_| {
17875 target_root_allocation_error(widened_roots.len() + 1, request_id)
17876 })?;
17877 }
17878 }
17879 let authority = self.graph.formula_authority();
17880 for span_ref in authority.active_span_refs() {
17881 let Some(span) = authority.plane.spans.get(span_ref) else {
17882 continue;
17883 };
17884 if sheet_ids.contains(&span.sheet_id) {
17885 widened_roots
17886 .push(crate::engine::target_preparation::TargetProducer::Span {
17887 span_ref,
17888 demanded: Region::from_domain(span.result_region.domain()),
17889 })
17890 .map_err(|_| {
17891 target_root_allocation_error(widened_roots.len() + 1, request_id)
17892 })?;
17893 }
17894 }
17895 roots = widened_roots.into_vec();
17896 }
17897 self.begin_evaluation_request();
17898 self.graph.flush_pending_edge_deltas();
17899 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
17900 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17901 && self.graph.formula_authority().active_span_count() > 0
17902 {
17903 if workbook_scope {
17904 self.evaluate_authoritative_formula_plane(None, delta)
17905 } else {
17906 self.evaluate_authoritative_formula_plane_targets(&roots, delta)
17907 }
17908 } else {
17909 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
17910 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
17911 }
17912 if workbook_scope {
17913 if let Some(delta) = delta {
17914 self.evaluate_all_with_delta_collector(delta)
17915 } else {
17916 self.evaluate_all_legacy_impl()
17917 }
17918 } else {
17919 self.evaluate_legacy_target_roots(&roots, delta)
17920 }
17921 }
17922 }
17923
17924 fn legacy_coordinate_targets(
17925 &mut self,
17926 targets: &[(&str, u32, u32)],
17927 ) -> Vec<crate::engine::EvaluationTarget> {
17928 targets
17929 .iter()
17930 .map(|(sheet, row, col)| {
17931 self.graph.sheet_id_mut(sheet);
17934 crate::engine::EvaluationTarget::Cell {
17935 sheet: (*sheet).to_string(),
17936 row: *row,
17937 col: *col,
17938 }
17939 })
17940 .collect()
17941 }
17942
17943 pub fn evaluate_targets(
17945 &mut self,
17946 targets: &[crate::engine::EvaluationTarget],
17947 ) -> Result<EvalResult, ExcelError> {
17948 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17949 engine.observe_function_semantic_epoch()?;
17950 engine.validate_deterministic_mode()?;
17951 engine.evaluate_mixed_targets(targets, None)
17952 })
17953 }
17954
17955 pub fn evaluate_targets_with_options(
17957 &mut self,
17958 targets: &[crate::engine::EvaluationTarget],
17959 options: crate::engine::TargetEvalOptions<'_>,
17960 ) -> Result<EvalResult, ExcelError> {
17961 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17962 engine.active_cancel_flag = options.cancel.clone();
17963 engine.active_evaluation_deadline = options.deadline;
17964 let result = (|| {
17965 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
17966 engine.observe_function_semantic_epoch()?;
17967 engine.validate_deterministic_mode()?;
17968 let _source_cache = engine.source_cache_session();
17969 engine.prepare_and_execute_target_recipe(targets, &options, None)
17970 })();
17971 engine.active_cancel_flag = None;
17972 engine.active_evaluation_deadline = None;
17973 result
17974 })
17975 }
17976
17977 pub fn evaluate_targets_with_delta(
17979 &mut self,
17980 targets: &[crate::engine::EvaluationTarget],
17981 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
17982 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
17983 engine.observe_function_semantic_epoch()?;
17984 engine.validate_deterministic_mode()?;
17985 let mut collector = DeltaCollector::new(DeltaMode::Cells);
17986 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
17987 Ok((result, collector.finish_target()))
17988 })
17989 }
17990
17991 pub fn evaluate_until(
17993 &mut self,
17994 targets: &[(&str, u32, u32)],
17995 ) -> Result<EvalResult, ExcelError> {
17996 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17997 engine.evaluate_until_unobserved(targets)
17998 })
17999 }
18000
18001 fn evaluate_until_unobserved(
18002 &mut self,
18003 targets: &[(&str, u32, u32)],
18004 ) -> Result<EvalResult, ExcelError> {
18005 self.observe_function_semantic_epoch()?;
18006 let targets = self.legacy_coordinate_targets(targets);
18007 self.evaluate_mixed_targets(&targets, None)
18008 }
18009
18010 fn evaluate_until_with_delta_collector(
18011 &mut self,
18012 targets: &[(&str, u32, u32)],
18013 delta: &mut DeltaCollector,
18014 ) -> Result<EvalResult, ExcelError> {
18015 let targets = self.legacy_coordinate_targets(targets);
18016 self.evaluate_mixed_targets(&targets, Some(delta))
18017 }
18018
18019 fn evaluate_legacy_target_roots(
18020 &mut self,
18021 roots: &[crate::engine::target_preparation::TargetProducer],
18022 mut delta: Option<&mut DeltaCollector>,
18023 ) -> Result<EvalResult, ExcelError> {
18024 use crate::engine::target_preparation::TargetProducer;
18025 let start = crate::instant::FzInstant::now();
18026 let root_vertices = roots
18027 .iter()
18028 .filter_map(|root| match root {
18029 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
18030 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
18031 })
18032 .collect::<Vec<_>>();
18033 let mut computed_vertices = 0usize;
18034 let mut cycle_errors = 0usize;
18035 let mut replans = 0usize;
18036 const MAX_REPLAN: usize = 5;
18037 loop {
18038 let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
18039 if precedents_to_eval.is_empty() {
18040 break;
18041 }
18042 let scheduler = Scheduler::new(&self.graph);
18043 let schedule =
18044 scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
18045 for &unit in &schedule.units {
18046 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
18047 match unit {
18048 ScheduleUnit::Cycle(index) => {
18049 if self.handle_cycle_unit(
18050 schedule.unit_cycle(index),
18051 delta.as_deref_mut(),
18052 None,
18053 None,
18054 )? > 0
18055 {
18056 cycle_errors = cycle_errors.saturating_add(1);
18057 }
18058 }
18059 ScheduleUnit::Layer(index) => {
18060 let layer = schedule.unit_layer(index);
18061 let evaluated = if let Some(delta) = delta.as_deref_mut() {
18062 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18063 self.evaluate_layer_parallel_with_delta(layer, delta)?
18064 } else {
18065 self.evaluate_layer_sequential_with_delta(layer, delta)?
18066 }
18067 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18068 self.evaluate_layer_parallel(layer)?
18069 } else {
18070 self.evaluate_layer_sequential(layer)?
18071 };
18072 computed_vertices = computed_vertices.saturating_add(evaluated);
18073 }
18074 }
18075 }
18076 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
18077 self.resource_checkpoint(0)?;
18078 self.graph.clear_dirty_flags(&precedents_to_eval);
18079 for vertex in &changed {
18080 self.graph.set_dirty(*vertex, true);
18081 }
18082 if changed.is_empty() {
18083 break;
18084 }
18085 if replans >= MAX_REPLAN {
18086 return Err(self.replan_exhausted_error(
18087 MAX_REPLAN,
18088 "targeted legacy dynamic dependency evaluation",
18089 ));
18090 }
18091 replans = replans.saturating_add(1);
18092 }
18093 self.redirty_for_next_recalc();
18094 Ok(EvalResult {
18095 computed_vertices,
18096 cycle_errors,
18097 elapsed: start.elapsed(),
18098 })
18099 }
18100
18101 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
18103 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
18104 return Err(
18105 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
18106 "compatibility recalculation plans require all staged formulas to be prepared",
18107 ),
18108 );
18109 }
18110 let key = self.recalc_plan_key();
18111 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
18112 vertices.sort_unstable();
18113 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
18114 let schedule = if vertices.is_empty() {
18115 crate::engine::Schedule {
18116 units: Vec::new(),
18117 layers: Vec::new(),
18118 cycles: Vec::new(),
18119 }
18120 } else {
18121 self.create_evaluation_schedule_uncached(&vertices)?.0
18122 };
18123 self.validate_recalc_plan_key(&key)?;
18124 Ok(RecalcPlan {
18125 key,
18126 kind: RecalcPlanKind::CompatibilityFull {
18127 schedule,
18128 has_dynamic_refs,
18129 },
18130 })
18131 }
18132
18133 pub fn build_recalc_plan_for_targets(
18135 &mut self,
18136 targets: &[crate::engine::EvaluationTarget],
18137 ) -> Result<RecalcPlan, ExcelError> {
18138 self.build_recalc_plan_for_targets_with_options(
18139 targets,
18140 crate::engine::TargetEvalOptions::default(),
18141 )
18142 }
18143
18144 pub fn build_recalc_plan_for_targets_with_options(
18145 &mut self,
18146 targets: &[crate::engine::EvaluationTarget],
18147 options: crate::engine::TargetEvalOptions<'_>,
18148 ) -> Result<RecalcPlan, ExcelError> {
18149 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18150 engine.observe_function_semantic_epoch()?;
18151 engine.validate_deterministic_mode()?;
18152 let _source_cache = engine.source_cache_session();
18153 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
18154 engine.graph.flush_pending_edge_deltas();
18155 let topology = if matches!(
18156 preparation.widened_scope,
18157 crate::engine::PrepareScope::Workbook
18158 ) {
18159 RecalcTopology::Workbook
18160 } else {
18161 RecalcTopology::RunLocalRecipe
18162 };
18163 Ok(RecalcPlan {
18164 key: engine.recalc_plan_key(),
18165 kind: RecalcPlanKind::Target {
18166 targets: targets.to_vec(),
18167 scope: preparation.widened_scope,
18168 topology,
18169 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
18170 },
18171 })
18172 })
18173 }
18174
18175 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
18177 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18178 engine.evaluate_recalc_plan_unobserved(plan)
18179 })
18180 }
18181
18182 pub fn evaluate_recalc_plan_with_controls(
18183 &mut self,
18184 plan: &RecalcPlan,
18185 cancel: Option<crate::engine::CancelToken>,
18186 deadline: Option<Instant>,
18187 ) -> Result<EvalResult, ExcelError> {
18188 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18189 engine.active_cancel_flag = cancel.clone();
18190 engine.active_evaluation_deadline = deadline;
18191 let result = engine.evaluate_recalc_plan_unobserved(plan);
18192 engine.active_cancel_flag = None;
18193 engine.active_evaluation_deadline = None;
18194 result
18195 })
18196 }
18197
18198 fn evaluate_recalc_plan_unobserved(
18199 &mut self,
18200 plan: &RecalcPlan,
18201 ) -> Result<EvalResult, ExcelError> {
18202 self.graph.flush_pending_edge_deltas();
18203 self.validate_recalc_plan_key(&plan.key)?;
18204 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
18205 self.validate_deterministic_mode()?;
18206
18207 match &plan.kind {
18208 RecalcPlanKind::Target {
18209 targets,
18210 scope,
18211 topology,
18212 dynamic_policy,
18213 } => {
18214 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
18215 debug_assert_eq!(
18216 matches!(topology, RecalcTopology::Workbook),
18217 matches!(scope, crate::engine::PrepareScope::Workbook)
18218 );
18219 let _source_cache = self.source_cache_session();
18220 self.execute_prepared_target_recipe(targets, scope, None)
18221 }
18222 RecalcPlanKind::CompatibilityFull {
18223 schedule,
18224 has_dynamic_refs,
18225 } => {
18226 let _source_cache = self.source_cache_session();
18227 self.begin_evaluation_request();
18228 if self.graph.formula_authority().active_span_count() > 0 {
18229 return self.evaluate_authoritative_formula_plane_all();
18230 }
18231 if *has_dynamic_refs {
18232 self.virtual_dep_fallback_activations =
18233 self.virtual_dep_fallback_activations.saturating_add(1);
18234 return self.evaluate_all_coordinator();
18235 }
18236
18237 let start = crate::instant::FzInstant::now();
18238 let dirty_vertices = self.graph.get_evaluation_vertices();
18239 if dirty_vertices.is_empty() {
18240 return Ok(EvalResult {
18241 computed_vertices: 0,
18242 cycle_errors: 0,
18243 elapsed: start.elapsed(),
18244 });
18245 }
18246
18247 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
18248 let mut computed_vertices = 0;
18249 let mut cycle_errors = 0;
18250 for &unit in &schedule.units {
18251 self.cancellation_checkpoint(
18252 "Evaluation cancelled before recalculation plan schedule unit",
18253 )?;
18254 match unit {
18255 ScheduleUnit::Cycle(i) => {
18256 let stamped = self.handle_cycle_unit(
18257 schedule.unit_cycle(i),
18258 None,
18259 Some(&dirty_set),
18260 None,
18261 )?;
18262 if stamped > 0 {
18263 cycle_errors += 1;
18264 }
18265 }
18266 ScheduleUnit::Layer(i) => {
18267 let work: Vec<VertexId> = schedule
18268 .unit_layer(i)
18269 .vertices
18270 .iter()
18271 .copied()
18272 .filter(|v| dirty_set.contains(v))
18273 .collect();
18274 if work.is_empty() {
18275 continue;
18276 }
18277 let temp_layer = crate::engine::scheduler::Layer { vertices: work };
18278 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
18279 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
18280 } else {
18281 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
18282 }
18283 }
18284 }
18285 }
18286
18287 self.resource_checkpoint(0)?;
18288 self.graph.clear_dirty_flags(&dirty_vertices);
18289 self.redirty_for_next_recalc();
18290 Ok(EvalResult {
18291 computed_vertices,
18292 cycle_errors,
18293 elapsed: start.elapsed(),
18294 })
18295 }
18296 }
18297 }
18298 fn evaluate_all_legacy_and_ack_dirty(
18299 &mut self,
18300 formula_dirty: FormulaDirtyLease,
18301 ) -> Result<EvalResult, ExcelError> {
18302 let result = self.evaluate_all_legacy_impl()?;
18303 self.checked_ack_formula_dirty_observed(formula_dirty)?;
18304 Ok(result)
18305 }
18306
18307 fn evaluate_formula_plane_capacity_fallback(
18308 &mut self,
18309 mut formula_dirty: FormulaDirtyLease,
18310 owned_dirty_events: &[usize],
18311 selected_span_refs: &[FormulaSpanRef],
18312 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18313 delta: Option<&mut DeltaCollector>,
18314 ) -> Result<EvalResult, ExcelError> {
18315 let leased_len = formula_dirty.len();
18316 if !selected_span_refs.is_empty() {
18317 let materialization_started = crate::instant::FzInstant::now();
18318 let prepared = self
18319 .prepare_formula_span_demotion(selected_span_refs)
18320 .map_err(|error| {
18321 if let FormulaSpanDemotionError::Resource(error) = error {
18322 error
18323 } else {
18324 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18325 "FormulaPlane capacity fallback demotion preparation failed: {error}"
18326 ))
18327 }
18328 })?;
18329 let report = self
18330 .commit_prepared_formula_span_demotion(prepared)
18331 .map_err(|error| {
18332 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18333 "FormulaPlane capacity fallback demotion commit failed: {error}"
18334 ))
18335 })?;
18336 self.observe_materialization(
18337 report.placements_materialized,
18338 false,
18339 materialization_started.elapsed(),
18340 );
18341 }
18342 #[cfg(test)]
18343 {
18344 self.last_formula_plane_span_eval_report = None;
18345 }
18346 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
18347 formula_dirty = extended;
18348 }
18349 let mut acknowledged = owned_dirty_events.to_vec();
18350 acknowledged.extend(leased_len..formula_dirty.len());
18351 let result = if let Some(target_roots) = target_roots {
18352 let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
18353 self.evaluate_legacy_target_roots(&refreshed, delta)?
18354 } else if let Some(delta) = delta {
18355 self.evaluate_all_with_delta_collector(delta)?
18356 } else {
18357 self.evaluate_all_legacy_impl()?
18358 };
18359 self.formula_plane_capacity_bailouts =
18360 self.formula_plane_capacity_bailouts.saturating_add(1);
18361 if target_roots.is_some() {
18362 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
18363 } else {
18364 self.checked_ack_formula_dirty_observed(formula_dirty)?;
18365 }
18366 Ok(result)
18367 }
18368
18369 fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
18370 self.evaluate_authoritative_formula_plane(None, None)
18371 }
18372
18373 fn evaluate_authoritative_formula_plane_targets(
18374 &mut self,
18375 roots: &[crate::engine::target_preparation::TargetProducer],
18376 delta: Option<&mut DeltaCollector>,
18377 ) -> Result<EvalResult, ExcelError> {
18378 self.evaluate_authoritative_formula_plane(Some(roots), delta)
18379 }
18380
18381 fn evaluate_authoritative_formula_plane(
18382 &mut self,
18383 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18384 mut delta: Option<&mut DeltaCollector>,
18385 ) -> Result<EvalResult, ExcelError> {
18386 let mut formula_dirty = self.graph.lease_formula_dirty();
18390 self.observe_dirty_lease_acquired(formula_dirty.is_empty());
18391
18392 if self.graph.formula_authority().active_span_count() == 0 {
18395 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
18396 #[cfg(test)]
18397 {
18398 self.last_formula_plane_span_eval_report = None;
18399 }
18400 if let Some(target_roots) = target_roots {
18401 let result =
18402 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18403 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18404 return Ok(result);
18405 }
18406 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18407 }
18408
18409 if formula_dirty.is_empty() {
18412 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
18413 #[cfg(test)]
18414 {
18415 self.last_formula_plane_span_eval_report = None;
18416 }
18417 if let Some(target_roots) = target_roots {
18418 let result =
18419 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18420 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18421 return Ok(result);
18422 }
18423 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18424 }
18425
18426 let start = crate::instant::FzInstant::now();
18427 let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
18428 let mut workbook_exact_attempted = target_roots.is_none();
18429 let mut prepass_cycle_errors = 0usize;
18432 const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
18433 let mut cycle_demote_iters = 0usize;
18434 let mut include_dirty_regions = true;
18435 let mut retry_whole_spans = Vec::new();
18436 let mut computed_vertices = 0usize;
18437 let mut runtime_replan_iterations = 0usize;
18438 const MAX_RUNTIME_REPLAN: usize = 5;
18439 let mut virtual_telemetry = self
18440 .config
18441 .enable_virtual_dep_telemetry
18442 .then(|| self.start_virtual_dep_telemetry());
18443 'virtual_replan: loop {
18444 self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
18445 let (
18446 schedule,
18447 span_refs_by_id,
18448 plane_epoch,
18449 legacy_vertices,
18450 owned_dirty_events,
18451 island_plan,
18452 ) = loop {
18453 let (
18454 schedule,
18455 span_refs_by_id,
18456 plane_epoch,
18457 legacy_vertices,
18458 owned_dirty_events,
18459 island_plan,
18460 ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
18461 self.build_formula_plane_target_schedule(
18462 &formula_dirty,
18463 &retry_whole_spans,
18464 include_dirty_regions,
18465 target_roots,
18466 )?
18467 } else {
18468 self.build_formula_plane_mixed_schedule(
18469 &formula_dirty,
18470 &retry_whole_spans,
18471 include_dirty_regions,
18472 )?
18473 };
18474 #[cfg(test)]
18475 let mut schedule = schedule;
18476 #[cfg(test)]
18477 if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
18478 let producer = schedule
18479 .layers
18480 .iter()
18481 .flat_map(|layer| layer.work.iter())
18482 .map(|work| work.producer)
18483 .next()
18484 .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
18485 schedule.fallbacks.push(
18486 crate::formula_plane::scheduler::MixedScheduleFallback {
18487 producer,
18488 reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
18489 },
18490 );
18491 }
18492
18493 if schedule.is_authoritative_safe() {
18494 break (
18495 schedule,
18496 span_refs_by_id,
18497 plane_epoch,
18498 legacy_vertices,
18499 owned_dirty_events,
18500 island_plan,
18501 );
18502 }
18503
18504 let has_cycle_fallback = schedule.stats.cycle_count > 0
18509 || schedule
18510 .fallbacks
18511 .iter()
18512 .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
18513 if !has_cycle_fallback {
18514 let selected_span_refs = schedule
18515 .layers
18516 .iter()
18517 .flat_map(|layer| layer.work.iter())
18518 .filter_map(|work| match work.producer {
18519 FormulaProducerId::Span(span_id) => {
18520 span_refs_by_id.get(&span_id).copied()
18521 }
18522 FormulaProducerId::Legacy(_) => None,
18523 })
18524 .collect::<Vec<_>>();
18525
18526 return self.evaluate_formula_plane_capacity_fallback(
18527 formula_dirty,
18528 &owned_dirty_events,
18529 &selected_span_refs,
18530 runtime_target_roots.as_deref(),
18531 delta.as_deref_mut(),
18532 );
18533 }
18534
18535 let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
18545 if !cyclic_spans.is_empty() {
18546 self.demote_cyclic_spans(&cyclic_spans)?;
18547 }
18548
18549 if self.graph.formula_authority().active_span_count() == 0 {
18550 if let Some(target_roots) = runtime_target_roots.as_deref() {
18553 let refreshed =
18554 self.resolve_target_producers_from_existing_roots(target_roots)?;
18555 let result =
18556 self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
18557 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18558 return Ok(result);
18559 }
18560 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18561 }
18562
18563 prepass_cycle_errors =
18567 prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
18568
18569 retry_whole_spans = if runtime_target_roots.is_some() {
18573 let active = self
18574 .graph
18575 .formula_authority()
18576 .active_span_refs()
18577 .into_iter()
18578 .map(|span_ref| (span_ref.id, span_ref))
18579 .collect::<BTreeMap<_, _>>();
18580 let mut refs = schedule
18581 .layers
18582 .iter()
18583 .flat_map(|layer| layer.work.iter())
18584 .filter_map(|work| match work.producer {
18585 FormulaProducerId::Span(id) => active.get(&id).copied(),
18586 FormulaProducerId::Legacy(_) => None,
18587 })
18588 .collect::<Vec<_>>();
18589 refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
18590 match fallback.producer {
18591 FormulaProducerId::Span(id) => active.get(&id).copied(),
18592 FormulaProducerId::Legacy(_) => None,
18593 }
18594 }));
18595 refs.sort_by_key(|span_ref| span_ref.id);
18596 refs.dedup();
18597 refs
18598 } else {
18599 self.graph.formula_authority().active_span_refs()
18600 };
18601 self.graph.mark_formula_spans_dirty(
18602 retry_whole_spans.iter().copied(),
18603 WholeSpanDirtyReason::CycleRetry,
18604 );
18605 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
18610 {
18611 formula_dirty = extended;
18612 #[cfg(test)]
18613 if std::mem::take(
18614 &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
18615 ) {
18616 self.graph.mark_formula_spans_dirty(
18617 retry_whole_spans.iter().copied().take(1),
18618 WholeSpanDirtyReason::GlobalInvalidation,
18619 );
18620 }
18621 }
18622 include_dirty_regions = false;
18623
18624 cycle_demote_iters += 1;
18625 if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
18626 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18627 "FormulaPlane cyclic demotion did not converge".to_string(),
18628 ));
18629 }
18630 };
18631 self.observe_formula_plane_route(
18632 &island_plan,
18633 FormulaPlaneRoutePhase::Planned,
18634 FormulaPlaneRouteTransitionReason::ProvenIsolated,
18635 cycle_demote_iters,
18636 runtime_replan_iterations,
18637 );
18638 if !island_plan.dirty_vertices.is_empty() {
18639 self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
18640 let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
18641 &island_plan.dirty_vertices,
18642 delta.as_deref_mut(),
18643 )?;
18644 computed_vertices = computed_vertices.saturating_add(island_computed);
18645 prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
18646 self.observe_formula_plane_route(
18647 &island_plan,
18648 FormulaPlaneRoutePhase::Executed,
18649 FormulaPlaneRouteTransitionReason::ProvenIsolated,
18650 cycle_demote_iters,
18651 runtime_replan_iterations,
18652 );
18653 self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
18654 }
18655 let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
18656 let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18657
18658 #[cfg(test)]
18659 {
18660 self.last_formula_plane_span_eval_report = None;
18661 }
18662 for layer in schedule.layers {
18663 self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
18664 let mut buffer = ComputedWriteBuffer::default();
18665 let mut sink = SpanComputedWriteSink::new(&mut buffer);
18666 let work_items = layer.work;
18667 let mut work_index = 0usize;
18668 while work_index < work_items.len() {
18669 match work_items[work_index].producer {
18670 FormulaProducerId::Span(span_id) => {
18671 let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
18672 ExcelError::new(ExcelErrorKind::NImpl)
18673 .with_message("FormulaPlane schedule referenced a stale span")
18674 })?;
18675 let sheet_id = {
18676 let authority = self.graph.formula_authority();
18677 let span =
18678 authority.plane.spans.get(span_ref).ok_or_else(|| {
18679 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18680 "FormulaPlane schedule referenced a stale span",
18681 )
18682 })?;
18683 span.sheet_id
18684 };
18685 let current_sheet = self.graph.sheet_name(sheet_id);
18686 let authority = self.graph.formula_authority();
18687 let evaluator = SpanEvaluator::new_with_cancel(
18688 &authority.plane,
18689 self,
18690 current_sheet,
18691 self.graph.data_store(),
18692 self.graph.sheet_reg(),
18693 self.active_cancel_flag.as_ref(),
18694 );
18695 #[cfg(test)]
18696 let mut last_group_report = None;
18697 let mut selected_group_work = 0_u64;
18698 while work_index < work_items.len() {
18699 let FormulaProducerId::Span(group_span_id) =
18700 work_items[work_index].producer
18701 else {
18702 break;
18703 };
18704 let group_span_ref =
18705 *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
18706 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18707 "FormulaPlane schedule referenced a stale span",
18708 )
18709 })?;
18710 let group_sheet_id = {
18711 let authority = self.graph.formula_authority();
18712 let span =
18713 authority.plane.spans.get(group_span_ref).ok_or_else(
18714 || {
18715 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18716 "FormulaPlane schedule referenced a stale span",
18717 )
18718 },
18719 )?;
18720 span.sheet_id
18721 };
18722 if group_sheet_id != sheet_id {
18723 break;
18724 }
18725
18726 let dirty = producer_dirty_to_span_dirty(
18727 work_items[work_index].dirty.clone(),
18728 group_span_ref,
18729 );
18730 let task = SpanEvalTask {
18731 span: group_span_ref,
18732 dirty,
18733 plane_epoch,
18734 };
18735 self.cancellation_checkpoint(
18736 "Evaluation cancelled during FormulaPlane span",
18737 )?;
18738 let report = evaluator.evaluate_task(&task, &mut sink).map_err(
18739 |err| {
18740 if err
18741 == crate::formula_plane::span_eval::SpanEvalError::Cancelled
18742 {
18743 ExcelError::new(ExcelErrorKind::Cancelled).with_message(
18744 "Evaluation cancelled during FormulaPlane span",
18745 )
18746 } else {
18747 ExcelError::new(ExcelErrorKind::NImpl).with_message(
18748 format!(
18749 "FormulaPlane span evaluation failed: {err:?}"
18750 ),
18751 )
18752 }
18753 },
18754 )?;
18755 #[cfg(test)]
18756 {
18757 last_group_report = Some(report.clone());
18758 }
18759 selected_group_work = selected_group_work
18760 .saturating_add(report.span_eval_placement_count);
18761 computed_vertices = computed_vertices
18762 .saturating_add(report.span_eval_placement_count as usize);
18763 work_index = work_index.saturating_add(1);
18764 }
18765 self.charge_bounded_work(selected_group_work)?;
18768 #[cfg(test)]
18769 {
18770 if let Some(report) = last_group_report {
18771 self.last_formula_plane_span_eval_report = Some(report);
18772 }
18773 }
18774 }
18775 FormulaProducerId::Legacy(_) => {
18776 let mut vertices = Vec::new();
18785 while work_index < work_items.len() {
18786 let FormulaProducerId::Legacy(vertex_id) =
18787 work_items[work_index].producer
18788 else {
18789 break;
18790 };
18791 vertices.push(vertex_id);
18792 work_index = work_index.saturating_add(1);
18793 }
18794 let legacy_layer = crate::engine::scheduler::Layer { vertices };
18795 let evaluated = if let Some(delta) = delta.as_deref_mut() {
18796 if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
18797 self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
18798 } else {
18799 self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
18800 }
18801 } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
18802 {
18803 self.evaluate_layer_parallel(&legacy_layer)?
18804 } else {
18805 self.evaluate_layer_sequential(&legacy_layer)?
18806 };
18807 computed_vertices = computed_vertices.saturating_add(evaluated);
18808 }
18809 }
18810 }
18811 self.resource_checkpoint(0)?;
18812 self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
18813 if let Some(delta) = delta.as_deref_mut() {
18814 self.record_computed_write_buffer_delta(&buffer, delta);
18815 }
18816 self.flush_computed_write_buffer(&mut buffer)?;
18817 }
18818
18819 let mut changed_virtual =
18820 self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
18821 let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18822 let mut dynamic_candidates = old_dynamic_regions
18823 .keys()
18824 .chain(new_dynamic_regions.keys())
18825 .copied()
18826 .collect::<FxHashSet<_>>();
18827 for vertex in dynamic_candidates.drain() {
18828 if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
18829 && !changed_virtual.contains(&vertex)
18830 {
18831 changed_virtual.push(vertex);
18832 }
18833 }
18834 if let Some(telemetry) = virtual_telemetry.as_mut() {
18835 telemetry.changed_vdeps_total = telemetry
18836 .changed_vdeps_total
18837 .saturating_add(changed_virtual.len());
18838 }
18839 self.resource_checkpoint(0)?;
18840 self.graph.clear_dirty_flags(&legacy_vertices);
18841 if !changed_virtual.is_empty() {
18842 for vertex in &changed_virtual {
18843 self.graph.set_dirty(*vertex, true);
18844 }
18845 let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
18846 let can_attempt_workbook =
18847 !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
18848 if can_replan || can_attempt_workbook {
18849 if can_replan {
18850 if let Some(roots) = runtime_target_roots.as_mut() {
18851 self.extend_target_roots_with_dynamic_regions(
18852 roots,
18853 old_dynamic_regions
18854 .values()
18855 .flatten()
18856 .chain(new_dynamic_regions.values().flatten()),
18857 )?;
18858 }
18859 runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
18860 } else {
18861 runtime_target_roots = None;
18862 workbook_exact_attempted = true;
18863 }
18864 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
18865 if can_replan {
18866 stats.runtime_replan_rounds =
18867 stats.runtime_replan_rounds.saturating_add(1);
18868 }
18869 stats.runtime_widening_rounds =
18870 stats.runtime_widening_rounds.saturating_add(1);
18871 if can_attempt_workbook {
18872 stats.workbook_exact_attempts = 1;
18873 }
18874 }
18875 self.cached_mixed_topology = None;
18876 if let Some(ledger) = self.active_resource_ledger.as_mut() {
18877 ledger
18878 .account_mixed_cache(0)
18879 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
18880 }
18881 include_dirty_regions = true;
18882 continue 'virtual_replan;
18883 }
18884 if let Some(telemetry) = virtual_telemetry.as_mut() {
18885 telemetry.replan_iterations = runtime_replan_iterations;
18886 telemetry.bailout_reason = Some("max_replan");
18887 self.last_virtual_dep_telemetry = telemetry.clone();
18888 }
18889 return Err(self.replan_exhausted_error(
18890 MAX_RUNTIME_REPLAN,
18891 "mixed dynamic dependency evaluation",
18892 ));
18893 }
18894 if let Some(mut telemetry) = virtual_telemetry {
18895 telemetry.replan_iterations = runtime_replan_iterations;
18896 telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
18897 "converged"
18898 } else {
18899 "max_replan"
18900 });
18901 self.last_virtual_dep_telemetry = telemetry;
18902 }
18903 self.redirty_for_next_recalc();
18907 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
18908 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
18909 return Ok(EvalResult {
18910 computed_vertices,
18911 cycle_errors: prepass_cycle_errors,
18912 elapsed: start.elapsed(),
18913 });
18914 }
18915 }
18916
18917 fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
18918 let budgets = &self.evaluation_resource_budgets;
18919 let candidate_limit = budgets
18920 .optimization
18921 .mixed_cache_candidates
18922 .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
18923 limit.min(self.config.max_formula_plane_cache_candidates)
18924 });
18925 let edge_limit = budgets
18926 .optimization
18927 .mixed_cache_edges
18928 .map_or(self.config.max_formula_plane_cache_edges, |limit| {
18929 limit.min(self.config.max_formula_plane_cache_edges)
18930 });
18931 let mut byte_limit =
18932 u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
18933 if let Some(limit) = budgets.retained.total_bytes {
18934 byte_limit = byte_limit.min(limit);
18935 }
18936 if let Some(limit) = budgets.retained.mixed_cache_bytes {
18937 byte_limit = byte_limit.min(limit);
18938 }
18939 (
18940 candidate_limit,
18941 edge_limit,
18942 usize::try_from(byte_limit).unwrap_or(usize::MAX),
18943 )
18944 }
18945
18946 fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
18947 let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
18948 MixedTopologyCacheKey {
18949 engine_topology_epoch: self.topology_epoch,
18950 graph_topology_revision: self.graph.topology_revision(),
18951 authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
18952 legacy_island_revision: self.graph.topology_revision(),
18953 boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
18954 function_semantic_epoch: self.function_semantic_epoch_seen,
18955 function_provider_revision: self.function_provider_revision_seen,
18956 max_candidates,
18957 max_edges,
18958 max_memory_bytes,
18959 }
18960 }
18961
18962 fn mixed_topology_index_preflight_bytes(&self) -> u64 {
18963 use crate::formula_plane::region_index::AxisRange;
18964
18965 const FIXED_BYTES: u64 = 8 * 1024;
18966 const ENTRY_BYTES: u64 = 512;
18967 const INDEX_UNIT_BYTES: u64 = 1024;
18968 const BINDING_BYTES: u64 = 128;
18969 let index_units = |region: Region| -> u64 {
18970 match region.normalized().axis_ranges() {
18971 (AxisRange::Point(_), AxisRange::Point(_)) => 2,
18972 (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
18973 let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
18974 let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
18975 row_buckets.saturating_mul(col_buckets).saturating_mul(2)
18976 }
18977 _ => 1,
18978 }
18979 };
18980
18981 let authority = self.graph.formula_authority();
18982 let span_refs = authority.active_span_refs();
18983 let legacy_vertices = self.graph.formula_vertices();
18984 let mut entry_count = 0_u64;
18985 let mut units = 0_u64;
18986 for span_ref in &span_refs {
18987 if let Some(span) = authority.plane.spans.get(*span_ref) {
18988 let result_region = Region::from_domain(span.result_region.domain());
18989 entry_count = entry_count.saturating_add(1);
18990 units = units.saturating_add(index_units(result_region));
18991 if let Some(summary_id) = span.read_summary_id
18992 && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
18993 {
18994 for dependency in &summary.dependencies {
18995 entry_count = entry_count.saturating_add(1);
18996 units = units.saturating_add(index_units(dependency.read_region));
18997 }
18998 }
18999 }
19000 }
19001 for &vertex in &legacy_vertices {
19002 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
19003 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19004 entry_count = entry_count.saturating_add(1);
19005 units = units.saturating_add(index_units(point));
19006 }
19007 for dependency in self.graph.get_dependencies(vertex) {
19008 if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
19009 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19010 entry_count = entry_count.saturating_add(1);
19011 units = units.saturating_add(index_units(point));
19012 }
19013 }
19014 if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
19015 for range in ranges {
19016 if let Ok(Some(region)) = self.shared_range_to_region_pattern(range) {
19017 entry_count = entry_count.saturating_add(1);
19018 units = units.saturating_add(index_units(region));
19019 }
19020 }
19021 }
19022 }
19023 FIXED_BYTES
19024 .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
19025 .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
19026 .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
19027 }
19028
19029 fn prove_sheet_disjoint_legacy_island(
19030 &self,
19031 span_results: &FormulaProducerResultIndex,
19032 span_reads: &FormulaConsumerReadIndex,
19033 legacy_vertices: &[VertexId],
19034 ) -> Option<LegacyIslandPlan> {
19035 if !self.legacy_island_structural_summaries_trusted
19036 || self.graph.named_ranges_iter().next().is_some()
19037 || self.graph.sheet_named_ranges_iter().next().is_some()
19038 || self.graph.spill_registry_counts() != (0, 0)
19039 || legacy_vertices.iter().copied().any(|vertex| {
19040 self.graph.is_dynamic(vertex)
19041 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19042 })
19043 {
19044 return None;
19045 }
19046 let span_result_sheets = span_results
19047 .entries()
19048 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19049 .map(|entry| entry.result_region.sheet_id())
19050 .collect::<FxHashSet<_>>();
19051 let span_read_sheets = span_reads
19052 .entries()
19053 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
19054 .map(|entry| entry.read_region.sheet_id())
19055 .collect::<FxHashSet<_>>();
19056
19057 let span_boundary_sheets = span_result_sheets
19058 .union(&span_read_sheets)
19059 .copied()
19060 .collect::<FxHashSet<_>>();
19061 for vertex in legacy_vertices {
19062 let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
19063 if span_boundary_sheets.contains(&cell.sheet_id) {
19064 return None;
19065 }
19066 for dependency in self.graph.get_dependencies(*vertex) {
19067 let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
19068 if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
19069 return None;
19070 }
19071 }
19072 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19073 for range in ranges {
19074 let region = self.shared_range_to_region_pattern(range).ok()??;
19075 if span_boundary_sheets.contains(®ion.sheet_id()) {
19076 return None;
19077 }
19078 }
19079 }
19080 }
19081
19082 let mut membership = legacy_vertices.to_vec();
19083 membership.sort_unstable();
19084 if membership.is_empty() {
19085 return None;
19086 }
19087 let mut sheet_ids = membership
19088 .iter()
19089 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19090 .map(|cell| cell.sheet_id)
19091 .collect::<Vec<_>>();
19092 sheet_ids.sort_unstable();
19093 sheet_ids.dedup();
19094 let island_id = membership
19095 .iter()
19096 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19097 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19098 });
19099 let retained_bytes = membership
19100 .capacity()
19101 .saturating_mul(std::mem::size_of::<VertexId>())
19102 .saturating_add(
19103 sheet_ids
19104 .capacity()
19105 .saturating_mul(std::mem::size_of::<SheetId>()),
19106 );
19107 Some(LegacyIslandPlan {
19108 membership,
19109 dirty_vertices: Vec::new(),
19110 sheet_ids,
19111 island_id,
19112 retained_bytes,
19113 boundary_relationships: 0,
19114 omitted_relationships: legacy_vertices.len(),
19115 omitted_direct_relationships: 0,
19116 })
19117 }
19118
19119 fn contract_legacy_islands(
19120 &self,
19121 full_results: &FormulaProducerResultIndex,
19122 full_reads: &FormulaConsumerReadIndex,
19123 legacy_reads_complete: bool,
19124 candidate_cap: usize,
19125 ) -> Option<(
19126 FormulaProducerResultIndex,
19127 FormulaConsumerReadIndex,
19128 LegacyIslandPlan,
19129 )> {
19130 use crate::formula_plane::producer::ProjectionResult;
19131 use crate::formula_plane::region_index::BoundedRegionQueryResult;
19132
19133 if !self.legacy_island_structural_summaries_trusted
19134 || !legacy_reads_complete
19135 || self.graph.named_ranges_iter().next().is_some()
19136 || self.graph.sheet_named_ranges_iter().next().is_some()
19137 || self.graph.spill_registry_counts() != (0, 0)
19138 {
19139 return None;
19140 }
19141 let legacy_vertices = self.graph.formula_vertices();
19142 if legacy_vertices.iter().copied().any(|vertex| {
19143 self.graph.is_dynamic(vertex)
19144 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19145 }) {
19146 return None;
19147 }
19148
19149 let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
19150 for entry in full_reads.entries() {
19151 if let FormulaProducerId::Legacy(vertex) = entry.consumer {
19152 reads_by_consumer
19153 .entry(vertex)
19154 .or_default()
19155 .push(entry.read_region);
19156 }
19157 }
19158
19159 let mut connected = FxHashSet::default();
19160 let mut queue = VecDeque::new();
19161 let mut observed_candidates = 0usize;
19162 let mut boundary_relationships = 0usize;
19163 let add_connected = |vertex: VertexId,
19164 connected: &mut FxHashSet<VertexId>,
19165 queue: &mut VecDeque<VertexId>| {
19166 if connected.insert(vertex) {
19167 queue.push_back(vertex);
19168 }
19169 };
19170
19171 for result in full_results.entries() {
19173 if !matches!(result.producer, FormulaProducerId::Span(_)) {
19174 continue;
19175 }
19176 let remaining = candidate_cap.saturating_sub(observed_candidates);
19177 let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
19178 let BoundedRegionQueryResult::Complete(query) = query else {
19179 return None;
19180 };
19181 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19182 for matched in query.matches {
19183 if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
19184 match matched.value.dirty {
19185 ProjectionResult::NoIntersection => {}
19186 ProjectionResult::Unsupported(_) => return None,
19187 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19188 boundary_relationships = boundary_relationships.saturating_add(1);
19189 add_connected(vertex, &mut connected, &mut queue);
19190 }
19191 }
19192 }
19193 }
19194 }
19195
19196 for read in full_reads.entries() {
19198 if !matches!(read.consumer, FormulaProducerId::Span(_)) {
19199 continue;
19200 }
19201 let remaining = candidate_cap.saturating_sub(observed_candidates);
19202 let query = full_results.query_bounded(read.read_region, remaining);
19203 let BoundedRegionQueryResult::Complete(query) = query else {
19204 return None;
19205 };
19206 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19207 for matched in query.matches {
19208 if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
19209 boundary_relationships = boundary_relationships.saturating_add(1);
19210 add_connected(vertex, &mut connected, &mut queue);
19211 }
19212 }
19213 }
19214
19215 while let Some(vertex) = queue.pop_front() {
19219 let producer = FormulaProducerId::Legacy(vertex);
19220 if let Some(result_region) = full_results.producer_result_region(producer) {
19221 let remaining = candidate_cap.saturating_sub(observed_candidates);
19222 let query = full_reads.query_changed_region_bounded(result_region, remaining);
19223 let BoundedRegionQueryResult::Complete(query) = query else {
19224 return None;
19225 };
19226 observed_candidates =
19227 observed_candidates.saturating_add(query.stats.candidate_count);
19228 for matched in query.matches {
19229 if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
19230 match matched.value.dirty {
19231 ProjectionResult::NoIntersection => {}
19232 ProjectionResult::Unsupported(_) => return None,
19233 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19234 add_connected(consumer, &mut connected, &mut queue);
19235 }
19236 }
19237 }
19238 }
19239 }
19240 for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
19241 let remaining = candidate_cap.saturating_sub(observed_candidates);
19242 let query = full_results.query_bounded(*read_region, remaining);
19243 let BoundedRegionQueryResult::Complete(query) = query else {
19244 return None;
19245 };
19246 observed_candidates =
19247 observed_candidates.saturating_add(query.stats.candidate_count);
19248 for matched in query.matches {
19249 if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
19250 add_connected(precedent, &mut connected, &mut queue);
19251 }
19252 }
19253 }
19254 }
19255
19256 let mut membership = legacy_vertices
19257 .iter()
19258 .copied()
19259 .filter(|vertex| !connected.contains(vertex))
19260 .collect::<Vec<_>>();
19261 membership.sort_unstable();
19262 if membership.is_empty() {
19263 return None;
19264 }
19265
19266 let mut producer_results = FormulaProducerResultIndex::default();
19267 let mut consumer_reads = FormulaConsumerReadIndex::default();
19268 let retained_span_results = full_results
19269 .entries()
19270 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19271 .count();
19272 producer_results
19273 .try_reserve(retained_span_results.saturating_add(connected.len()))
19274 .ok()?;
19275 for entry in full_results.entries() {
19276 let retain = match entry.producer {
19277 FormulaProducerId::Span(_) => true,
19278 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19279 };
19280 if retain {
19281 producer_results.insert_producer(entry.producer, entry.result_region);
19282 }
19283 }
19284 for entry in full_reads.entries() {
19285 let retain = match entry.consumer {
19286 FormulaProducerId::Span(_) => true,
19287 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19288 };
19289 if retain {
19290 consumer_reads.try_reserve(1).ok()?;
19291 consumer_reads.insert_read(
19292 entry.consumer,
19293 entry.read_region,
19294 entry.consumer_result_region,
19295 entry.projection,
19296 );
19297 }
19298 }
19299
19300 let mut sheet_ids = membership
19301 .iter()
19302 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19303 .map(|cell| cell.sheet_id)
19304 .collect::<Vec<_>>();
19305 sheet_ids.sort_unstable();
19306 sheet_ids.dedup();
19307 let island_id = membership
19308 .iter()
19309 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19310 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19311 });
19312 let retained_bytes = membership
19313 .capacity()
19314 .saturating_mul(std::mem::size_of::<VertexId>())
19315 .saturating_add(
19316 sheet_ids
19317 .capacity()
19318 .saturating_mul(std::mem::size_of::<SheetId>()),
19319 );
19320 let omitted_relationships = full_reads
19321 .entries()
19322 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
19323 .count();
19324 let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
19325 let omitted_direct_relationships = membership
19326 .iter()
19327 .map(|vertex| {
19328 self.graph
19329 .get_dependencies(*vertex)
19330 .into_iter()
19331 .filter(|dependency| membership_set.contains(dependency))
19332 .count()
19333 })
19334 .sum();
19335 Some((
19336 producer_results,
19337 consumer_reads,
19338 LegacyIslandPlan {
19339 membership,
19340 dirty_vertices: Vec::new(),
19341 sheet_ids,
19342 island_id,
19343 retained_bytes,
19344 boundary_relationships,
19345 omitted_relationships,
19346 omitted_direct_relationships,
19347 },
19348 ))
19349 }
19350
19351 fn compile_formula_plane_mixed_topology(
19352 &mut self,
19353 key: MixedTopologyCacheKey,
19354 ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
19355 #[cfg(test)]
19356 {
19357 self.mixed_topology_index_builds_for_test =
19358 self.mixed_topology_index_builds_for_test.saturating_add(1);
19359 }
19360 let authority = self.graph.formula_authority();
19361 let mut producer_results = FormulaProducerResultIndex::default();
19362 let mut consumer_reads = FormulaConsumerReadIndex::default();
19363 let mut legacy_reads_complete = true;
19364 let span_refs = authority.active_span_refs();
19365 let legacy_vertices = self.graph.formula_vertices();
19366 let _producer_count = span_refs
19367 .len()
19368 .checked_add(legacy_vertices.len())
19369 .ok_or_else(|| {
19370 ExcelError::new(ExcelErrorKind::NImpl)
19371 .with_message("FormulaPlane cache producer count overflow")
19372 })?;
19373 producer_results.try_reserve(_producer_count).map_err(|_| {
19374 ExcelError::new(ExcelErrorKind::NImpl)
19375 .with_message("FormulaPlane cache producer index reservation failed")
19376 })?;
19377 let span_refs_by_id = span_refs
19378 .iter()
19379 .copied()
19380 .map(|span_ref| (span_ref.id, span_ref))
19381 .collect::<BTreeMap<_, _>>();
19382 for span_ref in &span_refs {
19383 let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
19384 ExcelError::new(ExcelErrorKind::NImpl)
19385 .with_message("FormulaPlane active span ref is stale")
19386 })?;
19387 let result_region = Region::from_domain(span.result_region.domain());
19388 producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
19389 let Some(read_summary_id) = span.read_summary_id else {
19390 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19391 .with_message("FormulaPlane active span is missing read summary"));
19392 };
19393 let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
19394 else {
19395 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19396 .with_message("FormulaPlane active span has stale read summary"));
19397 };
19398 if read_summary.result_region != result_region {
19399 return Err(ExcelError::new(ExcelErrorKind::NImpl)
19400 .with_message("FormulaPlane active span read summary is stale"));
19401 }
19402 for dependency in &read_summary.dependencies {
19403 consumer_reads.try_reserve(1).map_err(|_| {
19404 ExcelError::new(ExcelErrorKind::NImpl)
19405 .with_message("FormulaPlane cache read-index reservation failed")
19406 })?;
19407 consumer_reads.insert_read(
19408 FormulaProducerId::Span(span.id),
19409 dependency.read_region,
19410 read_summary.result_region,
19411 dependency.projection,
19412 );
19413 }
19414 }
19415
19416 let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
19417 &producer_results,
19418 &consumer_reads,
19419 &legacy_vertices,
19420 );
19421 if sheet_disjoint_island.is_none() {
19422 for vertex in &legacy_vertices {
19423 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19424 continue;
19425 };
19426 producer_results.insert_producer(
19427 FormulaProducerId::Legacy(*vertex),
19428 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
19429 );
19430 }
19431 for vertex in &legacy_vertices {
19432 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19433 continue;
19434 };
19435 let result_region =
19436 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19437 let mut seen = rustc_hash::FxHashSet::default();
19438 for dep in self.graph.get_dependencies(*vertex) {
19439 let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
19440 legacy_reads_complete = false;
19443 continue;
19444 };
19445 let read_region = Region::point(
19446 dep_cell.sheet_id,
19447 dep_cell.coord.row(),
19448 dep_cell.coord.col(),
19449 );
19450 if seen.insert(read_region) {
19451 consumer_reads.try_reserve(1).map_err(|_| {
19452 ExcelError::new(ExcelErrorKind::NImpl)
19453 .with_message("FormulaPlane cache read-index reservation failed")
19454 })?;
19455 consumer_reads.insert_read(
19456 FormulaProducerId::Legacy(*vertex),
19457 read_region,
19458 result_region,
19459 DirtyProjectionRule::WholeResult,
19460 );
19461 }
19462 }
19463 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19464 for range in ranges {
19465 let Some(read_region) = self.shared_range_to_region_pattern(range)? else {
19466 legacy_reads_complete = false;
19467 continue;
19468 };
19469 if seen.insert(read_region) {
19470 consumer_reads.try_reserve(1).map_err(|_| {
19471 ExcelError::new(ExcelErrorKind::NImpl).with_message(
19472 "FormulaPlane cache read-index reservation failed",
19473 )
19474 })?;
19475 consumer_reads.insert_read(
19476 FormulaProducerId::Legacy(*vertex),
19477 read_region,
19478 result_region,
19479 DirtyProjectionRule::WholeResult,
19480 );
19481 }
19482 }
19483 }
19484 let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
19485 for dependency in virtual_dependencies
19486 .get(vertex)
19487 .into_iter()
19488 .flatten()
19489 .copied()
19490 {
19491 let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
19492 continue;
19493 };
19494 let read_region =
19495 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19496 if seen.insert(read_region) {
19497 consumer_reads.try_reserve(1).map_err(|_| {
19498 ExcelError::new(ExcelErrorKind::NImpl)
19499 .with_message("FormulaPlane cache virtual read reservation failed")
19500 })?;
19501 consumer_reads.insert_read(
19502 FormulaProducerId::Legacy(*vertex),
19503 read_region,
19504 result_region,
19505 DirtyProjectionRule::WholeResult,
19506 );
19507 }
19508 }
19509 for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
19510 {
19511 if seen.insert(read_region) {
19512 consumer_reads.try_reserve(1).map_err(|_| {
19513 ExcelError::new(ExcelErrorKind::NImpl)
19514 .with_message("FormulaPlane dynamic-region reservation failed")
19515 })?;
19516 consumer_reads.insert_read(
19517 FormulaProducerId::Legacy(*vertex),
19518 read_region,
19519 result_region,
19520 DirtyProjectionRule::WholeResult,
19521 );
19522 }
19523 }
19524 }
19525 }
19526
19527 let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
19531 {
19532 (producer_results, consumer_reads, island)
19533 } else {
19534 self.contract_legacy_islands(
19535 &producer_results,
19536 &consumer_reads,
19537 legacy_reads_complete,
19538 key.max_candidates,
19539 )
19540 .unwrap_or((
19541 producer_results,
19542 consumer_reads,
19543 LegacyIslandPlan::default(),
19544 ))
19545 };
19546
19547 let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
19548 .checked_add(
19549 producer_results
19550 .estimated_memory_bytes()
19551 .unwrap_or(usize::MAX),
19552 )
19553 .and_then(|bytes| {
19554 bytes.checked_add(
19555 consumer_reads
19556 .estimated_memory_bytes()
19557 .unwrap_or(usize::MAX),
19558 )
19559 })
19560 .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
19561 .and_then(|bytes| bytes.checked_add(island.retained_bytes))
19562 .unwrap_or(usize::MAX);
19563 let config = MixedTopologyConfig {
19564 max_candidates: key.max_candidates,
19565 max_edges: key.max_edges,
19566 max_memory_bytes: key.max_memory_bytes,
19567 retained_memory_bytes,
19568 };
19569 let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
19570 match &mut compiled {
19574 MixedTopologyCompileResult::Cached(topology) => {
19575 topology.stats.producers = topology
19576 .stats
19577 .producers
19578 .saturating_add(island.membership.len());
19579 topology.stats.candidates = topology
19580 .stats
19581 .candidates
19582 .saturating_add(island.omitted_direct_relationships);
19583 topology.stats.relationships = topology
19584 .stats
19585 .relationships
19586 .saturating_add(island.omitted_direct_relationships);
19587 }
19588 MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
19589 observed.producers = observed.producers.saturating_add(island.membership.len());
19590 observed.candidates = observed
19591 .candidates
19592 .saturating_add(island.omitted_direct_relationships);
19593 observed.relationships = observed
19594 .relationships
19595 .saturating_add(island.omitted_direct_relationships);
19596 }
19597 }
19598 let plane_epoch = authority.plane.epoch().0;
19599 Ok(match compiled {
19600 MixedTopologyCompileResult::Cached(topology) => {
19601 FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
19602 key,
19603 topology,
19604 producer_results,
19605 consumer_reads,
19606 span_refs_by_id,
19607 plane_epoch,
19608 island,
19609 })
19610 }
19611 MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
19612 FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
19613 reason,
19614 observed,
19615 producer_results,
19616 consumer_reads,
19617 span_refs_by_id,
19618 plane_epoch,
19619 island,
19620 })
19621 }
19622 })
19623 }
19624
19625 fn build_formula_plane_mixed_schedule(
19626 &mut self,
19627 formula_dirty: &FormulaDirtyLease,
19628 retry_whole_spans: &[FormulaSpanRef],
19629 include_dirty_regions: bool,
19630 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19631 self.build_formula_plane_mixed_schedule_inner(
19632 formula_dirty,
19633 retry_whole_spans,
19634 include_dirty_regions,
19635 None,
19636 )
19637 }
19638
19639 fn build_formula_plane_target_schedule(
19640 &mut self,
19641 formula_dirty: &FormulaDirtyLease,
19642 retry_whole_spans: &[FormulaSpanRef],
19643 include_dirty_regions: bool,
19644 target_roots: &[crate::engine::target_preparation::TargetProducer],
19645 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19646 self.build_formula_plane_mixed_schedule_inner(
19647 formula_dirty,
19648 retry_whole_spans,
19649 include_dirty_regions,
19650 Some(target_roots),
19651 )
19652 }
19653
19654 fn build_formula_plane_mixed_schedule_inner(
19655 &mut self,
19656 formula_dirty: &FormulaDirtyLease,
19657 retry_whole_spans: &[FormulaSpanRef],
19658 include_dirty_regions: bool,
19659 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19660 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19661 let baseline = self.graph.baseline_stats();
19662 self.charge_bounded_work(
19663 baseline
19664 .graph_vertex_count
19665 .saturating_add(baseline.graph_edge_count) as u64,
19666 )?;
19667 let key = self.mixed_topology_cache_key();
19668 let active_refs = self
19669 .graph
19670 .formula_authority()
19671 .active_span_refs()
19672 .into_iter()
19673 .map(|span_ref| (span_ref.id, span_ref))
19674 .collect::<BTreeMap<_, _>>();
19675 let has_dynamic_legacy = self
19676 .graph
19677 .formula_vertices()
19678 .into_iter()
19679 .any(|vertex| self.graph.is_dynamic(vertex));
19680 let cache_hit = !has_dynamic_legacy
19681 && self
19682 .cached_mixed_topology
19683 .as_ref()
19684 .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
19685 let mut skipped_topology = None;
19686 let mut index_scratch_reserved = 0_u64;
19687 let mut demand_scratch_reserved = 0_u64;
19688 let topology_started = crate::instant::FzInstant::now();
19689 let (cache_outcome, strategy, compile_stats) = if cache_hit {
19690 self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
19691 self.mixed_topology_cache_skip_streak = 0;
19692 let cached = self
19693 .cached_mixed_topology
19694 .as_ref()
19695 .expect("cache hit has topology");
19696 let stats = cached.topology.stats.clone();
19697 let strategy = if cached.topology.is_complete() {
19698 FormulaPlaneTopologyStrategy::Cached
19699 } else {
19700 FormulaPlaneTopologyStrategy::ExactPagedIndexed
19701 };
19702 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19703 ledger
19704 .account_mixed_cache(stats.estimated_memory_bytes as u64)
19705 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19706 }
19707 (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
19708 } else {
19709 self.cached_mixed_topology = None;
19712 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19713 ledger
19714 .account_mixed_cache(0)
19715 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19716 }
19717
19718 let preflight = self.mixed_topology_index_preflight_bytes();
19719 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19720 ledger
19721 .reserve_schedule_discovery(preflight)
19722 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19723 }
19724 index_scratch_reserved = preflight;
19725 let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
19726 Ok(compiled) => compiled,
19727 Err(error) => {
19728 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19729 let _ = ledger.release_scratch(index_scratch_reserved);
19730 }
19731 return Err(error);
19732 }
19733 };
19734 let actual_index_bytes = match &compiled {
19735 FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
19736 .producer_results
19737 .estimated_memory_bytes()
19738 .and_then(|bytes| {
19739 bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
19740 })
19741 .and_then(|bytes| {
19742 bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
19743 })
19744 .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
19745 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
19746 .producer_results
19747 .estimated_memory_bytes()
19748 .and_then(|bytes| {
19749 bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
19750 })
19751 .and_then(|bytes| {
19752 bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
19753 })
19754 .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
19755 }
19756 .unwrap_or(usize::MAX) as u64;
19757 debug_assert!(
19758 actual_index_bytes <= index_scratch_reserved,
19759 "mixed topology index preflight underestimated actual allocation"
19760 );
19761 if actual_index_bytes > index_scratch_reserved {
19762 let additional = actual_index_bytes - index_scratch_reserved;
19763 let reserve_result = self
19764 .active_resource_ledger
19765 .as_mut()
19766 .map_or(Ok(()), |ledger| {
19767 ledger.reserve_schedule_discovery(additional)
19768 });
19769 if let Err(error) = reserve_result {
19770 drop(compiled);
19771 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19772 let _ = ledger.release_scratch(index_scratch_reserved);
19773 }
19774 return Err(error.into_excel_error());
19775 }
19776 } else if actual_index_bytes < index_scratch_reserved
19777 && let Some(ledger) = self.active_resource_ledger.as_mut()
19778 {
19779 ledger
19780 .release_scratch(index_scratch_reserved - actual_index_bytes)
19781 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19782 }
19783 index_scratch_reserved = actual_index_bytes;
19784
19785 self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
19786 match compiled {
19787 FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
19788 let stats = compiled.topology.stats.clone();
19789 self.mixed_topology_cache_skip_streak =
19790 self.mixed_topology_cache_skip_streak.saturating_add(1);
19791 skipped_topology = Some(SkippedMixedTopology {
19792 reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
19793 observed: stats.clone(),
19794 producer_results: compiled.producer_results,
19795 consumer_reads: compiled.consumer_reads,
19796 span_refs_by_id: compiled.span_refs_by_id,
19797 plane_epoch: compiled.plane_epoch,
19798 island: compiled.island,
19799 });
19800 (
19801 FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
19802 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19803 stats,
19804 )
19805 }
19806 FormulaPlaneTopologyCompileResult::Cached(compiled) => {
19807 let stats = compiled.topology.stats.clone();
19808 let strategy = if compiled.topology.is_complete() {
19809 FormulaPlaneTopologyStrategy::CompiledAndCached
19810 } else {
19811 FormulaPlaneTopologyStrategy::ExactPagedIndexed
19812 };
19813 let retained_result = self
19814 .active_resource_ledger
19815 .as_mut()
19816 .map_or(Ok(()), |ledger| {
19817 ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
19818 });
19819 if let Err(error) = retained_result {
19820 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19821 let _ = ledger.release_scratch(index_scratch_reserved);
19822 }
19823 return Err(error.into_excel_error());
19824 }
19825 self.mixed_topology_cache_skip_streak = 0;
19826 self.cached_mixed_topology = Some(compiled);
19827 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19828 ledger
19829 .release_scratch(index_scratch_reserved)
19830 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19831 }
19832 index_scratch_reserved = 0;
19833 (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
19834 }
19835 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
19836 debug_assert!(self.cached_mixed_topology.is_none());
19837 self.mixed_topology_cache_overflows =
19838 self.mixed_topology_cache_overflows.saturating_add(1);
19839 self.mixed_topology_cache_skip_streak =
19840 self.mixed_topology_cache_skip_streak.saturating_add(1);
19841 let stats = skipped.observed.clone();
19842 skipped_topology = Some(skipped);
19843 (
19844 FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
19845 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19846 stats,
19847 )
19848 }
19849 }
19850 };
19851 self.observe_topology(
19852 cache_outcome,
19853 strategy,
19854 &compile_stats,
19855 topology_started.elapsed(),
19856 );
19857 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
19858 stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
19859 stats.topology.candidate_cap = Some(key.max_candidates as u64);
19860 stats.topology.edge_cap = Some(key.max_edges as u64);
19861 stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
19862 if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
19863 stats.topology.operator_guidance =
19864 Some("dynamic legacy dependencies require an exact request topology rebuild");
19865 } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
19866 stats.topology.operator_guidance =
19867 Some(if compile_stats.candidate_overflow_count > 0 {
19868 "raise the mixed-cache candidate limit to avoid exact request rebuilds"
19869 } else if compile_stats.edge_overflow_count > 0 {
19870 "raise the mixed-cache edge limit to avoid exact request rebuilds"
19871 } else {
19872 "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
19873 });
19874 }
19875 }
19876
19877 let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
19878 let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19879 let (
19880 producer_results,
19881 consumer_reads,
19882 span_refs_by_id,
19883 plane_epoch,
19884 cached_topology,
19885 partial_topology,
19886 island,
19887 ) = if let Some(skipped) = skipped_topology.as_ref() {
19888 (
19889 &skipped.producer_results,
19890 &skipped.consumer_reads,
19891 &skipped.span_refs_by_id,
19892 skipped.plane_epoch,
19893 None,
19894 None,
19895 &skipped.island,
19896 )
19897 } else {
19898 let cached = self
19899 .cached_mixed_topology
19900 .as_ref()
19901 .expect("mixed topology available for request");
19902 let (complete, partial) = if cached.topology.is_complete() {
19903 (Some(&cached.topology), None)
19904 } else {
19905 (None, Some(&cached.topology))
19906 };
19907 (
19908 &cached.producer_results,
19909 &cached.consumer_reads,
19910 &cached.span_refs_by_id,
19911 cached.plane_epoch,
19912 complete,
19913 partial,
19914 &cached.island,
19915 )
19916 };
19917 let demand = if let Some(target_roots) = target_roots {
19918 use crate::engine::target_preparation::TargetProducer;
19919 let mut roots = Vec::new();
19920 let mut graph_roots = Vec::new();
19921 for root in target_roots {
19922 match *root {
19923 TargetProducer::Span { span_ref, demanded } => {
19924 roots.push((FormulaProducerId::Span(span_ref.id), demanded))
19925 }
19926 TargetProducer::Legacy(vertex) => {
19927 graph_roots.push(vertex);
19928 if let Some(region) = producer_results
19929 .producer_result_region(FormulaProducerId::Legacy(vertex))
19930 {
19931 roots.push((FormulaProducerId::Legacy(vertex), region));
19932 }
19933 }
19934 TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
19935 TargetProducer::ValueOnly(_) => {}
19936 }
19937 }
19938 if !graph_roots.is_empty() {
19939 let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
19940 for vertex in legacy_demand {
19941 if let Some(region) = producer_results
19942 .producer_result_region(FormulaProducerId::Legacy(vertex))
19943 {
19944 roots.push((FormulaProducerId::Legacy(vertex), region));
19945 }
19946 }
19947 }
19948 let roots = roots;
19949 let closure_base = (producer_results.len() as u64)
19950 .saturating_mul(128)
19951 .saturating_add(4096);
19952 let scratch_estimates =
19953 exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
19954 let native_allowed = native_exact_demand_allowed(
19955 self.active_resource_ledger
19956 .as_ref()
19957 .and_then(ResourceLedger::disk_scratch_policy),
19958 !cfg!(target_arch = "wasm32"),
19959 );
19960 let (selected_strategy, reservation) = if cached_topology.is_some() {
19961 (None, closure_base)
19962 } else {
19963 let (strategy, reservation) =
19964 select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
19965 self.can_reserve_topology_scratch(bytes)
19966 });
19967 (Some(strategy), reservation)
19968 };
19969 if let Some(ledger) = self.active_resource_ledger.as_mut() {
19970 ledger
19971 .reserve_schedule_discovery(reservation)
19972 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19973 }
19974 demand_scratch_reserved = reservation;
19975 #[allow(unused_mut)]
19976 let mut native_disk_bytes = 0_u64;
19977 let (demand, actual_strategy, passes) = match selected_strategy {
19978 None => (
19979 build_demand_closure_cached(
19980 roots,
19981 producer_results,
19982 cached_topology.expect("cached demand strategy has topology"),
19983 ),
19984 None,
19985 0,
19986 ),
19987 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
19988 let (demand, passes) = build_demand_closure_paged(
19989 roots,
19990 producer_results,
19991 consumer_reads,
19992 |units| {
19993 Self::resource_loop_checkpoint(
19994 &mut self.active_resource_ledger,
19995 units,
19996 )
19997 },
19998 )?;
19999 (demand, selected_strategy, passes)
20000 }
20001 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20002 let (demand, passes) = build_demand_closure_in_memory_runs(
20003 roots,
20004 producer_results,
20005 consumer_reads,
20006 |units| {
20007 Self::resource_loop_checkpoint(
20008 &mut self.active_resource_ledger,
20009 units,
20010 )
20011 },
20012 )?;
20013 (demand, selected_strategy, passes)
20014 }
20015 #[cfg(not(target_arch = "wasm32"))]
20016 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20017 let request_id = self
20018 .active_evaluation_resource_request
20019 .as_ref()
20020 .map_or(0, |stats| stats.request_id);
20021 match NativeTopologyScratch::create(request_id) {
20022 Ok(mut native) => match build_demand_closure_native(
20023 roots.clone(),
20024 producer_results,
20025 consumer_reads,
20026 native.file.as_mut().expect("native scratch file"),
20027 native
20028 .auxiliary_file
20029 .as_mut()
20030 .expect("native auxiliary scratch file"),
20031 |units| {
20032 Self::resource_loop_checkpoint(
20033 &mut self.active_resource_ledger,
20034 units,
20035 )
20036 },
20037 ) {
20038 Ok((demand, passes, bytes)) => {
20039 native_disk_bytes = bytes;
20040 (demand, selected_strategy, passes)
20041 }
20042 Err(NativeExactDemandError::Work(error)) => return Err(error),
20043 Err(NativeExactDemandError::Io(_)) => {
20044 let (demand, passes) = build_demand_closure_repeated_passes(
20045 roots,
20046 producer_results,
20047 consumer_reads,
20048 |units| {
20049 Self::resource_loop_checkpoint(
20050 &mut self.active_resource_ledger,
20051 units,
20052 )
20053 },
20054 )?;
20055 (
20056 demand,
20057 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20058 passes,
20059 )
20060 }
20061 },
20062 Err(_) => {
20063 let (demand, passes) = build_demand_closure_repeated_passes(
20064 roots,
20065 producer_results,
20066 consumer_reads,
20067 |units| {
20068 Self::resource_loop_checkpoint(
20069 &mut self.active_resource_ledger,
20070 units,
20071 )
20072 },
20073 )?;
20074 (
20075 demand,
20076 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20077 passes,
20078 )
20079 }
20080 }
20081 }
20082 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20083 let (demand, passes) = build_demand_closure_repeated_passes(
20084 roots,
20085 producer_results,
20086 consumer_reads,
20087 |units| {
20088 Self::resource_loop_checkpoint(
20089 &mut self.active_resource_ledger,
20090 units,
20091 )
20092 },
20093 )?;
20094 (demand, selected_strategy, passes)
20095 }
20096 _ => unreachable!("exact demand strategy selection is exhaustive"),
20097 };
20098 let strategy_memory = match actual_strategy {
20099 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20100 scratch_estimates.paged.saturating_sub(closure_base)
20101 }
20102 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20103 scratch_estimates.runs.saturating_sub(closure_base)
20104 }
20105 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20106 scratch_estimates.native.saturating_sub(closure_base)
20107 }
20108 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20109 scratch_estimates.repeated.saturating_sub(closure_base)
20110 }
20111 None => 0,
20112 _ => 0,
20113 };
20114 let actual_scratch = demand
20115 .estimated_memory_bytes()
20116 .saturating_add(strategy_memory);
20117 if actual_scratch > demand_scratch_reserved {
20118 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20119 ledger
20120 .reserve_schedule_discovery(
20121 actual_scratch.saturating_sub(demand_scratch_reserved),
20122 )
20123 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20124 }
20125 } else if actual_scratch < demand_scratch_reserved
20126 && let Some(ledger) = self.active_resource_ledger.as_mut()
20127 {
20128 ledger
20129 .release_scratch(demand_scratch_reserved - actual_scratch)
20130 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20131 }
20132 demand_scratch_reserved = actual_scratch;
20133 if let Some(strategy) = actual_strategy
20134 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20135 {
20136 if strategy.severity() > stats.topology.strategy.severity() {
20137 stats.topology.strategy = strategy;
20138 }
20139 stats.topology.exact_pass_count = stats
20140 .topology
20141 .exact_pass_count
20142 .saturating_add(passes.max(1));
20143 stats.topology.native_topology_disk_bytes = stats
20144 .topology
20145 .native_topology_disk_bytes
20146 .saturating_add(native_disk_bytes);
20147 }
20148 Some(demand)
20149 } else {
20150 None
20151 };
20152
20153 let dirty_legacy = self.graph.get_evaluation_vertices();
20154 let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
20155 let island_target_demand = target_roots.map(|roots| {
20156 use crate::engine::target_preparation::TargetProducer;
20157 let graph_roots = roots
20158 .iter()
20159 .filter_map(|root| match *root {
20160 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
20161 Some(vertex)
20162 }
20163 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
20164 })
20165 .collect::<Vec<_>>();
20166 self.build_demand_subgraph(&graph_roots)
20167 .0
20168 .into_iter()
20169 .collect::<FxHashSet<_>>()
20170 });
20171 let mut island_plan = island.clone();
20172 island_plan.dirty_vertices = island_plan
20173 .membership
20174 .iter()
20175 .copied()
20176 .filter(|vertex| dirty_legacy_set.contains(vertex))
20177 .filter(|vertex| {
20178 island_target_demand
20179 .as_ref()
20180 .is_none_or(|demand| demand.contains(vertex))
20181 })
20182 .collect();
20183 let mut work = Vec::new();
20184 let mut scheduled_legacy_vertices = Vec::new();
20185 let demanded_dirty = |producer: FormulaProducerId,
20186 dirty: ProducerDirtyDomain|
20187 -> Option<ProducerDirtyDomain> {
20188 let Some(demand) = demand.as_ref() else {
20189 return Some(dirty);
20190 };
20191 let result = producer_results.producer_result_region(producer)?;
20192 dirty.intersect_demanded(result, demand.regions(producer))
20193 };
20194 for (span_ref, region) in formula_dirty.span_regions() {
20195 let producer = FormulaProducerId::Span(span_ref.id);
20196 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20197 && let Some(dirty) =
20198 demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
20199 {
20200 work.push(FormulaProducerWork { producer, dirty });
20201 }
20202 }
20203 for span_ref in formula_dirty
20204 .whole_spans()
20205 .chain(retry_whole_spans.iter().copied())
20206 {
20207 let producer = FormulaProducerId::Span(span_ref.id);
20208 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20209 && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20210 {
20211 work.push(FormulaProducerWork { producer, dirty });
20212 }
20213 }
20214 for vertex in dirty_legacy {
20215 let producer = FormulaProducerId::Legacy(vertex);
20216 if producer_results.producer_result_region(producer).is_some()
20217 && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20218 {
20219 scheduled_legacy_vertices.push(vertex);
20220 work.push(FormulaProducerWork { producer, dirty });
20221 }
20222 }
20223
20224 let pending_changed_regions = formula_dirty
20225 .regions()
20226 .chain(formula_dirty.span_regions().map(|(_, region)| region))
20227 .collect::<Vec<_>>();
20228 if include_dirty_regions && !pending_changed_regions.is_empty() {
20229 use crate::formula_plane::producer::compute_dirty_closure_checked;
20230 let closure = compute_dirty_closure_checked(
20231 consumer_reads,
20232 pending_changed_regions.iter().copied(),
20233 |producer| producer_results.producer_result_region(producer),
20234 |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
20235 )?;
20236 if closure.incomplete {
20237 work.clear();
20240 scheduled_legacy_vertices.clear();
20241 if let Some(demand) = demand.as_ref() {
20242 for producer in demand.producers() {
20243 if let Some(dirty) =
20244 demanded_dirty(producer, ProducerDirtyDomain::Whole)
20245 {
20246 if let FormulaProducerId::Legacy(vertex) = producer {
20247 scheduled_legacy_vertices.push(vertex);
20248 }
20249 work.push(FormulaProducerWork { producer, dirty });
20250 }
20251 }
20252 } else {
20253 work.extend(span_refs_by_id.keys().copied().map(|id| {
20254 FormulaProducerWork {
20255 producer: FormulaProducerId::Span(id),
20256 dirty: ProducerDirtyDomain::Whole,
20257 }
20258 }));
20259 for vertex in self.graph.formula_vertices() {
20260 let producer = FormulaProducerId::Legacy(vertex);
20261 if producer_results.producer_result_region(producer).is_some() {
20262 scheduled_legacy_vertices.push(vertex);
20263 work.push(FormulaProducerWork {
20264 producer,
20265 dirty: ProducerDirtyDomain::Whole,
20266 });
20267 }
20268 }
20269 }
20270 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20271 stats.topology.incomplete_reason =
20272 Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
20273 }
20274 } else if let Some(demand) = demand.as_ref() {
20275 for item in closure.work {
20276 if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
20277 work.push(FormulaProducerWork {
20278 producer: item.producer,
20279 dirty,
20280 });
20281 }
20282 }
20283 } else {
20284 work.extend(closure.work);
20285 }
20286 if !closure.incomplete && !closure.fallbacks.is_empty() {
20287 let mut already_whole = work
20288 .iter()
20289 .filter_map(|item| match (item.producer, &item.dirty) {
20290 (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
20291 _ => None,
20292 })
20293 .collect::<rustc_hash::FxHashSet<_>>();
20294 for fallback in closure.fallbacks {
20295 if let FormulaProducerId::Span(id) = fallback.consumer
20296 && already_whole.insert(id)
20297 {
20298 work.push(FormulaProducerWork {
20299 producer: FormulaProducerId::Span(id),
20300 dirty: ProducerDirtyDomain::Whole,
20301 });
20302 }
20303 }
20304 }
20305 }
20306
20307 let owned_dirty_events = if let Some(demand) = demand.as_ref() {
20308 use crate::formula_plane::producer::compute_dirty_closure_checked;
20309 let mut owned = Vec::new();
20310 for (index, event) in formula_dirty.events() {
20311 let (seed_regions, initial) = match event {
20312 FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
20313 FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
20314 vec![region],
20315 Some(FormulaProducerWork {
20316 producer: FormulaProducerId::Span(span_ref.id),
20317 dirty: ProducerDirtyDomain::Regions(vec![region]),
20318 }),
20319 ),
20320 FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
20321 let producer = FormulaProducerId::Span(span_ref.id);
20322 let Some(result) = producer_results.producer_result_region(producer)
20323 else {
20324 continue;
20325 };
20326 (
20327 vec![result],
20328 Some(FormulaProducerWork {
20329 producer,
20330 dirty: ProducerDirtyDomain::Whole,
20331 }),
20332 )
20333 }
20334 };
20335 let closure = compute_dirty_closure_checked(
20336 consumer_reads,
20337 seed_regions,
20338 |producer| producer_results.producer_result_region(producer),
20339 |units| {
20340 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
20341 },
20342 )?;
20343 if closure.incomplete || !closure.fallbacks.is_empty() {
20344 continue;
20345 }
20346 let fully_demanded =
20347 initial
20348 .into_iter()
20349 .chain(closure.work.into_iter())
20350 .all(|item| {
20351 let Some(result) =
20352 producer_results.producer_result_region(item.producer)
20353 else {
20354 return false;
20355 };
20356 item.dirty
20357 .result_regions(result)
20358 .into_iter()
20359 .all(|dirty_region| {
20360 demand
20361 .regions(item.producer)
20362 .iter()
20363 .any(|demanded| demanded.contains_region(dirty_region))
20364 })
20365 });
20366 if fully_demanded {
20367 owned.push(index);
20368 }
20369 }
20370 owned
20371 } else {
20372 (0..formula_dirty.len()).collect()
20373 };
20374
20375 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
20376 let producer_count = producer_results.len();
20377 let dirty_units = work.iter().fold(0usize, |total, item| {
20378 total.saturating_add(match &item.dirty {
20379 ProducerDirtyDomain::Whole => 0,
20380 ProducerDirtyDomain::Cells(cells) => cells.len(),
20381 ProducerDirtyDomain::Regions(regions) => regions.len(),
20382 })
20383 });
20384 let work_bytes = work
20385 .len()
20386 .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
20387 .saturating_add(
20388 dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
20389 std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
20390 )),
20391 ) as u64;
20392 let binding_bytes =
20393 estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
20394 let schedule_bytes = work_bytes
20395 .saturating_mul(4)
20396 .saturating_add((producer_count as u64).saturating_mul(192))
20397 .saturating_add(binding_bytes)
20398 .saturating_add(1024);
20399 let maximum_edges =
20400 producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
20401 let paged_bytes = schedule_bytes
20405 .saturating_add(maximum_edges.saturating_mul(160))
20406 .saturating_add(16 * 1024);
20407 let runs_bytes = schedule_bytes
20408 .saturating_add(maximum_edges.saturating_mul(48))
20409 .saturating_add(8192);
20410 let native_bytes = schedule_bytes
20411 .saturating_add((producer_count as u64).saturating_mul(128))
20412 .saturating_add(2048);
20413 let repeated_bytes = schedule_bytes
20414 .saturating_add((producer_count as u64).saturating_mul(96))
20415 .saturating_add(1024);
20416 let native_allowed = self
20417 .active_resource_ledger
20418 .as_ref()
20419 .and_then(ResourceLedger::disk_scratch_policy)
20420 == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
20421 && !cfg!(target_arch = "wasm32");
20422
20423 let cached_schedule_bytes = schedule_bytes
20424 .saturating_add(
20425 cached_topology
20426 .map_or(0, |topology| topology.stats.relationships as u64)
20427 .saturating_mul(160),
20428 )
20429 .saturating_add(1024);
20430 let use_cached_schedule = cached_topology.is_some()
20431 && self.can_reserve_topology_scratch(cached_schedule_bytes);
20432 let (selected_strategy, scratch) = if use_cached_schedule {
20433 (None, cached_schedule_bytes)
20434 } else if self.can_reserve_topology_scratch(paged_bytes) {
20435 (
20436 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
20437 paged_bytes,
20438 )
20439 } else if self.can_reserve_topology_scratch(runs_bytes) {
20440 (
20441 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
20442 runs_bytes,
20443 )
20444 } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
20445 (
20446 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20447 native_bytes,
20448 )
20449 } else if self.can_reserve_topology_scratch(repeated_bytes) {
20450 (
20451 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20452 repeated_bytes,
20453 )
20454 } else {
20455 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20456 ledger
20457 .reserve_schedule_discovery(repeated_bytes)
20458 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20459 }
20460 unreachable!("failed scratch reservation must return a typed error")
20461 };
20462 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20463 ledger
20464 .reserve_schedule_discovery(scratch)
20465 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20466 }
20467 let request_id = self
20468 .active_evaluation_resource_request
20469 .as_ref()
20470 .map_or(0, |stats| stats.request_id);
20471 let build_result = (|| -> Result<_, ExcelError> {
20472 let (schedule, actual_strategy, passes, native_disk_bytes) =
20473 match (cached_topology, selected_strategy) {
20474 (Some(topology), None) => (
20475 schedule_dirty_work(work, producer_results, topology, 256),
20476 None,
20477 0,
20478 0,
20479 ),
20480 (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
20481 let (schedule, passes) = schedule_dirty_work_paged_hybrid(
20482 work,
20483 producer_results,
20484 consumer_reads,
20485 partial_topology.filter(|_| reuse_partial_topology),
20486 256,
20487 |units| {
20488 Self::resource_loop_checkpoint(
20489 &mut self.active_resource_ledger,
20490 units,
20491 )
20492 },
20493 )?;
20494 (schedule, selected_strategy, passes.max(1), 0)
20495 }
20496 (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
20497 let (schedule, passes) = schedule_dirty_work_in_memory_runs(
20498 work,
20499 producer_results,
20500 consumer_reads,
20501 256,
20502 |units| {
20503 Self::resource_loop_checkpoint(
20504 &mut self.active_resource_ledger,
20505 units,
20506 )
20507 },
20508 )?;
20509 (schedule, selected_strategy, passes.max(1), 0)
20510 }
20511 #[cfg(not(target_arch = "wasm32"))]
20512 (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
20513 let native = NativeTopologyScratch::create(request_id);
20514 if let Ok(mut native) = native {
20515 match schedule_dirty_work_native(
20516 work.clone(),
20517 producer_results,
20518 consumer_reads,
20519 256,
20520 native.file.as_mut().expect("native scratch file"),
20521 |units| {
20522 Self::resource_loop_checkpoint(
20523 &mut self.active_resource_ledger,
20524 units,
20525 )
20526 },
20527 ) {
20528 Ok((schedule, passes, topology_bytes)) => (
20529 schedule,
20530 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20531 passes.max(1),
20532 topology_bytes,
20533 ),
20534 Err(NativeExactScheduleError::Work(error)) => {
20535 return Err(error);
20536 }
20537 Err(NativeExactScheduleError::Io(_)) => {
20538 let (schedule, passes) =
20539 schedule_dirty_work_repeated_passes(
20540 work,
20541 producer_results,
20542 consumer_reads,
20543 256,
20544 |units| {
20545 Self::resource_loop_checkpoint(
20546 &mut self.active_resource_ledger,
20547 units,
20548 )
20549 },
20550 )?;
20551 (
20552 schedule,
20553 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20554 passes.max(1),
20555 0,
20556 )
20557 }
20558 }
20559 } else {
20560 let (schedule, passes) = schedule_dirty_work_repeated_passes(
20561 work,
20562 producer_results,
20563 consumer_reads,
20564 256,
20565 |units| {
20566 Self::resource_loop_checkpoint(
20567 &mut self.active_resource_ledger,
20568 units,
20569 )
20570 },
20571 )?;
20572 (
20573 schedule,
20574 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20575 passes.max(1),
20576 0,
20577 )
20578 }
20579 }
20580 (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
20581 let (schedule, passes) = schedule_dirty_work_repeated_passes(
20582 work,
20583 producer_results,
20584 consumer_reads,
20585 256,
20586 |units| {
20587 Self::resource_loop_checkpoint(
20588 &mut self.active_resource_ledger,
20589 units,
20590 )
20591 },
20592 )?;
20593 (schedule, selected_strategy, passes.max(1), 0)
20594 }
20595 _ => unreachable!(
20596 "topology cache and exact strategy selection are exhaustive"
20597 ),
20598 };
20599 Ok((
20600 schedule,
20601 actual_strategy,
20602 passes,
20603 native_disk_bytes,
20604 span_refs_by_id.clone(),
20605 ))
20606 })();
20607 let release_result = self
20608 .active_resource_ledger
20609 .as_mut()
20610 .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
20611 .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20612 let (schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
20613 match build_result {
20614 Ok(output) => {
20615 release_result?;
20616 output
20617 }
20618 Err(error) => {
20619 let _ = release_result;
20620 return Err(error);
20621 }
20622 };
20623 if let Some(strategy) = exact_strategy
20624 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20625 {
20626 if strategy.severity() > stats.topology.strategy.severity() {
20627 stats.topology.strategy = strategy;
20628 }
20629 stats.topology.exact_pass_count = pass_count;
20630 stats.topology.native_topology_disk_bytes = stats
20631 .topology
20632 .native_topology_disk_bytes
20633 .saturating_add(native_disk_bytes);
20634 }
20635 Ok((
20636 schedule,
20637 result_span_refs,
20638 plane_epoch,
20639 scheduled_legacy_vertices,
20640 owned_dirty_events,
20641 island_plan,
20642 ))
20643 })();
20644 let scratch_release_result = self
20645 .active_resource_ledger
20646 .as_mut()
20647 .map_or(Ok(()), |ledger| {
20648 ledger
20649 .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
20650 })
20651 .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20652 match schedule_result {
20653 Ok(output) => {
20654 scratch_release_result?;
20655 Ok(output)
20656 }
20657 Err(error) => {
20658 let _ = scratch_release_result;
20659 Err(error)
20660 }
20661 }
20662 }
20663}
20664
20665impl<R> Engine<R>
20666where
20667 R: EvaluationContext,
20668{
20669 fn shared_range_to_region_pattern(
20670 &self,
20671 range: &crate::reference::SharedRangeRef<'static>,
20672 ) -> Result<Option<Region>, ExcelError> {
20673 use crate::reference::SharedSheetLocator;
20674 let sheet_id = match range.sheet {
20675 SharedSheetLocator::Id(id) => id,
20676 SharedSheetLocator::Current => self.graph.default_sheet_id(),
20677 SharedSheetLocator::Name(_) => return Ok(None),
20678 };
20679 match (
20680 range.start_row,
20681 range.end_row,
20682 range.start_col,
20683 range.end_col,
20684 ) {
20685 (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
20686 sheet_id, sr.index, er.index, sc.index, ec.index,
20687 ))),
20688 (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
20689 Ok(Some(Region::whole_col(sheet_id, sc.index)))
20690 }
20691 (Some(sr), Some(er), None, None) if sr.index == er.index => {
20692 Ok(Some(Region::whole_row(sheet_id, sr.index)))
20693 }
20694 _ => Ok(None),
20695 }
20696 }
20697
20698 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
20700 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
20701 engine.evaluate_all_unobserved()
20702 })
20703 }
20704
20705 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
20706 debug_assert!(
20707 !self.graph.deferred_dirty_active(),
20708 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
20709 was not balanced by end_deferred_dirty"
20710 );
20711 self.observe_function_semantic_epoch()?;
20712 self.lookup_index_cache.reset_counters();
20713 let _source_cache = self.source_cache_session();
20714 self.validate_deterministic_mode()?;
20715 if self.config.defer_graph_building {
20716 self.build_graph_all()?;
20718 }
20719 self.evaluate_all_coordinator()
20720 }
20721
20722 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
20727 self.begin_evaluation_request();
20728 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
20729 return self.evaluate_authoritative_formula_plane_all();
20730 }
20731 self.evaluate_all_legacy_impl()
20732 }
20733
20734 fn legacy_pass_run_units(
20740 &mut self,
20741 schedule: &crate::engine::scheduler::Schedule,
20742 ) -> Result<(usize, usize), ExcelError> {
20743 let mut computed_vertices = 0;
20744 let mut cycle_count = 0;
20745 for &unit in &schedule.units {
20746 match unit {
20747 ScheduleUnit::Cycle(i) => {
20748 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
20749 cycle_count += 1;
20750 }
20751 }
20752 ScheduleUnit::Layer(i) => {
20753 let layer = schedule.unit_layer(i);
20754 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20755 computed_vertices += self.evaluate_layer_parallel(layer)?;
20756 } else {
20757 computed_vertices += self.evaluate_layer_sequential(layer)?;
20758 }
20759 }
20760 }
20761 }
20762 Ok((computed_vertices, cycle_count))
20763 }
20764
20765 fn evaluate_legacy_island_non_finalizing(
20770 &mut self,
20771 vertices: &[VertexId],
20772 mut delta: Option<&mut DeltaCollector>,
20773 ) -> Result<(usize, usize), ExcelError> {
20774 if vertices.is_empty() {
20775 return Ok((0, 0));
20776 }
20777 let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
20778 let mut computed = 0usize;
20779 let mut cycles = 0usize;
20780 for &unit in &schedule.units {
20781 self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
20782 match unit {
20783 ScheduleUnit::Cycle(index) => {
20784 if self.handle_cycle_unit(
20785 schedule.unit_cycle(index),
20786 delta.as_deref_mut(),
20787 None,
20788 None,
20789 )? > 0
20790 {
20791 cycles = cycles.saturating_add(1);
20792 }
20793 }
20794 ScheduleUnit::Layer(index) => {
20795 let layer = schedule.unit_layer(index);
20796 let evaluated = if let Some(delta) = delta.as_deref_mut() {
20797 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20798 self.evaluate_layer_parallel_with_delta(layer, delta)?
20799 } else {
20800 self.evaluate_layer_sequential_with_delta(layer, delta)?
20801 }
20802 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20803 self.evaluate_layer_parallel(layer)?
20804 } else {
20805 self.evaluate_layer_sequential(layer)?
20806 };
20807 computed = computed.saturating_add(evaluated);
20808 }
20809 }
20810 }
20811 self.graph.clear_dirty_flags(vertices);
20812 Ok((computed, cycles))
20813 }
20814
20815 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
20827 self.reset_virtual_dep_telemetry_if_disabled();
20828 #[cfg(feature = "tracing")]
20829 let _span_eval = tracing::info_span!("evaluate_all").entered();
20830 let start = crate::instant::FzInstant::now();
20831 let mut computed_vertices = 0;
20832 let mut cycle_errors = 0;
20833 let mut replan_iterations = 0;
20834 const MAX_REPLAN: usize = 5;
20835 let mut telemetry = self
20836 .config
20837 .enable_virtual_dep_telemetry
20838 .then(|| self.start_virtual_dep_telemetry());
20839
20840 loop {
20841 let to_evaluate = self.graph.get_evaluation_vertices();
20842 if to_evaluate.is_empty() {
20843 if let Some(t) = telemetry.as_mut()
20844 && t.bailout_reason.is_none()
20845 {
20846 t.bailout_reason = Some("no_work");
20847 }
20848 break;
20849 }
20850
20851 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20852 if let Some(t) = telemetry.as_mut() {
20853 Self::accumulate_schedule_meta(t, &meta);
20854 }
20855
20856 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
20857 computed_vertices += pass_computed;
20858 cycle_errors += pass_cycles;
20859
20860 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
20862 if let Some(t) = telemetry.as_mut() {
20863 t.changed_vdeps_total += changed_vertices.len();
20864 }
20865
20866 self.resource_checkpoint(0)?;
20867 self.graph.clear_dirty_flags(&to_evaluate);
20868 for v in &changed_vertices {
20869 self.graph.set_dirty(*v, true);
20870 }
20871
20872 if changed_vertices.is_empty() {
20873 if let Some(t) = telemetry.as_mut() {
20874 t.bailout_reason = Some("converged");
20875 }
20876 break;
20877 }
20878 if replan_iterations >= MAX_REPLAN {
20879 if let Some(mut t) = telemetry.take() {
20880 t.bailout_reason = Some("max_replan");
20881 t.replan_iterations = replan_iterations;
20882 self.last_virtual_dep_telemetry = t;
20883 }
20884 return Err(
20885 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
20886 );
20887 }
20888
20889 replan_iterations += 1;
20890 }
20891
20892 if let Some(mut t) = telemetry {
20893 t.replan_iterations = replan_iterations;
20894 self.last_virtual_dep_telemetry = t;
20895 }
20896
20897 self.redirty_for_next_recalc();
20899
20900 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20902
20903 Ok(EvalResult {
20904 computed_vertices,
20905 cycle_errors,
20906 elapsed: start.elapsed(),
20907 })
20908 }
20909
20910 pub fn evaluate_all_with_target_delta(
20911 &mut self,
20912 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
20913 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20914 engine.observe_function_semantic_epoch()?;
20915 let mut collector = DeltaCollector::new(DeltaMode::Cells);
20916 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20917 Ok((result, collector.finish_target()))
20918 })
20919 }
20920
20921 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
20922 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
20923 }
20924
20925 pub fn evaluate_all_with_delta_policy(
20926 &mut self,
20927 policy: EvalDeltaCompatibilityPolicy,
20928 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
20929 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20930 engine.observe_function_semantic_epoch()?;
20931 let mut collector = DeltaCollector::new(DeltaMode::Cells);
20932 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20933 Ok((result, collector.finish_with_policy(policy)?))
20934 })
20935 }
20936
20937 fn evaluate_all_with_delta_collector(
20938 &mut self,
20939 delta: &mut DeltaCollector,
20940 ) -> Result<EvalResult, ExcelError> {
20941 self.begin_evaluation_request();
20942 let _source_cache = self.source_cache_session();
20943 if self.config.defer_graph_building {
20944 self.build_graph_all()?;
20945 }
20946 if self.graph.formula_authority().active_span_count() > 0 {
20947 return self.evaluate_authoritative_formula_plane(None, Some(delta));
20948 }
20949 self.reset_virtual_dep_telemetry_if_disabled();
20950 #[cfg(feature = "tracing")]
20951 let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
20952 let start = crate::instant::FzInstant::now();
20953 let mut computed_vertices = 0;
20954 let mut cycle_errors = 0;
20955
20956 let mut replan_iterations = 0;
20957 const MAX_REPLAN: usize = 5;
20958 let mut telemetry = self
20959 .config
20960 .enable_virtual_dep_telemetry
20961 .then(|| self.start_virtual_dep_telemetry());
20962
20963 loop {
20964 let to_evaluate = self.graph.get_evaluation_vertices();
20965 if to_evaluate.is_empty() {
20966 if let Some(t) = telemetry.as_mut()
20967 && t.bailout_reason.is_none()
20968 {
20969 t.bailout_reason = Some("no_work");
20970 }
20971 break;
20972 }
20973
20974 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20975 if let Some(t) = telemetry.as_mut() {
20976 Self::accumulate_schedule_meta(t, &meta);
20977 }
20978
20979 for &unit in &schedule.units {
20980 match unit {
20981 ScheduleUnit::Cycle(i) => {
20982 if self.handle_cycle_unit(
20983 schedule.unit_cycle(i),
20984 Some(delta),
20985 None,
20986 None,
20987 )? > 0
20988 {
20989 cycle_errors += 1;
20990 }
20991 }
20992 ScheduleUnit::Layer(i) => {
20993 let layer = schedule.unit_layer(i);
20994 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20995 computed_vertices +=
20996 self.evaluate_layer_parallel_with_delta(layer, delta)?;
20997 } else {
20998 computed_vertices +=
20999 self.evaluate_layer_sequential_with_delta(layer, delta)?;
21000 }
21001 }
21002 }
21003 }
21004
21005 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21006 if let Some(t) = telemetry.as_mut() {
21007 t.changed_vdeps_total += changed_vertices.len();
21008 }
21009 self.resource_checkpoint(0)?;
21010 self.graph.clear_dirty_flags(&to_evaluate);
21011 for v in &changed_vertices {
21012 self.graph.set_dirty(*v, true);
21013 }
21014
21015 if changed_vertices.is_empty() {
21016 if let Some(t) = telemetry.as_mut() {
21017 t.bailout_reason = Some("converged");
21018 }
21019 break;
21020 }
21021 if replan_iterations >= MAX_REPLAN {
21022 if let Some(mut t) = telemetry.take() {
21023 t.bailout_reason = Some("max_replan");
21024 t.replan_iterations = replan_iterations;
21025 self.last_virtual_dep_telemetry = t;
21026 }
21027 return Err(
21028 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21029 );
21030 }
21031 replan_iterations += 1;
21032 }
21033
21034 if let Some(mut t) = telemetry {
21035 t.replan_iterations = replan_iterations;
21036 self.last_virtual_dep_telemetry = t;
21037 }
21038
21039 self.redirty_for_next_recalc();
21040 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21041
21042 Ok(EvalResult {
21043 computed_vertices,
21044 cycle_errors,
21045 elapsed: start.elapsed(),
21046 })
21047 }
21048
21049 pub fn evaluate_cell(
21059 &mut self,
21060 sheet: &str,
21061 row: u32,
21062 col: u32,
21063 ) -> Result<Option<LiteralValue>, ExcelError> {
21064 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
21065 engine.evaluate_cell_unobserved(sheet, row, col)
21066 })
21067 }
21068
21069 fn evaluate_cell_unobserved(
21070 &mut self,
21071 sheet: &str,
21072 row: u32,
21073 col: u32,
21074 ) -> Result<Option<LiteralValue>, ExcelError> {
21075 if row == 0 || col == 0 {
21076 return Err(ExcelError::new(ExcelErrorKind::Ref)
21077 .with_message("Row and column must be >= 1".to_string()));
21078 }
21079
21080 let result = self.evaluate_cells(&[(sheet, row, col)])?;
21081
21082 match result.len() {
21083 0 => Ok(None),
21084 1 => {
21085 let v = result.into_iter().next().unwrap();
21086 Ok(v)
21087 }
21088 _ => unreachable!("evaluate_cells returned unexpected length"),
21089 }
21090 }
21091
21092 pub fn evaluate_cells(
21099 &mut self,
21100 targets: &[(&str, u32, u32)],
21101 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21102 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
21103 engine.evaluate_cells_unobserved(targets)
21104 })
21105 }
21106
21107 fn evaluate_cells_unobserved(
21108 &mut self,
21109 targets: &[(&str, u32, u32)],
21110 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21111 self.observe_function_semantic_epoch()?;
21112 debug_assert!(
21113 !self.graph.deferred_dirty_active(),
21114 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
21115 was not balanced by end_deferred_dirty"
21116 );
21117 self.validate_deterministic_mode()?;
21118 if targets.is_empty() {
21119 return Ok(Vec::new());
21120 }
21121 let typed_targets = self.legacy_coordinate_targets(targets);
21122 self.evaluate_mixed_targets(&typed_targets, None)?;
21123 Ok(targets
21124 .iter()
21125 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21126 .collect())
21127 }
21128
21129 pub fn evaluate_cells_cancellable(
21130 &mut self,
21131 targets: &[(&str, u32, u32)],
21132 cancel: crate::engine::CancelToken,
21133 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21134 self.observe_evaluation_resource_request(
21135 EvaluationRequestKind::CellsCancellable,
21136 |engine| {
21137 engine.observe_function_semantic_epoch()?;
21138 engine.active_cancel_flag = Some(cancel.clone());
21139 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
21140 engine.active_cancel_flag = None;
21141 res
21142 },
21143 )
21144 }
21145
21146 fn evaluate_cells_cancellable_impl(
21147 &mut self,
21148 targets: &[(&str, u32, u32)],
21149 cancel_flag: &AtomicBool,
21150 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21151 self.validate_deterministic_mode()?;
21152 if targets.is_empty() {
21153 return Ok(Vec::new());
21154 }
21155 if cancel_flag.load(Ordering::Relaxed) {
21156 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21157 .with_message("Evaluation cancelled before target preparation"));
21158 }
21159 let typed_targets = self.legacy_coordinate_targets(targets);
21160 self.evaluate_mixed_targets(&typed_targets, None)?;
21161 Ok(targets
21162 .iter()
21163 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
21164 .collect())
21165 }
21166
21167 pub fn evaluate_cells_with_target_delta(
21168 &mut self,
21169 targets: &[(&str, u32, u32)],
21170 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
21171 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21172 engine.observe_function_semantic_epoch()?;
21173 engine.validate_deterministic_mode()?;
21174 if targets.is_empty() {
21175 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
21176 }
21177 let mut collector = DeltaCollector::new(DeltaMode::Cells);
21178 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
21179 let values = targets
21180 .iter()
21181 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
21182 .collect();
21183 Ok((values, collector.finish_target()))
21184 })
21185 }
21186
21187 pub fn evaluate_cells_with_delta(
21188 &mut self,
21189 targets: &[(&str, u32, u32)],
21190 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21191 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
21192 }
21193
21194 pub fn evaluate_cells_with_delta_policy(
21195 &mut self,
21196 targets: &[(&str, u32, u32)],
21197 policy: EvalDeltaCompatibilityPolicy,
21198 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21199 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21200 engine.evaluate_cells_with_delta_unobserved(targets, policy)
21201 })
21202 }
21203
21204 fn evaluate_cells_with_delta_unobserved(
21205 &mut self,
21206 targets: &[(&str, u32, u32)],
21207 policy: EvalDeltaCompatibilityPolicy,
21208 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21209 self.observe_function_semantic_epoch()?;
21210 self.validate_deterministic_mode()?;
21211 if targets.is_empty() {
21212 return Ok((Vec::new(), EvalDelta::default()));
21213 }
21214 let mut collector = DeltaCollector::new(DeltaMode::Cells);
21215 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
21216 let values = targets
21217 .iter()
21218 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21219 .collect();
21220 Ok((values, collector.finish_with_policy(policy)?))
21221 }
21222
21223 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
21225 if targets.is_empty() {
21226 return Ok(EvalPlan {
21227 total_vertices_to_evaluate: 0,
21228 layers: Vec::new(),
21229 cycles_detected: 0,
21230 dirty_count: 0,
21231 volatile_count: 0,
21232 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21233 estimated_parallel_layers: 0,
21234 target_cells: Vec::new(),
21235 });
21236 }
21237 if self.config.defer_graph_building && self.has_staged_formulas() {
21238 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
21239 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
21240 ));
21241 }
21242
21243 let addresses: Vec<String> = targets
21245 .iter()
21246 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
21247 .collect();
21248
21249 let mut target_addrs = Vec::new();
21251 for (sheet, row, col) in targets {
21252 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
21253 let coord = Coord::from_excel(*row, *col, true, true);
21254 target_addrs.push(CellRef::new(sheet_id, coord));
21255 }
21256 }
21257
21258 let mut target_vertex_ids = Vec::new();
21260 for addr in &target_addrs {
21261 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
21262 target_vertex_ids.push(*vertex_id);
21263 }
21264 }
21265
21266 if target_vertex_ids.is_empty() {
21267 return Ok(EvalPlan {
21268 total_vertices_to_evaluate: 0,
21269 layers: Vec::new(),
21270 cycles_detected: 0,
21271 dirty_count: 0,
21272 volatile_count: 0,
21273 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21274 estimated_parallel_layers: 0,
21275 target_cells: addresses,
21276 });
21277 }
21278
21279 let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
21281
21282 if precedents_to_eval.is_empty() {
21283 return Ok(EvalPlan {
21284 total_vertices_to_evaluate: 0,
21285 layers: Vec::new(),
21286 cycles_detected: 0,
21287 dirty_count: 0,
21288 volatile_count: 0,
21289 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21290 estimated_parallel_layers: 0,
21291 target_cells: addresses,
21292 });
21293 }
21294
21295 let mut dirty_count = 0;
21297 let mut volatile_count = 0;
21298 for &vertex_id in &precedents_to_eval {
21299 if self.graph.is_dirty(vertex_id) {
21300 dirty_count += 1;
21301 }
21302 if self.graph.is_volatile(vertex_id) {
21303 volatile_count += 1;
21304 }
21305 }
21306
21307 let scheduler = Scheduler::new(&self.graph);
21309 let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
21310
21311 let mut layers = Vec::new();
21313 let mut estimated_parallel_layers = 0;
21314 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
21315
21316 for layer in &schedule.layers {
21317 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
21318 if parallel_eligible {
21319 estimated_parallel_layers += 1;
21320 }
21321
21322 let sample_cells: Vec<String> = layer
21324 .vertices
21325 .iter()
21326 .take(5)
21327 .filter_map(|&vertex_id| {
21328 self.graph
21329 .get_cell_ref_for_vertex(vertex_id)
21330 .map(|cell_ref| {
21331 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
21332 format!(
21333 "{}!{}{}",
21334 sheet_name,
21335 Self::col_to_letters(cell_ref.coord.col()),
21336 cell_ref.coord.row() + 1
21337 )
21338 })
21339 })
21340 .collect();
21341
21342 layers.push(LayerInfo {
21343 vertex_count: layer.vertices.len(),
21344 parallel_eligible,
21345 sample_cells,
21346 });
21347 }
21348
21349 Ok(EvalPlan {
21350 total_vertices_to_evaluate: precedents_to_eval.len(),
21351 layers,
21352 cycles_detected: schedule.cycles.len(),
21353 dirty_count,
21354 volatile_count,
21355 parallel_enabled,
21356 estimated_parallel_layers,
21357 target_cells: addresses,
21358 })
21359 }
21360 fn create_evaluation_schedule(
21362 &mut self,
21363 to_evaluate: &[VertexId],
21364 ) -> Result<ScheduleBuildOutput, ExcelError> {
21365 self.graph.flush_pending_edge_deltas();
21368 if self.can_use_static_schedule_cache(to_evaluate) {
21369 if let Some(cached) = self.cached_static_schedule.as_ref()
21370 && cached.topology_epoch == self.topology_epoch
21371 && cached.candidate_vertices.as_slice() == to_evaluate
21372 {
21373 let meta = ScheduleBuildMeta {
21374 candidate_vertices: to_evaluate.len(),
21375 vdeps_vertices: 0,
21376 vdeps_edges: 0,
21377 builder_elapsed_ms: 0,
21378 used_virtual_schedule: false,
21379 schedule_cache_hit: true,
21380 schedule_cache_eligible: true,
21381 };
21382 return Ok((cached.schedule.clone(), FxHashMap::default(), meta));
21383 }
21384
21385 let (schedule, vdeps, mut meta) =
21386 self.create_evaluation_schedule_uncached(to_evaluate)?;
21387 meta.schedule_cache_hit = false;
21388 meta.schedule_cache_eligible = true;
21389 if vdeps.is_empty() {
21390 self.cached_static_schedule = Some(CachedScheduleEntry {
21391 topology_epoch: self.topology_epoch,
21392 candidate_vertices: to_evaluate.to_vec(),
21393 schedule: schedule.clone(),
21394 });
21395 }
21396 return Ok((schedule, vdeps, meta));
21397 }
21398
21399 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
21400 meta.schedule_cache_hit = false;
21401 meta.schedule_cache_eligible = false;
21402 Ok((schedule, vdeps, meta))
21403 }
21404
21405 fn create_evaluation_schedule_uncached(
21406 &self,
21407 to_evaluate: &[VertexId],
21408 ) -> Result<ScheduleBuildOutput, ExcelError> {
21409 let builder = VirtualDepBuilder::new(self);
21410 let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
21411 if self.config.enable_virtual_dep_telemetry {
21412 let build_started = crate::instant::FzInstant::now();
21413 let (vdeps, augmented) = builder.build(to_evaluate);
21414 let builder_elapsed_ms = build_started.elapsed().as_millis();
21415 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
21416 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
21417 } else {
21418 let (vdeps, augmented) = builder.build(to_evaluate);
21419 (vdeps, augmented, 0, 0)
21420 };
21421
21422 let mut final_evaluate = to_evaluate.to_vec();
21423 if !augmented.is_empty() {
21424 final_evaluate.extend(augmented);
21425 final_evaluate.sort_unstable();
21426 final_evaluate.dedup();
21427 }
21428
21429 let use_virtual = !vdeps.is_empty();
21430
21431 let scheduler = Scheduler::new(&self.graph);
21432 let schedule = if use_virtual {
21433 scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
21434 } else {
21435 scheduler.create_schedule(&final_evaluate)?
21436 };
21437
21438 let meta = ScheduleBuildMeta {
21439 candidate_vertices: to_evaluate.len(),
21440 vdeps_vertices: vdeps.len(),
21441 vdeps_edges,
21442 builder_elapsed_ms,
21443 used_virtual_schedule: use_virtual,
21444 schedule_cache_hit: false,
21445 schedule_cache_eligible: false,
21446 };
21447
21448 Ok((schedule, vdeps, meta))
21449 }
21450
21451 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
21452 !to_evaluate.is_empty()
21453 && to_evaluate.iter().copied().all(|v| {
21454 !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
21455 })
21456 }
21457
21458 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
21459 VirtualDepTelemetry {
21460 fallback_mode_activations: self.virtual_dep_fallback_activations,
21461 ..VirtualDepTelemetry::default()
21462 }
21463 }
21464
21465 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
21466 telemetry.candidate_vertices_total += meta.candidate_vertices;
21467 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
21468 telemetry.vdeps_edges_total += meta.vdeps_edges;
21469 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
21470 if meta.schedule_cache_eligible {
21471 if meta.schedule_cache_hit {
21472 telemetry.schedule_cache_hits += 1;
21473 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
21474 } else {
21475 telemetry.schedule_cache_misses += 1;
21476 }
21477 }
21478 if meta.used_virtual_schedule {
21479 telemetry.schedule_virtual_passes += 1;
21480 } else {
21481 telemetry.schedule_static_passes += 1;
21482 }
21483 }
21484
21485 fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
21486 vertices
21487 .iter()
21488 .copied()
21489 .filter(|vertex| self.graph.is_dynamic(*vertex))
21490 .filter_map(|vertex| {
21491 let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
21492 (!regions.is_empty()).then_some((vertex, regions))
21493 })
21494 .collect()
21495 }
21496
21497 fn extend_target_roots_with_dynamic_regions<'a>(
21498 &self,
21499 roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
21500 regions: impl Iterator<Item = &'a Region>,
21501 ) -> Result<(), ExcelError> {
21502 use crate::engine::target_preparation::TargetProducer;
21503 let request_id = self
21504 .active_evaluation_resource_request
21505 .as_ref()
21506 .map(|request| request.request_id);
21507 let root_count = roots.len();
21508 let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
21509 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
21510 for region in regions {
21511 for vertex in self.graph.formula_vertices() {
21512 let key = crate::formula_plane::region_index::RegionKey {
21513 sheet_id: self.graph.get_vertex_sheet_id(vertex),
21514 row: self.graph.vertex_coord(vertex).row(),
21515 col: self.graph.vertex_coord(vertex).col(),
21516 };
21517 if region.contains_key(key) {
21518 extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
21519 target_root_allocation_error(extended.len() + 1, request_id)
21520 })?;
21521 }
21522 }
21523 let authority = self.graph.formula_authority();
21524 for span_ref in authority.active_span_refs() {
21525 let Some(span) = authority.plane.spans.get(span_ref) else {
21526 continue;
21527 };
21528 let Some(demanded) =
21529 Region::from_domain(span.result_region.domain()).intersection(*region)
21530 else {
21531 continue;
21532 };
21533 extended
21534 .push(TargetProducer::Span { span_ref, demanded })
21535 .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
21536 }
21537 }
21538 *roots = extended.into_vec();
21539 Ok(())
21540 }
21541
21542 fn changed_virtual_dep_vertices(
21543 &mut self,
21544 to_evaluate: &[VertexId],
21545 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
21546 ) -> Vec<VertexId> {
21547 #[cfg(test)]
21548 if self.force_virtual_dep_changes_remaining_for_test > 0
21549 && let Some(vertex) = to_evaluate.first().copied()
21550 {
21551 self.force_virtual_dep_changes_remaining_for_test -= 1;
21552 return vec![vertex];
21553 }
21554 if !to_evaluate
21555 .iter()
21556 .copied()
21557 .any(|v| self.graph.is_dynamic(v))
21558 {
21559 return Vec::new();
21560 }
21561
21562 let builder = VirtualDepBuilder::new(self);
21563 let (new_vdeps, _) = builder.build(to_evaluate);
21564
21565 let mut candidates = FxHashSet::default();
21566 candidates.extend(old_vdeps.keys().copied());
21567 candidates.extend(new_vdeps.keys().copied());
21568
21569 let mut changed = Vec::new();
21570 for v in candidates {
21571 if old_vdeps.get(&v) != new_vdeps.get(&v) {
21572 changed.push(v);
21573 }
21574 }
21575 changed
21576 }
21577
21578 fn build_demand_subgraph(
21581 &self,
21582 target_vertices: &[VertexId],
21583 ) -> (
21584 Vec<VertexId>,
21585 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
21586 ) {
21587 #[cfg(feature = "tracing")]
21588 let _span =
21589 tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
21590 use rustc_hash::{FxHashMap, FxHashSet};
21591
21592 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
21593 let mut visited: FxHashSet<VertexId> = FxHashSet::default();
21594 let mut stack: Vec<VertexId> = Vec::new();
21595 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); for &t in target_vertices {
21598 stack.push(t);
21599 }
21600
21601 while let Some(v) = stack.pop() {
21602 if !visited.insert(v) {
21603 continue;
21604 }
21605 if !self.graph.vertex_exists(v) {
21606 continue;
21607 }
21608 match self.graph.get_vertex_kind(v) {
21616 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
21617 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
21618 to_evaluate.insert(v);
21619 }
21620 }
21621 VertexKind::NamedScalar
21622 | VertexKind::NamedArray
21623 | VertexKind::Range
21624 | VertexKind::InfiniteRange => {
21625 to_evaluate.insert(v);
21626 }
21627 _ => {}
21628 }
21629
21630 if let Some(dependencies) = self.graph.dependencies_slice(v) {
21641 for &dep in dependencies {
21642 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21643 stack.push(dep);
21644 }
21645 }
21646 } else {
21647 for dep in self.graph.get_dependencies(v) {
21648 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21649 stack.push(dep);
21650 }
21651 }
21652 } let builder = VirtualDepBuilder::new(self);
21654 let (vdeps_map, _) = builder.build(&[v]);
21655 if let Some(deps) = vdeps_map.get(&v) {
21656 for &u in deps {
21657 vdeps.entry(v).or_default().push(u);
21658 if !visited.contains(&u) {
21659 stack.push(u);
21660 }
21661 }
21662 }
21663 }
21664
21665 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
21666 result.sort_unstable();
21667 for deps in vdeps.values_mut() {
21669 deps.sort_unstable();
21670 deps.dedup();
21671 }
21672 (result, vdeps)
21673 }
21674
21675 fn col_to_letters(col: u32) -> String {
21677 col_letters_from_1based(col).expect("column index must be >= 1")
21678 }
21679
21680 pub fn evaluate_all_cancellable(
21682 &mut self,
21683 cancel: crate::engine::CancelToken,
21684 ) -> Result<EvalResult, ExcelError> {
21685 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
21686 engine.observe_function_semantic_epoch()?;
21687 engine.active_cancel_flag = Some(cancel.clone());
21688 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
21689 engine.active_cancel_flag = None;
21690 res
21691 })
21692 }
21693
21694 fn evaluate_all_cancellable_impl(
21695 &mut self,
21696 cancel_flag: &AtomicBool,
21697 ) -> Result<EvalResult, ExcelError> {
21698 self.begin_evaluation_request();
21699 let _source_cache = self.source_cache_session();
21700 self.validate_deterministic_mode()?;
21701 if self.config.defer_graph_building {
21702 self.build_graph_all()?;
21703 }
21704 if self.graph.formula_authority().active_span_count() > 0 {
21705 if cancel_flag.load(Ordering::Relaxed) {
21706 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21707 "Evaluation cancelled before FormulaPlane scheduling".to_string(),
21708 ));
21709 }
21710 return self.evaluate_authoritative_formula_plane_all();
21711 }
21712 self.reset_virtual_dep_telemetry_if_disabled();
21713 let start = crate::instant::FzInstant::now();
21714 let mut computed_vertices = 0;
21715 let mut cycle_errors = 0;
21716
21717 let mut replan_iterations = 0;
21718 const MAX_REPLAN: usize = 5;
21719 let mut telemetry = self
21720 .config
21721 .enable_virtual_dep_telemetry
21722 .then(|| self.start_virtual_dep_telemetry());
21723
21724 loop {
21725 if cancel_flag.load(Ordering::Relaxed) {
21726 if let Some(mut t) = telemetry {
21727 t.bailout_reason = Some("cancelled");
21728 t.replan_iterations = replan_iterations;
21729 self.last_virtual_dep_telemetry = t;
21730 }
21731 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21732 .with_message("Evaluation cancelled before scheduling".to_string()));
21733 }
21734
21735 let to_evaluate = self.graph.get_evaluation_vertices();
21736 if to_evaluate.is_empty() {
21737 if let Some(t) = telemetry.as_mut()
21738 && t.bailout_reason.is_none()
21739 {
21740 t.bailout_reason = Some("no_work");
21741 }
21742 break;
21743 }
21744
21745 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21746 if let Some(t) = telemetry.as_mut() {
21747 Self::accumulate_schedule_meta(t, &meta);
21748 }
21749
21750 for &unit in &schedule.units {
21753 match unit {
21754 ScheduleUnit::Cycle(i) => {
21755 if cancel_flag.load(Ordering::Relaxed) {
21757 if let Some(mut t) = telemetry {
21758 t.bailout_reason = Some("cancelled");
21759 t.replan_iterations = replan_iterations;
21760 self.last_virtual_dep_telemetry = t;
21761 }
21762 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21763 "Evaluation cancelled during cycle handling".to_string(),
21764 ));
21765 }
21766
21767 if self.handle_cycle_unit(
21768 schedule.unit_cycle(i),
21769 None,
21770 None,
21771 Some(cancel_flag),
21772 )? > 0
21773 {
21774 cycle_errors += 1;
21775 }
21776 }
21777 ScheduleUnit::Layer(i) => {
21778 let layer = schedule.unit_layer(i);
21779 if cancel_flag.load(Ordering::Relaxed) {
21781 if let Some(mut t) = telemetry {
21782 t.bailout_reason = Some("cancelled");
21783 t.replan_iterations = replan_iterations;
21784 self.last_virtual_dep_telemetry = t;
21785 }
21786 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21787 .with_message("Evaluation cancelled between layers".to_string()));
21788 }
21789
21790 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21792 computed_vertices +=
21793 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
21794 } else {
21795 computed_vertices +=
21796 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
21797 }
21798 }
21799 }
21800 }
21801
21802 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21803 if let Some(t) = telemetry.as_mut() {
21804 t.changed_vdeps_total += changed_vertices.len();
21805 }
21806 self.resource_checkpoint(0)?;
21807 self.graph.clear_dirty_flags(&to_evaluate);
21808 for v in &changed_vertices {
21809 self.graph.set_dirty(*v, true);
21810 }
21811
21812 if changed_vertices.is_empty() {
21813 if let Some(t) = telemetry.as_mut() {
21814 t.bailout_reason = Some("converged");
21815 }
21816 break;
21817 }
21818 if replan_iterations >= MAX_REPLAN {
21819 if let Some(mut t) = telemetry.take() {
21820 t.bailout_reason = Some("max_replan");
21821 t.replan_iterations = replan_iterations;
21822 self.last_virtual_dep_telemetry = t;
21823 }
21824 return Err(
21825 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21826 );
21827 }
21828 replan_iterations += 1;
21829 }
21830
21831 if let Some(mut t) = telemetry {
21832 t.replan_iterations = replan_iterations;
21833 self.last_virtual_dep_telemetry = t;
21834 }
21835
21836 self.redirty_for_next_recalc();
21838 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21839
21840 Ok(EvalResult {
21841 computed_vertices,
21842 cycle_errors,
21843 elapsed: start.elapsed(),
21844 })
21845 }
21846
21847 pub fn evaluate_until_cancellable(
21849 &mut self,
21850 targets: &[&str],
21851 cancel: crate::engine::CancelToken,
21852 ) -> Result<EvalResult, ExcelError> {
21853 self.observe_evaluation_resource_request(
21854 EvaluationRequestKind::TargetedCancellable,
21855 |engine| {
21856 engine.observe_function_semantic_epoch()?;
21857 engine.active_cancel_flag = Some(cancel.clone());
21858 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
21859 engine.active_cancel_flag = None;
21860 res
21861 },
21862 )
21863 }
21864
21865 fn evaluate_until_cancellable_impl(
21866 &mut self,
21867 targets: &[&str],
21868 cancel_flag: &AtomicBool,
21869 ) -> Result<EvalResult, ExcelError> {
21870 if cancel_flag.load(Ordering::Relaxed) {
21871 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21872 .with_message("Evaluation cancelled before target preparation"));
21873 }
21874 let mut typed_targets = Vec::with_capacity(targets.len());
21875 for target in targets {
21876 let (sheet, row, col) = self.parse_a1_notation(target)?;
21877 self.graph.sheet_id_mut(&sheet);
21878 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
21879 }
21880 self.evaluate_mixed_targets(&typed_targets, None)
21881 }
21882
21883 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
21884 let mut quoted = false;
21885 let mut separator = None;
21886 let bytes = address.as_bytes();
21887 let mut index = 0usize;
21888 while index < bytes.len() {
21889 match bytes[index] {
21890 b'\'' => {
21891 if quoted && bytes.get(index + 1) == Some(&b'\'') {
21892 index = index.saturating_add(1);
21893 } else {
21894 quoted = !quoted;
21895 }
21896 }
21897 b'!' if !quoted => separator = Some(index),
21898 _ => {}
21899 }
21900 index = index.saturating_add(1);
21901 }
21902 if quoted {
21903 return Err(ExcelError::new(ExcelErrorKind::Ref)
21904 .with_message(format!("Invalid quoted sheet reference `{address}`")));
21905 }
21906 let (sheet, cell_part) = match separator {
21907 Some(separator) => {
21908 let raw_sheet = &address[..separator];
21909 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
21910 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
21911 } else {
21912 raw_sheet.to_string()
21913 };
21914 (sheet, &address[separator + 1..])
21915 }
21916 None => (self.default_sheet_name().to_string(), address),
21917 };
21918
21919 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
21920 ExcelError::new(ExcelErrorKind::Ref)
21921 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
21922 })?;
21923
21924 Ok((sheet, row, col))
21925 }
21926
21927 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
21929 use formualizer_parse::parser::ASTNodeType;
21930 match &ast.node_type {
21931 ASTNodeType::Function { name, args, .. } => {
21932 if let Some(func) = self
21933 .get_function("", name)
21934 .or_else(|| crate::function_registry::get("", name))
21935 && func.caps().contains(crate::function::FnCaps::VOLATILE)
21936 {
21937 return true;
21938 }
21939 args.iter()
21940 .any(|arg| self.is_ast_volatile_with_provider(arg))
21941 }
21942 ASTNodeType::BinaryOp { left, right, .. } => {
21943 self.is_ast_volatile_with_provider(left)
21944 || self.is_ast_volatile_with_provider(right)
21945 }
21946 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
21947 ASTNodeType::Array(rows) => rows.iter().any(|row| {
21948 row.iter()
21949 .any(|cell| self.is_ast_volatile_with_provider(cell))
21950 }),
21951 _ => false,
21952 }
21953 }
21954
21955 fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
21957 let mut to_evaluate = FxHashSet::default();
21958 let mut visited = FxHashSet::default();
21959 let mut stack = Vec::new();
21960
21961 for &target in target_vertices {
21963 stack.push(target);
21964 }
21965
21966 while let Some(vertex_id) = stack.pop() {
21967 if !visited.insert(vertex_id) {
21968 continue; }
21970
21971 if self.graph.vertex_exists(vertex_id) {
21972 let kind = self.graph.get_vertex_kind(vertex_id);
21974 let needs_eval = match kind {
21975 super::vertex::VertexKind::FormulaScalar
21976 | super::vertex::VertexKind::FormulaArray => {
21977 self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
21978 }
21979 _ => false, };
21981
21982 if needs_eval {
21983 to_evaluate.insert(vertex_id);
21984 }
21985
21986 if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
21988 for &dep_id in dependencies {
21989 if !visited.contains(&dep_id) {
21990 stack.push(dep_id);
21991 }
21992 }
21993 } else {
21994 let dependencies = self.graph.get_dependencies(vertex_id);
21995 for dep_id in dependencies {
21996 if !visited.contains(&dep_id) {
21997 stack.push(dep_id);
21998 }
21999 }
22000 }
22001 }
22002 }
22003
22004 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
22005 result.sort_unstable();
22006 result
22007 }
22008
22009 fn evaluate_layer_sequential(
22011 &mut self,
22012 layer: &super::scheduler::Layer,
22013 ) -> Result<usize, ExcelError> {
22014 self.resource_checkpoint(layer.vertices.len() as u64)?;
22015 self.evaluate_layer_sequential_effects(layer)
22016 }
22017
22018 fn update_vertex_value_with_delta(
22019 &mut self,
22020 vertex_id: VertexId,
22021 new_value: LiteralValue,
22022 delta: &mut DeltaCollector,
22023 ) {
22024 if delta.mode != DeltaMode::Off
22025 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
22026 {
22027 let sheet_name = self.graph.sheet_name(cell.sheet_id);
22028 let old = self
22029 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22030 .unwrap_or(LiteralValue::Empty);
22031 if old != new_value {
22032 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
22033 }
22034 }
22035 self.graph.update_vertex_value(vertex_id, new_value.clone());
22036 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
22037 }
22038
22039 fn evaluate_layer_sequential_with_delta(
22040 &mut self,
22041 layer: &super::scheduler::Layer,
22042 delta: &mut DeltaCollector,
22043 ) -> Result<usize, ExcelError> {
22044 self.resource_checkpoint(layer.vertices.len() as u64)?;
22045 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
22046 }
22047
22048 fn evaluate_layer_sequential_cancellable(
22050 &mut self,
22051 layer: &super::scheduler::Layer,
22052 cancel_flag: &AtomicBool,
22053 ) -> Result<usize, ExcelError> {
22054 self.resource_checkpoint(layer.vertices.len() as u64)?;
22055 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
22056 }
22057
22058 fn evaluate_layer_sequential_cancellable_demand_driven(
22060 &mut self,
22061 layer: &super::scheduler::Layer,
22062 cancel_flag: &AtomicBool,
22063 ) -> Result<usize, ExcelError> {
22064 self.resource_checkpoint(layer.vertices.len() as u64)?;
22065 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
22066 }
22067
22068 fn evaluate_layer_parallel(
22070 &mut self,
22071 layer: &super::scheduler::Layer,
22072 ) -> Result<usize, ExcelError> {
22073 self.resource_checkpoint(layer.vertices.len() as u64)?;
22074 self.evaluate_layer_parallel_effects(layer)
22075 }
22076
22077 fn evaluate_layer_parallel_with_delta(
22078 &mut self,
22079 layer: &super::scheduler::Layer,
22080 delta: &mut DeltaCollector,
22081 ) -> Result<usize, ExcelError> {
22082 self.resource_checkpoint(layer.vertices.len() as u64)?;
22083 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
22084 }
22085
22086 fn evaluate_layer_parallel_cancellable(
22088 &mut self,
22089 layer: &super::scheduler::Layer,
22090 cancel_flag: &AtomicBool,
22091 ) -> Result<usize, ExcelError> {
22092 self.resource_checkpoint(layer.vertices.len() as u64)?;
22093 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
22094 }
22095
22096 fn apply_parallel_vertex_result(
22101 &mut self,
22102 vertex_id: VertexId,
22103 result: LiteralValue,
22104 mut delta: Option<&mut DeltaCollector>,
22105 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22106 ) -> Result<(), ExcelError> {
22107 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
22110 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
22111 && owner != vertex_id
22112 {
22113 return Ok(());
22114 }
22115
22116 let kind = self.graph.get_vertex_kind(vertex_id);
22117
22118 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
22120 if is_formula {
22121 match result {
22122 LiteralValue::Array(rows) => {
22123 self.apply_array_result_from_parallel(
22124 vertex_id,
22125 rows,
22126 delta.as_deref_mut(),
22127 overwritable_formulas,
22128 )?;
22129 }
22130 other => {
22131 self.apply_non_array_result_from_parallel(
22132 vertex_id,
22133 other,
22134 delta.as_deref_mut(),
22135 );
22136 }
22137 }
22138 return Ok(());
22139 }
22140
22141 if let Some(d) = delta {
22143 self.update_vertex_value_with_delta(vertex_id, result, d);
22144 } else {
22145 self.graph.update_vertex_value(vertex_id, result.clone());
22146 self.mirror_vertex_value_to_overlay(vertex_id, &result);
22147 }
22148 Ok(())
22149 }
22150
22151 fn apply_non_array_result_from_parallel(
22152 &mut self,
22153 vertex_id: VertexId,
22154 value: LiteralValue,
22155 delta: Option<&mut DeltaCollector>,
22156 ) {
22157 let spill_cells = self
22160 .graph
22161 .spill_cells_for_anchor(vertex_id)
22162 .map(|cells| cells.to_vec())
22163 .unwrap_or_default();
22164
22165 if let Some(d) = delta
22166 && d.mode != DeltaMode::Off
22167 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
22168 {
22169 if spill_cells.is_empty() {
22170 let old = self
22171 .read_cell_value(
22172 self.graph.sheet_name(anchor.sheet_id),
22173 anchor.coord.row() + 1,
22174 anchor.coord.col() + 1,
22175 )
22176 .unwrap_or(LiteralValue::Empty);
22177 if old != value {
22178 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22179 }
22180 } else {
22181 for cell in spill_cells.iter() {
22182 let sheet_name = self.graph.sheet_name(cell.sheet_id);
22183 let old = self
22184 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22185 .unwrap_or(LiteralValue::Empty);
22186 let new = if cell.sheet_id == anchor.sheet_id
22187 && cell.coord.row() == anchor.coord.row()
22188 && cell.coord.col() == anchor.coord.col()
22189 {
22190 value.clone()
22191 } else {
22192 LiteralValue::Empty
22193 };
22194 Self::record_cell_if_changed(d, cell, &old, &new);
22195 }
22196 }
22197 }
22198
22199 self.graph.clear_spill_region(vertex_id);
22200 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
22201 self.record_formula_plane_structural_change(scope);
22202 }
22203
22204 if self.config.arrow_storage_enabled
22205 && self.config.delta_overlay_enabled
22206 && self.config.write_formula_overlay_enabled
22207 {
22208 let empty = LiteralValue::Empty;
22209 for cell in spill_cells.iter() {
22210 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
22211 self.mirror_value_to_computed_overlay(
22212 &sheet_name,
22213 cell.coord.row() + 1,
22214 cell.coord.col() + 1,
22215 &empty,
22216 );
22217 }
22218 }
22219
22220 self.graph.update_vertex_value(vertex_id, value.clone());
22221 self.mirror_vertex_value_to_overlay(vertex_id, &value);
22222 }
22223
22224 fn apply_array_result_from_parallel(
22225 &mut self,
22226 vertex_id: VertexId,
22227 rows: Vec<Vec<LiteralValue>>,
22228 mut delta: Option<&mut DeltaCollector>,
22229 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22230 ) -> Result<(), ExcelError> {
22231 self.graph
22233 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
22234
22235 let anchor = self
22236 .graph
22237 .get_cell_ref(vertex_id)
22238 .expect("cell ref for vertex");
22239 let sheet_id = anchor.sheet_id;
22240 let h = rows.len() as u32;
22241 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
22242
22243 let spill_cells = (h as u64).saturating_mul(w as u64);
22245 if spill_cells > self.config.spill.max_spill_cells as u64 {
22246 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22247 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22248 .with_message("SpillTooLarge")
22249 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22250 expected_rows: h,
22251 expected_cols: w,
22252 });
22253 let spill_val = LiteralValue::Error(spill_err.clone());
22254 if let Some(d) = delta.as_deref_mut()
22255 && d.mode != DeltaMode::Off
22256 {
22257 let old = self
22258 .read_cell_value(
22259 self.graph.sheet_name(anchor.sheet_id),
22260 anchor.coord.row() + 1,
22261 anchor.coord.col() + 1,
22262 )
22263 .unwrap_or(LiteralValue::Empty);
22264 if old != spill_val {
22265 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22266 }
22267 }
22268 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22269 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22270 return Ok(());
22271 }
22272
22273 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
22277 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
22278 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
22279 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22280 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22281 .with_message("Spill exceeds sheet bounds")
22282 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22283 expected_rows: h,
22284 expected_cols: w,
22285 });
22286 let spill_val = LiteralValue::Error(spill_err.clone());
22287 if let Some(d) = delta.as_deref_mut()
22288 && d.mode != DeltaMode::Off
22289 {
22290 let old = self
22291 .read_cell_value(
22292 self.graph.sheet_name(anchor.sheet_id),
22293 anchor.coord.row() + 1,
22294 anchor.coord.col() + 1,
22295 )
22296 .unwrap_or(LiteralValue::Empty);
22297 if old != spill_val {
22298 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22299 }
22300 }
22301 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22302 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22303 return Ok(());
22304 }
22305
22306 let mut targets = Vec::new();
22307 for r in 0..h {
22308 for c in 0..w {
22309 targets.push(self.graph.make_cell_ref_internal(
22310 sheet_id,
22311 anchor.coord.row() + r,
22312 anchor.coord.col() + c,
22313 ));
22314 }
22315 }
22316
22317 match self.spill_mgr.reserve(
22318 vertex_id,
22319 anchor,
22320 SpillShape { rows: h, cols: w },
22321 SpillMeta {
22322 epoch: self.recalc_epoch,
22323 config: self.config.spill,
22324 },
22325 ) {
22326 Ok(()) => {
22327 if let Err(e) = self.commit_spill_and_mirror(
22328 vertex_id,
22329 &targets,
22330 rows.clone(),
22331 delta.as_deref_mut(),
22332 overwritable_formulas,
22333 ) {
22334 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22335 let err_val = LiteralValue::Error(e.clone());
22336 if let Some(d) = delta.as_deref_mut()
22337 && d.mode != DeltaMode::Off
22338 {
22339 let old = self
22340 .read_cell_value(
22341 self.graph.sheet_name(anchor.sheet_id),
22342 anchor.coord.row() + 1,
22343 anchor.coord.col() + 1,
22344 )
22345 .unwrap_or(LiteralValue::Empty);
22346 if old != err_val {
22347 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22348 }
22349 }
22350 self.graph.update_vertex_value(vertex_id, err_val.clone());
22351 self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
22352 return Ok(());
22353 }
22354
22355 let top_left = rows
22357 .first()
22358 .and_then(|r| r.first())
22359 .cloned()
22360 .unwrap_or(LiteralValue::Empty);
22361 self.graph.update_vertex_value(vertex_id, top_left.clone());
22362 self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
22363 Ok(())
22364 }
22365 Err(e) => {
22366 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22367 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22368 .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
22369 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22370 expected_rows: h,
22371 expected_cols: w,
22372 });
22373 let spill_val = LiteralValue::Error(spill_err.clone());
22374 if let Some(d) = delta
22375 && d.mode != DeltaMode::Off
22376 {
22377 let old = self
22378 .read_cell_value(
22379 self.graph.sheet_name(anchor.sheet_id),
22380 anchor.coord.row() + 1,
22381 anchor.coord.col() + 1,
22382 )
22383 .unwrap_or(LiteralValue::Empty);
22384 if old != spill_val {
22385 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22386 }
22387 }
22388 self.graph.update_vertex_value(vertex_id, spill_val.clone());
22389 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22390 Ok(())
22391 }
22392 }
22393 }
22394
22395 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
22397 if !self.graph.vertex_exists(vertex_id) {
22399 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
22400 .with_message(format!("Vertex not found: {vertex_id:?}")));
22401 }
22402
22403 let kind = self.graph.get_vertex_kind(vertex_id);
22405 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
22406
22407 let ast_id = match kind {
22408 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
22409 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
22410 ast_id
22411 } else {
22412 return Ok(LiteralValue::Number(0.0));
22413 }
22414 }
22415 VertexKind::Empty | VertexKind::Cell => {
22416 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
22417 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22418 let row = cell_ref.coord.row() + 1;
22419 let col = cell_ref.coord.col() + 1;
22420 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
22421 return Ok(v);
22422 }
22423 }
22424 return Ok(LiteralValue::Number(0.0));
22425 }
22426 VertexKind::NamedScalar => {
22427 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22428 ExcelError::new(ExcelErrorKind::Name)
22429 .with_message("Named range metadata missing".to_string())
22430 })?;
22431
22432 return match &named_range.definition {
22433 NamedDefinition::Cell(cell_ref) => {
22434 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22435 Ok(self
22436 .get_cell_value(
22437 sheet_name,
22438 cell_ref.coord.row() + 1,
22439 cell_ref.coord.col() + 1,
22440 )
22441 .unwrap_or(LiteralValue::Empty))
22442 }
22443 NamedDefinition::Literal(v) => Ok(v.clone()),
22444 NamedDefinition::Formula { ast, .. } => {
22445 let context_sheet = match named_range.scope {
22446 NameScope::Sheet(id) => id,
22447 NameScope::Workbook => sheet_id,
22448 };
22449 let sheet_name = self.graph.sheet_name(context_sheet);
22450 let cell_ref = self
22451 .graph
22452 .get_cell_ref(vertex_id)
22453 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22454 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22455 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
22456 }
22457 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
22458 .with_message("Range-valued name evaluated as scalar".to_string())),
22459 };
22460 }
22461 VertexKind::NamedArray => {
22462 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22463 ExcelError::new(ExcelErrorKind::Name)
22464 .with_message("Named range metadata missing".to_string())
22465 })?;
22466
22467 return match &named_range.definition {
22468 NamedDefinition::Range(range_ref) => {
22469 if range_ref.start.sheet_id != range_ref.end.sheet_id {
22470 return Err(ExcelError::new(ExcelErrorKind::Ref)
22471 .with_message("Named range cannot span sheets".to_string()));
22472 }
22473 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
22474 let sr0 = range_ref.start.coord.row();
22475 let sc0 = range_ref.start.coord.col();
22476 let er0 = range_ref.end.coord.row();
22477 let ec0 = range_ref.end.coord.col();
22478 if sr0 > er0 || sc0 > ec0 {
22479 return Err(ExcelError::new(ExcelErrorKind::Ref)
22480 .with_message("Invalid named range bounds".to_string()));
22481 }
22482
22483 let h = (er0 - sr0 + 1) as usize;
22484 let w = (ec0 - sc0 + 1) as usize;
22485 let cell_count = (h as u64).saturating_mul(w as u64);
22486 if cell_count > self.config.spill.max_spill_cells as u64 {
22487 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
22488 "Named range too large to materialize as an array".to_string(),
22489 ));
22490 }
22491
22492 let mut rows = Vec::with_capacity(h);
22493 for r0 in sr0..=er0 {
22494 let mut row = Vec::with_capacity(w);
22495 for c0 in sc0..=ec0 {
22496 let v = self
22497 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
22498 .unwrap_or(LiteralValue::Empty);
22499 row.push(v);
22500 }
22501 rows.push(row);
22502 }
22503 Ok(LiteralValue::Array(rows))
22504 }
22505 NamedDefinition::Cell(cell_ref) => {
22506 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22507 let row = cell_ref.coord.row() + 1;
22508 let col = cell_ref.coord.col() + 1;
22509 let v = self
22510 .get_cell_value(sheet_name, row, col)
22511 .unwrap_or(LiteralValue::Empty);
22512 Ok(LiteralValue::Array(vec![vec![v]]))
22513 }
22514 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
22515 NamedDefinition::Formula { ast, .. } => {
22516 let context_sheet = match named_range.scope {
22517 NameScope::Sheet(id) => id,
22518 NameScope::Workbook => sheet_id,
22519 };
22520 let sheet_name = self.graph.sheet_name(context_sheet);
22521 let cell_ref = self
22522 .graph
22523 .get_cell_ref(vertex_id)
22524 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22525 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22526 match interpreter.evaluate_ast(ast) {
22527 Ok(cv) => {
22528 let v = cv.into_literal();
22529 match v {
22530 LiteralValue::Array(_) => Ok(v),
22531 other => Ok(LiteralValue::Array(vec![vec![other]])),
22532 }
22533 }
22534 Err(err) => Ok(LiteralValue::Error(err)),
22535 }
22536 }
22537 };
22538 }
22539 VertexKind::InfiniteRange
22540 | VertexKind::Range
22541 | VertexKind::External
22542 | VertexKind::Table => {
22543 return Ok(LiteralValue::Number(0.0));
22545 }
22546 };
22547
22548 let sheet_name = self.graph.sheet_name(sheet_id);
22550 let cell_ref = self
22551 .graph
22552 .get_cell_ref(vertex_id)
22553 .expect("cell ref for vertex");
22554 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22555
22556 interpreter
22557 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
22558 .map(|cv| {
22559 crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
22560 })
22561 }
22562
22563 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
22565 self.thread_pool.as_ref()
22566 }
22567}
22568
22569#[derive(Default)]
22570struct RowBoundsCache {
22571 snapshot: u64,
22572 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
22574}
22575
22576impl RowBoundsCache {
22577 fn new(snapshot: u64) -> Self {
22578 Self {
22579 snapshot,
22580 map: Default::default(),
22581 }
22582 }
22583 fn get_row_bounds(
22584 &self,
22585 sheet_id: SheetId,
22586 col_idx: usize,
22587 snapshot: u64,
22588 ) -> Option<(Option<u32>, Option<u32>)> {
22589 if self.snapshot != snapshot {
22590 return None;
22591 }
22592 self.map.get(&(sheet_id as u32, col_idx)).copied()
22593 }
22594 fn put_row_bounds(
22595 &mut self,
22596 sheet_id: SheetId,
22597 col_idx: usize,
22598 snapshot: u64,
22599 bounds: (Option<u32>, Option<u32>),
22600 ) {
22601 if self.snapshot != snapshot {
22602 self.snapshot = snapshot;
22603 self.map.clear();
22604 }
22605 self.map.insert((sheet_id as u32, col_idx), bounds);
22606 }
22607}
22608
22609struct UsedAxisBoundsCache {
22610 snapshot: u64,
22611 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22612 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22613 #[cfg(test)]
22614 row_hits: std::sync::atomic::AtomicUsize,
22615 #[cfg(test)]
22616 row_misses: std::sync::atomic::AtomicUsize,
22617 #[cfg(test)]
22618 col_hits: std::sync::atomic::AtomicUsize,
22619 #[cfg(test)]
22620 col_misses: std::sync::atomic::AtomicUsize,
22621}
22622
22623impl UsedAxisBoundsCache {
22624 fn new(snapshot: u64) -> Self {
22625 Self {
22626 snapshot,
22627 row_bounds_by_col_span: Default::default(),
22628 col_bounds_by_row_span: Default::default(),
22629 #[cfg(test)]
22630 row_hits: std::sync::atomic::AtomicUsize::new(0),
22631 #[cfg(test)]
22632 row_misses: std::sync::atomic::AtomicUsize::new(0),
22633 #[cfg(test)]
22634 col_hits: std::sync::atomic::AtomicUsize::new(0),
22635 #[cfg(test)]
22636 col_misses: std::sync::atomic::AtomicUsize::new(0),
22637 }
22638 }
22639
22640 fn reset_for_snapshot(&mut self, snapshot: u64) {
22641 if self.snapshot != snapshot {
22642 self.snapshot = snapshot;
22643 self.row_bounds_by_col_span.clear();
22644 self.col_bounds_by_row_span.clear();
22645 }
22646 }
22647
22648 fn get_row_bounds(
22649 &self,
22650 sheet_id: SheetId,
22651 start_col: u32,
22652 end_col: u32,
22653 snapshot: u64,
22654 ) -> Option<Option<(u32, u32)>> {
22655 if self.snapshot != snapshot {
22656 return None;
22657 }
22658 let cached = self
22659 .row_bounds_by_col_span
22660 .get(&(sheet_id, start_col, end_col))
22661 .copied();
22662 #[cfg(test)]
22663 if cached.is_some() {
22664 self.row_hits.fetch_add(1, Ordering::Relaxed);
22665 }
22666 cached
22667 }
22668
22669 fn put_row_bounds(
22670 &mut self,
22671 sheet_id: SheetId,
22672 start_col: u32,
22673 end_col: u32,
22674 snapshot: u64,
22675 bounds: Option<(u32, u32)>,
22676 ) {
22677 self.reset_for_snapshot(snapshot);
22678 self.row_bounds_by_col_span
22679 .insert((sheet_id, start_col, end_col), bounds);
22680 #[cfg(test)]
22681 self.row_misses.fetch_add(1, Ordering::Relaxed);
22682 }
22683
22684 fn get_col_bounds(
22685 &self,
22686 sheet_id: SheetId,
22687 start_row: u32,
22688 end_row: u32,
22689 snapshot: u64,
22690 ) -> Option<Option<(u32, u32)>> {
22691 if self.snapshot != snapshot {
22692 return None;
22693 }
22694 let cached = self
22695 .col_bounds_by_row_span
22696 .get(&(sheet_id, start_row, end_row))
22697 .copied();
22698 #[cfg(test)]
22699 if cached.is_some() {
22700 self.col_hits.fetch_add(1, Ordering::Relaxed);
22701 }
22702 cached
22703 }
22704
22705 fn put_col_bounds(
22706 &mut self,
22707 sheet_id: SheetId,
22708 start_row: u32,
22709 end_row: u32,
22710 snapshot: u64,
22711 bounds: Option<(u32, u32)>,
22712 ) {
22713 self.reset_for_snapshot(snapshot);
22714 self.col_bounds_by_row_span
22715 .insert((sheet_id, start_row, end_row), bounds);
22716 #[cfg(test)]
22717 self.col_misses.fetch_add(1, Ordering::Relaxed);
22718 }
22719}
22720
22721#[derive(Default)]
22723pub struct ShimSpillManager {
22724 region_locks: RegionLockManager,
22725 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
22726}
22727
22728impl ShimSpillManager {
22729 pub(crate) fn reserve(
22730 &mut self,
22731 owner: VertexId,
22732 anchor_cell: CellRef,
22733 shape: SpillShape,
22734 _meta: SpillMeta,
22735 ) -> Result<(), ExcelError> {
22736 let region = crate::engine::spill::Region {
22738 sheet_id: anchor_cell.sheet_id as u32,
22739 row_start: anchor_cell.coord.row(),
22740 row_end: anchor_cell
22741 .coord
22742 .row()
22743 .saturating_add(shape.rows)
22744 .saturating_sub(1),
22745 col_start: anchor_cell.coord.col(),
22746 col_end: anchor_cell
22747 .coord
22748 .col()
22749 .saturating_add(shape.cols)
22750 .saturating_sub(1),
22751 };
22752 match self.region_locks.reserve(region, owner) {
22753 Ok(id) => {
22754 if id != 0 {
22755 self.active_locks.insert(owner, id);
22756 }
22757 Ok(())
22758 }
22759 Err(e) => Err(e),
22760 }
22761 }
22762
22763 pub(crate) fn release_owner(&mut self, owner: VertexId) {
22769 if let Some(id) = self.active_locks.remove(&owner) {
22770 self.region_locks.release(id);
22771 }
22772 }
22773
22774 pub(crate) fn commit_array_with_value_probe<F>(
22775 &mut self,
22776 graph: &mut DependencyGraph,
22777 anchor_vertex: VertexId,
22778 targets: &[CellRef],
22779 rows: Vec<Vec<LiteralValue>>,
22780 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22781 mut value_probe: F,
22782 ) -> Result<(), ExcelError>
22783 where
22784 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
22785 {
22786 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
22787
22788 let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
22792 anchor_vertex,
22793 targets,
22794 overwritable_formulas,
22795 );
22796 if let Err(e) = plan_res {
22797 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22798 self.region_locks.release(id);
22799 }
22800 return Err(e);
22801 }
22802
22803 if !graph.value_cache_enabled() {
22804 let (expected_rows, expected_cols) = if targets.is_empty() {
22806 (0u32, 0u32)
22807 } else {
22808 let mut min_r = u32::MAX;
22809 let mut max_r = 0u32;
22810 let mut min_c = u32::MAX;
22811 let mut max_c = 0u32;
22812 for cell in targets {
22813 let r = cell.coord.row();
22814 let c = cell.coord.col();
22815 min_r = min_r.min(r);
22816 max_r = max_r.max(r);
22817 min_c = min_c.min(c);
22818 max_c = max_c.max(c);
22819 }
22820 (
22821 max_r.saturating_sub(min_r).saturating_add(1),
22822 max_c.saturating_sub(min_c).saturating_add(1),
22823 )
22824 };
22825
22826 let anchor_cell = graph
22827 .get_cell_ref(anchor_vertex)
22828 .expect("anchor cell ref for spill commit");
22829
22830 for cell in targets {
22831 if *cell == anchor_cell {
22833 continue;
22834 }
22835 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
22837 continue;
22838 }
22839 if let Some(&vid) = graph.get_vertex_id_for_address(cell)
22841 && vid != anchor_vertex
22842 {
22843 match graph.get_vertex_kind(vid) {
22844 crate::engine::vertex::VertexKind::FormulaScalar
22845 | crate::engine::vertex::VertexKind::FormulaArray => {
22846 continue;
22848 }
22849 _ => {}
22850 }
22851 }
22852
22853 if let Some(v) = value_probe(graph, cell)
22854 && !matches!(v, LiteralValue::Empty)
22855 {
22856 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22857 self.region_locks.release(id);
22858 }
22859 return Err(ExcelError::new(ExcelErrorKind::Spill)
22860 .with_message("BlockedByValue")
22861 .with_extra(ExcelErrorExtra::Spill {
22862 expected_rows,
22863 expected_cols,
22864 }));
22865 }
22866 }
22867 }
22868
22869 let commit_res = graph.commit_spill_region_atomic_with_fault(
22870 anchor_vertex,
22871 targets.to_vec(),
22872 rows,
22873 None,
22874 );
22875 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22876 self.region_locks.release(id);
22877 }
22878 commit_res.map(|_| ())
22879 }
22880
22881 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
22883 &mut self,
22884 engine: &mut Engine<R>,
22885 anchor_vertex: VertexId,
22886 targets: &[CellRef],
22887 rows: Vec<Vec<LiteralValue>>,
22888 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22889 ) -> Result<(), ExcelError> {
22890 let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
22892 anchor_vertex,
22893 targets,
22894 overwritable_formulas,
22895 );
22896 if let Err(e) = plan_res {
22897 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22898 self.region_locks.release(id);
22899 }
22900 return Err(e);
22901 }
22902
22903 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
22904 anchor_vertex,
22905 targets.to_vec(),
22906 rows.clone(),
22907 None,
22908 );
22909 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22910 self.region_locks.release(id);
22911 }
22912 commit_res.map(|_| ())?;
22913
22914 if engine.config.arrow_storage_enabled
22916 && engine.config.delta_overlay_enabled
22917 && engine.config.write_formula_overlay_enabled
22918 {
22919 for (idx, cell) in targets.iter().enumerate() {
22921 let (r_off, c_off) = {
22922 if rows.is_empty() || rows[0].is_empty() {
22923 (0usize, 0usize)
22924 } else {
22925 let width = rows[0].len();
22926 (idx / width, idx % width)
22927 }
22928 };
22929 let v = rows
22930 .get(r_off)
22931 .and_then(|r| r.get(c_off))
22932 .cloned()
22933 .unwrap_or(LiteralValue::Empty);
22934 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
22935 engine.mirror_value_to_computed_overlay(
22936 &sheet_name,
22937 cell.coord.row() + 1,
22938 cell.coord.col() + 1,
22939 &v,
22940 );
22941 }
22942 }
22943 Ok(())
22944 }
22945}
22946
22947impl<R> Engine<R>
22948where
22949 R: EvaluationContext,
22950{
22951 fn resolve_shared_ref(
22952 &self,
22953 reference: &ReferenceType,
22954 current_sheet: &str,
22955 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
22956 use formualizer_common::{
22957 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
22958 SheetRef as SharedRef,
22959 };
22960
22961 let sr = match reference {
22963 ReferenceType::Cell {
22964 sheet,
22965 row,
22966 col,
22967 row_abs,
22968 col_abs,
22969 } => {
22970 let row0 = row
22971 .checked_sub(1)
22972 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
22973 let col0 = col
22974 .checked_sub(1)
22975 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
22976 let sheet_loc = match sheet.as_deref() {
22977 Some(name) => SheetLocator::from_name(name),
22978 None => SheetLocator::Current,
22979 };
22980 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
22981 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
22982 }
22983 ReferenceType::Range {
22984 sheet,
22985 start_row,
22986 start_col,
22987 end_row,
22988 end_col,
22989 start_row_abs,
22990 start_col_abs,
22991 end_row_abs,
22992 end_col_abs,
22993 } => {
22994 let sheet_loc = match sheet.as_deref() {
22995 Some(name) => SheetLocator::from_name(name),
22996 None => SheetLocator::Current,
22997 };
22998 let sr = start_row
22999 .map(|r| {
23000 r.checked_sub(1)
23001 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23002 })
23003 .transpose()?;
23004 let sc = start_col
23005 .map(|c| {
23006 c.checked_sub(1)
23007 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23008 })
23009 .transpose()?;
23010 let er = end_row
23011 .map(|r| {
23012 r.checked_sub(1)
23013 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23014 })
23015 .transpose()?;
23016 let ec = end_col
23017 .map(|c| {
23018 c.checked_sub(1)
23019 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23020 })
23021 .transpose()?;
23022 let range = SharedRangeRef::from_parts(
23023 sheet_loc,
23024 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
23025 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
23026 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
23027 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
23028 )
23029 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
23030 SharedRef::Range(range)
23031 }
23032 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23033 };
23034
23035 let current_id = self
23036 .graph
23037 .sheet_id(current_sheet)
23038 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23039
23040 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
23041 match loc {
23042 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
23043 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
23044 SheetLocator::Name(name) => {
23045 let n = name.as_ref();
23046 self.graph
23047 .sheet_id(n)
23048 .map(SheetLocator::Id)
23049 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23050 }
23051 }
23052 };
23053
23054 match sr {
23055 SharedRef::Cell(cell) => {
23056 let owned = cell.into_owned();
23057 let sheet = resolve_loc(owned.sheet)?;
23058 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
23059 }
23060 SharedRef::Range(range) => {
23061 let owned = range.into_owned();
23062 let sheet = resolve_loc(owned.sheet)?;
23063 Ok(SharedRef::Range(SharedRangeRef {
23064 sheet,
23065 start_row: owned.start_row,
23066 start_col: owned.start_col,
23067 end_row: owned.end_row,
23068 end_col: owned.end_col,
23069 }))
23070 }
23071 }
23072 }
23073}
23074
23075impl<R> crate::traits::ReferenceResolver for Engine<R>
23078where
23079 R: EvaluationContext,
23080{
23081 fn resolve_cell_reference(
23082 &self,
23083 sheet: Option<&str>,
23084 row: u32,
23085 col: u32,
23086 ) -> Result<LiteralValue, ExcelError> {
23087 let Some(sheet_name) = sheet else {
23098 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23099 "Unqualified cell reference resolved without sheet context".to_string(),
23100 ));
23101 };
23102 if let Some(v) = self.get_cell_value(sheet_name, row, col) {
23105 Ok(v)
23106 } else {
23107 Ok(LiteralValue::Number(0.0))
23109 }
23110 }
23111}
23112
23113impl<R> crate::traits::RangeResolver for Engine<R>
23114where
23115 R: EvaluationContext,
23116{
23117 fn resolve_range_reference(
23118 &self,
23119 sheet: Option<&str>,
23120 sr: Option<u32>,
23121 sc: Option<u32>,
23122 er: Option<u32>,
23123 ec: Option<u32>,
23124 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
23125 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
23128 }
23129}
23130
23131impl<R> crate::traits::NamedRangeResolver for Engine<R>
23132where
23133 R: EvaluationContext,
23134{
23135 fn resolve_named_range_reference(
23136 &self,
23137 name: &str,
23138 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
23139 self.resolver.resolve_named_range_reference(name)
23140 }
23141}
23142
23143impl<R> crate::traits::TableResolver for Engine<R>
23144where
23145 R: EvaluationContext,
23146{
23147 fn resolve_table_reference(
23148 &self,
23149 tref: &formualizer_parse::parser::TableReference,
23150 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23151 self.resolver.resolve_table_reference(tref)
23152 }
23153}
23154
23155impl<R> crate::traits::SourceResolver for Engine<R>
23156where
23157 R: EvaluationContext,
23158{
23159 fn source_scalar_version(&self, name: &str) -> Option<u64> {
23160 self.resolver.source_scalar_version(name)
23161 }
23162
23163 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
23164 self.resolver.resolve_source_scalar(name)
23165 }
23166
23167 fn source_table_version(&self, name: &str) -> Option<u64> {
23168 self.resolver.source_table_version(name)
23169 }
23170
23171 fn resolve_source_table(
23172 &self,
23173 name: &str,
23174 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23175 self.resolver.resolve_source_table(name)
23176 }
23177}
23178
23179impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
23181
23182impl<R> crate::traits::FunctionProvider for Engine<R>
23184where
23185 R: EvaluationContext,
23186{
23187 fn planning_semantic_revision(&self) -> Option<u64> {
23188 self.resolver.planning_semantic_revision()
23189 }
23190
23191 fn get_function(
23192 &self,
23193 prefix: &str,
23194 name: &str,
23195 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23196 self.resolver.get_function(prefix, name)
23197 }
23198
23199 fn get_function_for_planning(
23200 &self,
23201 prefix: &str,
23202 name: &str,
23203 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23204 self.resolver.get_function_for_planning(prefix, name)
23205 }
23206}
23207
23208impl<R> Engine<R>
23209where
23210 R: EvaluationContext,
23211{
23212 pub(crate) fn semantic_used_rows_for_columns(
23220 &self,
23221 sheet: &str,
23222 start_col: u32,
23223 end_col: u32,
23224 ) -> Option<(u32, u32)> {
23225 let arrow_bounds = self
23226 .sheet_store()
23227 .sheet(sheet)
23228 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23229 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23230 Self::union_used_bounds(arrow_bounds, formula_bounds)
23231 }
23232
23233 pub(crate) fn semantic_used_cols_for_rows(
23234 &self,
23235 sheet: &str,
23236 start_row: u32,
23237 end_row: u32,
23238 ) -> Option<(u32, u32)> {
23239 let arrow_bounds = self
23240 .sheet_store()
23241 .sheet(sheet)
23242 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23243 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23244 Self::union_used_bounds(arrow_bounds, formula_bounds)
23245 }
23246}
23247
23248impl<R> crate::traits::EvaluationContext for Engine<R>
23250where
23251 R: EvaluationContext,
23252{
23253 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
23254 &self.clock
23255 }
23256
23257 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
23258 self.thread_pool.as_ref()
23259 }
23260
23261 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
23262 self.active_cancel_flag.clone()
23263 }
23264
23265 fn chunk_hint(&self) -> Option<usize> {
23266 let hint =
23268 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
23269 Some(hint.clamp(1024, 1 << 20)) }
23271
23272 fn volatile_level(&self) -> crate::traits::VolatileLevel {
23273 self.config.volatile_level
23274 }
23275
23276 fn workbook_seed(&self) -> u64 {
23277 self.config.workbook_seed
23278 }
23279
23280 fn recalc_epoch(&self) -> u64 {
23281 self.recalc_epoch
23282 }
23283
23284 fn workbook_sheet_count(&self) -> Option<usize> {
23285 Some(self.graph.sheet_reg().active_len())
23286 }
23287
23288 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
23289 self.graph.sheet_reg().active_position(sheet)
23290 }
23291
23292 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
23293 self.sheet_index_by_name(current_sheet)
23294 }
23295
23296 fn inspect_reference(
23297 &self,
23298 reference: &ReferenceType,
23299 current_sheet: &str,
23300 ) -> Result<Option<ReferenceInfo>, ExcelError> {
23301 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
23302 let sheet_id = self
23303 .graph
23304 .sheet_id(sheet_name)
23305 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23306 let sheet_index = self
23307 .graph
23308 .sheet_reg()
23309 .active_position_by_id(sheet_id)
23310 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23311 Ok((sheet_id, sheet_index))
23312 };
23313
23314 let cell_info =
23315 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
23316 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23317 let row0 = row
23318 .checked_sub(1)
23319 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23320 let col0 = col
23321 .checked_sub(1)
23322 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23323 Ok(ReferenceInfo {
23324 first_sheet_index: Some(sheet_index),
23325 sheet_count: Some(1),
23326 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23327 })
23328 };
23329
23330 let range_info = |sheet_name: &str,
23331 start_row: Option<u32>,
23332 start_col: Option<u32>|
23333 -> Result<ReferenceInfo, ExcelError> {
23334 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23335 let row = start_row.unwrap_or(1);
23336 let col = start_col.unwrap_or(1);
23337 let row0 = row
23338 .checked_sub(1)
23339 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23340 let col0 = col
23341 .checked_sub(1)
23342 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23343 Ok(ReferenceInfo {
23344 first_sheet_index: Some(sheet_index),
23345 sheet_count: Some(1),
23346 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23347 })
23348 };
23349
23350 let info = match reference {
23351 ReferenceType::Cell {
23352 sheet, row, col, ..
23353 } => {
23354 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23355 cell_info(sheet_name, *row, *col)?
23356 }
23357 ReferenceType::Range {
23358 sheet,
23359 start_row,
23360 start_col,
23361 ..
23362 } => {
23363 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23364 range_info(sheet_name, *start_row, *start_col)?
23365 }
23366 ReferenceType::Cell3D {
23367 sheet_first,
23368 sheet_last,
23369 row,
23370 col,
23371 ..
23372 } => {
23373 let first = cell_info(sheet_first, *row, *col)?;
23374 ReferenceInfo {
23375 first_sheet_index: first.first_sheet_index,
23376 sheet_count: self
23377 .graph
23378 .sheet_reg()
23379 .active_span_len(sheet_first, sheet_last),
23380 first_cell: first.first_cell,
23381 }
23382 }
23383 ReferenceType::Range3D {
23384 sheet_first,
23385 sheet_last,
23386 start_row,
23387 start_col,
23388 ..
23389 } => {
23390 let first = range_info(sheet_first, *start_row, *start_col)?;
23391 ReferenceInfo {
23392 first_sheet_index: first.first_sheet_index,
23393 sheet_count: self
23394 .graph
23395 .sheet_reg()
23396 .active_span_len(sheet_first, sheet_last),
23397 first_cell: first.first_cell,
23398 }
23399 }
23400 ReferenceType::NamedRange(name) => {
23401 let current_id = self
23402 .graph
23403 .sheet_id(current_sheet)
23404 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23405 let named = self
23406 .graph
23407 .resolve_name_entry(name, current_id)
23408 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23409 match &named.definition {
23410 NamedDefinition::Cell(cell) => ReferenceInfo {
23411 first_sheet_index: self
23412 .graph
23413 .sheet_reg()
23414 .active_position_by_id(cell.sheet_id),
23415 sheet_count: Some(1),
23416 first_cell: Some(*cell),
23417 },
23418 NamedDefinition::Range(range) => ReferenceInfo {
23419 first_sheet_index: self
23420 .graph
23421 .sheet_reg()
23422 .active_position_by_id(range.start.sheet_id),
23423 sheet_count: Some(1),
23424 first_cell: Some(range.start),
23425 },
23426 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
23427 ReferenceInfo {
23428 first_sheet_index: None,
23429 sheet_count: None,
23430 first_cell: None,
23431 }
23432 }
23433 }
23434 }
23435 ReferenceType::Table(tref) => {
23436 let table = self
23437 .graph
23438 .resolve_table_entry(&tref.name)
23439 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23440 ReferenceInfo {
23441 first_sheet_index: self
23442 .graph
23443 .sheet_reg()
23444 .active_position_by_id(table.range.start.sheet_id),
23445 sheet_count: Some(1),
23446 first_cell: Some(table.range.start),
23447 }
23448 }
23449 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23450 };
23451
23452 Ok(Some(info))
23453 }
23454
23455 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
23456 let sheet_name = self.graph.sheet_name(cell.sheet_id);
23457 if sheet_name.is_empty() {
23458 return Err(ExcelError::new(ExcelErrorKind::Ref));
23459 }
23460 let row = cell.coord.row() + 1;
23461 let col = cell.coord.col() + 1;
23462
23463 if let Some(entries) = self.staged_formulas.get(sheet_name)
23464 && let Some(text) = entries.get(row, col)
23465 {
23466 return Ok(Some(if text.starts_with('=') {
23467 text.to_owned()
23468 } else {
23469 format!("={text}")
23470 }));
23471 }
23472
23473 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
23474 return Ok(None);
23475 };
23476 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
23477 }
23478
23479 fn used_rows_for_columns(
23480 &self,
23481 sheet: &str,
23482 start_col: u32,
23483 end_col: u32,
23484 ) -> Option<(u32, u32)> {
23485 let sheet_id = self.graph.sheet_id(sheet)?;
23487 let snap = self.data_snapshot_id();
23488 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23489 guard
23490 .as_ref()
23491 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
23492 }) {
23493 return cached;
23494 }
23495
23496 let arrow_bounds = self
23497 .sheet_store()
23498 .sheet(sheet)
23499 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23500 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23501 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23502 Some(bounds)
23503 } else {
23504 let sc0 = start_col.saturating_sub(1);
23505 let ec0 = end_col.saturating_sub(1);
23506 self.graph
23507 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
23508 .map(|(a0, b0)| (a0 + 1, b0 + 1))
23509 };
23510
23511 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23512 guard
23513 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23514 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
23515 }
23516
23517 computed
23518 }
23519
23520 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
23521 let sheet_id = self.graph.sheet_id(sheet)?;
23523 let snap = self.data_snapshot_id();
23524 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23525 guard
23526 .as_ref()
23527 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
23528 }) {
23529 return cached;
23530 }
23531
23532 let arrow_bounds = self
23533 .sheet_store()
23534 .sheet(sheet)
23535 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23536 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23537 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23538 Some(bounds)
23539 } else {
23540 let sr0 = start_row.saturating_sub(1);
23541 let er0 = end_row.saturating_sub(1);
23542 self.graph
23543 .used_col_bounds_for_rows(sheet_id, sr0, er0)
23544 .map(|(a0, b0)| (a0 + 1, b0 + 1))
23545 };
23546
23547 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23548 guard
23549 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23550 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
23551 }
23552
23553 computed
23554 }
23555
23556 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
23557 let _ = self.graph.sheet_id(sheet)?;
23558 Some((1_048_576, 16_384)) }
23562
23563 fn data_snapshot_id(&self) -> u64 {
23564 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
23565 }
23566
23567 fn backend_caps(&self) -> crate::traits::BackendCaps {
23568 crate::traits::BackendCaps {
23569 streaming: true,
23570 used_region: true,
23571 write: false,
23572 tables: false,
23573 async_stream: false,
23574 }
23575 }
23576
23577 fn build_lookup_index(
23578 &self,
23579 view: &RangeView<'_>,
23580 axis: LookupAxis,
23581 ) -> Option<Arc<LookupIndex>> {
23582 self.build_lookup_index_impl(view, axis)
23583 }
23584
23585 fn date_system(&self) -> crate::engine::DateSystem {
23588 self.config.date_system
23589 }
23590 fn resolve_range_view<'c>(
23592 &'c self,
23593 reference: &ReferenceType,
23594 current_sheet: &str,
23595 ) -> Result<RangeView<'c>, ExcelError> {
23596 match reference {
23597 ReferenceType::External(ext) => {
23598 let name = ext.raw.as_str();
23599 match ext.kind {
23600 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
23601 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
23602 return Err(ExcelError::new(ExcelErrorKind::Name)
23603 .with_message(format!("Undefined name: {name}")));
23604 };
23605 let version = source
23606 .version
23607 .or_else(|| self.resolver.source_scalar_version(name));
23608 let v = self.resolve_source_scalar_cached(name, version)?;
23609 Ok(RangeView::from_owned_rows(
23610 vec![vec![v]],
23611 self.config.date_system,
23612 ))
23613 }
23614 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
23615 let Some(source) = self.graph.resolve_source_table_entry(name) else {
23616 return Err(ExcelError::new(ExcelErrorKind::Name)
23617 .with_message(format!("Undefined table: {name}")));
23618 };
23619 let version = source
23620 .version
23621 .or_else(|| self.resolver.source_table_version(name));
23622 let table = self.resolve_source_table_cached(name, version)?;
23623 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
23624 self.source_table_to_range_view(table.as_ref(), &spec)
23625 }
23626 }
23627 }
23628 ReferenceType::Range { .. } => {
23629 let shared = self.resolve_shared_ref(reference, current_sheet)?;
23630 let formualizer_common::SheetRef::Range(range) = shared else {
23631 return Err(ExcelError::new(ExcelErrorKind::Ref));
23632 };
23633 let sheet_id = match range.sheet {
23634 formualizer_common::SheetLocator::Id(id) => id,
23635 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23636 };
23637 let sheet_name = self.graph.sheet_name(sheet_id);
23638
23639 let bounded_range = if range.start_row.is_some()
23640 && range.start_col.is_some()
23641 && range.end_row.is_some()
23642 && range.end_col.is_some()
23643 {
23644 Some(RangeRef::try_from_shared(range.as_ref())?)
23645 } else {
23646 None
23647 };
23648
23649 let sr = bounded_range
23650 .as_ref()
23651 .map(|r| r.start.coord.row() + 1)
23652 .or_else(|| range.start_row.map(|b| b.index + 1));
23653 let sc = bounded_range
23654 .as_ref()
23655 .map(|r| r.start.coord.col() + 1)
23656 .or_else(|| range.start_col.map(|b| b.index + 1));
23657 let er = bounded_range
23658 .as_ref()
23659 .map(|r| r.end.coord.row() + 1)
23660 .or_else(|| range.end_row.map(|b| b.index + 1));
23661 let ec = bounded_range
23662 .as_ref()
23663 .map(|r| r.end.coord.col() + 1)
23664 .or_else(|| range.end_col.map(|b| b.index + 1));
23665
23666 let extent = resolve_used_extent_with_fallback(
23667 OpenRangeBounds {
23668 start_row: sr,
23669 start_column: sc,
23670 end_row: er,
23671 end_column: ec,
23672 },
23673 ExtentPolicy::EvaluationCompat {
23674 fallback_row: None,
23675 fallback_column: None,
23676 },
23677 || {
23678 self.sheet_bounds(sheet_name)
23679 .map(|_| self.config.max_open_ended_rows)
23680 },
23681 || {
23682 self.sheet_bounds(sheet_name)
23683 .map(|_| self.config.max_open_ended_cols)
23684 },
23685 |first, last| self.used_rows_for_columns(sheet_name, first, last),
23686 |first, last| self.used_cols_for_rows(sheet_name, first, last),
23687 );
23688 let (sr, sc, er, ec) = extent
23689 .map(|extent| {
23690 (
23691 extent.start_row,
23692 extent.start_column,
23693 extent.end_row,
23694 extent.end_column,
23695 )
23696 })
23697 .unwrap_or((1, 1, 0, 0));
23698
23699 if self.force_materialize_range_views {
23700 if er < sr || ec < sc {
23701 return Ok(RangeView::from_owned_rows(
23702 Vec::new(),
23703 self.config.date_system,
23704 ));
23705 }
23706 let h = (er - sr + 1) as u64;
23707 let w = (ec - sc + 1) as u64;
23708 let cell_count = h.saturating_mul(w);
23709 if cell_count <= self.config.spill.max_spill_cells as u64 {
23710 let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
23711 for r in sr..=er {
23712 let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
23713 for c in sc..=ec {
23714 rowv.push(
23715 self.get_cell_value(sheet_name, r, c)
23716 .unwrap_or(LiteralValue::Empty),
23717 );
23718 }
23719 rows.push(rowv);
23720 }
23721 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
23722 }
23723 }
23724
23725 let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
23726 return Ok(RangeView::from_owned_rows(
23727 Vec::new(),
23728 self.config.date_system,
23729 ));
23730 };
23731
23732 let rv = if er < sr || ec < sc {
23733 asheet.range_view(1, 1, 0, 0)
23734 } else {
23735 let sr0 = sr.saturating_sub(1) as usize;
23736 let sc0 = sc.saturating_sub(1) as usize;
23737 let er0 = er.saturating_sub(1) as usize;
23738 let ec0 = ec.saturating_sub(1) as usize;
23739 asheet.range_view(sr0, sc0, er0, ec0)
23740 };
23741
23742 Ok(rv)
23743 }
23744 ReferenceType::Cell { .. } => {
23745 let shared = self.resolve_shared_ref(reference, current_sheet)?;
23746 let formualizer_common::SheetRef::Cell(cell) = shared else {
23747 return Err(ExcelError::new(ExcelErrorKind::Ref));
23748 };
23749 let addr = CellRef::try_from_shared(cell)?;
23750 let sheet_id = addr.sheet_id;
23751 let sheet_name = self.graph.sheet_name(sheet_id);
23752 let row = addr.coord.row() + 1;
23753 let col = addr.coord.col() + 1;
23754
23755 if self.force_materialize_range_views {
23756 let v = self
23757 .get_cell_value(sheet_name, row, col)
23758 .unwrap_or(LiteralValue::Empty);
23759 return Ok(RangeView::from_owned_rows(
23760 vec![vec![v]],
23761 self.config.date_system,
23762 ));
23763 }
23764
23765 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
23766 let r0 = row.saturating_sub(1) as usize;
23767 let c0 = col.saturating_sub(1) as usize;
23768 let rv = asheet.range_view(r0, c0, r0, c0);
23769 Ok(rv)
23770 } else {
23771 let v = self
23772 .get_cell_value(sheet_name, row, col)
23773 .unwrap_or(LiteralValue::Empty);
23774 Ok(RangeView::from_owned_rows(
23775 vec![vec![v]],
23776 self.config.date_system,
23777 ))
23778 }
23779 }
23780 ReferenceType::NamedRange(name) => {
23781 if let Some(current_id) = self.graph.sheet_id(current_sheet)
23782 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
23783 {
23784 match &named.definition {
23785 NamedDefinition::Cell(cell_ref) => {
23786 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23787 if self.force_materialize_range_views {
23788 let v = self
23789 .get_cell_value(
23790 sheet_name,
23791 cell_ref.coord.row() + 1,
23792 cell_ref.coord.col() + 1,
23793 )
23794 .unwrap_or(LiteralValue::Empty);
23795 return Ok(RangeView::from_owned_rows(
23796 vec![vec![v]],
23797 self.config.date_system,
23798 ));
23799 } else {
23800 let asheet = self
23801 .sheet_store()
23802 .sheet(sheet_name)
23803 .expect("Arrow sheet missing for named cell");
23804 let r0 = cell_ref.coord.row() as usize;
23805 let c0 = cell_ref.coord.col() as usize;
23806 let rv = asheet.range_view(r0, c0, r0, c0);
23807 return Ok(rv);
23808 }
23809 }
23810 NamedDefinition::Range(range_ref) => {
23811 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
23812 let sr = range_ref.start.coord.row() + 1;
23813 let sc = range_ref.start.coord.col() + 1;
23814 let er = range_ref.end.coord.row() + 1;
23815 let ec = range_ref.end.coord.col() + 1;
23816 if self.force_materialize_range_views {
23817 let h = (er.saturating_sub(sr) + 1) as u64;
23818 let w = (ec.saturating_sub(sc) + 1) as u64;
23819 let cell_count = h.saturating_mul(w);
23820 if cell_count <= self.config.spill.max_spill_cells as u64 {
23821 let mut rows: Vec<Vec<LiteralValue>> =
23822 Vec::with_capacity(h as usize);
23823 for r in sr..=er {
23824 let mut rowv: Vec<LiteralValue> =
23825 Vec::with_capacity(w as usize);
23826 for c in sc..=ec {
23827 rowv.push(
23828 self.get_cell_value(sheet_name, r, c)
23829 .unwrap_or(LiteralValue::Empty),
23830 );
23831 }
23832 rows.push(rowv);
23833 }
23834 return Ok(RangeView::from_owned_rows(
23835 rows,
23836 self.config.date_system,
23837 ));
23838 }
23839 }
23840 let asheet = self
23841 .sheet_store()
23842 .sheet(sheet_name)
23843 .expect("Arrow sheet missing for named range");
23844 let sr0 = range_ref.start.coord.row() as usize;
23845 let sc0 = range_ref.start.coord.col() as usize;
23846 let er0 = range_ref.end.coord.row() as usize;
23847 let ec0 = range_ref.end.coord.col() as usize;
23848 let rv = asheet.range_view(sr0, sc0, er0, ec0);
23849 return Ok(rv);
23850 }
23851 NamedDefinition::Literal(v) => {
23852 return Ok(RangeView::from_owned_rows(
23853 vec![vec![v.clone()]],
23854 self.config.date_system,
23855 ));
23856 }
23857 NamedDefinition::Formula { .. } => {
23858 if let Some(value) = self.graph.get_value(named.vertex) {
23859 return Ok(RangeView::from_owned_rows(
23860 vec![vec![value]],
23861 self.config.date_system,
23862 ));
23863 }
23864 }
23865 }
23866 }
23867
23868 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
23869 let version = source
23870 .version
23871 .or_else(|| self.resolver.source_scalar_version(name));
23872 let v = self.resolve_source_scalar_cached(name, version)?;
23873 return Ok(RangeView::from_owned_rows(
23874 vec![vec![v]],
23875 self.config.date_system,
23876 ));
23877 }
23878
23879 let data = self.resolver.resolve_named_range_reference(name)?;
23880 Ok(RangeView::from_owned_rows(data, self.config.date_system))
23881 }
23882 ReferenceType::Table(tref) => {
23883 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
23884 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
23885 let asheet = self
23886 .sheet_store()
23887 .sheet(sheet_name)
23888 .expect("Arrow sheet missing for table reference");
23889
23890 let sr0 = table.range.start.coord.row() as usize;
23891 let sc0 = table.range.start.coord.col() as usize;
23892 let er0 = table.range.end.coord.row() as usize;
23893 let ec0 = table.range.end.coord.col() as usize;
23894
23895 let has_totals = table.totals_row;
23896 let has_headers = table.header_row;
23897 let data_sr = if has_headers {
23898 sr0.saturating_add(1)
23899 } else {
23900 sr0
23901 };
23902 let data_er = if has_totals {
23903 er0.saturating_sub(1)
23904 } else {
23905 er0
23906 };
23907
23908 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
23909 if sr > er || sc > ec {
23910 asheet.range_view(1, 1, 0, 0)
23911 } else {
23912 asheet.range_view(sr, sc, er, ec)
23913 }
23914 };
23915
23916 let av = match &tref.specifier {
23917 None => {
23918 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23919 "Table reference without specifier is unsupported".to_string(),
23920 ));
23921 }
23922 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
23923 let Some(idx) = table.col_index(col) else {
23924 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23925 "Column refers to unknown table column".to_string(),
23926 ));
23927 };
23928 let c0 = sc0 + idx;
23929 select(data_sr, c0, data_er, c0)
23930 }
23931 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
23932 start,
23933 end,
23934 )) => {
23935 let Some(si) = table.col_index(start) else {
23936 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23937 "Column range refers to unknown column(s)".to_string(),
23938 ));
23939 };
23940 let Some(ei) = table.col_index(end) else {
23941 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23942 "Column range refers to unknown column(s)".to_string(),
23943 ));
23944 };
23945 let (mut a, mut b) = (si, ei);
23946 if a > b {
23947 std::mem::swap(&mut a, &mut b);
23948 }
23949 let c_start = sc0 + a;
23950 let c_end = sc0 + b;
23951 select(data_sr, c_start, data_er, c_end)
23952 }
23953 Some(formualizer_parse::parser::TableSpecifier::All)
23954 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23955 formualizer_parse::parser::SpecialItem::All,
23956 )) => select(sr0, sc0, er0, ec0),
23957 Some(formualizer_parse::parser::TableSpecifier::Data)
23958 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23959 formualizer_parse::parser::SpecialItem::Data,
23960 )) => select(data_sr, sc0, data_er, ec0),
23961 Some(formualizer_parse::parser::TableSpecifier::Headers)
23962 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23963 formualizer_parse::parser::SpecialItem::Headers,
23964 )) => {
23965 if !has_headers {
23966 asheet.range_view(1, 1, 0, 0)
23967 } else {
23968 select(sr0, sc0, sr0, ec0)
23969 }
23970 }
23971 Some(formualizer_parse::parser::TableSpecifier::Totals)
23972 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23973 formualizer_parse::parser::SpecialItem::Totals,
23974 )) => {
23975 if !has_totals {
23976 asheet.range_view(1, 1, 0, 0)
23977 } else {
23978 select(er0, sc0, er0, ec0)
23979 }
23980 }
23981 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23982 formualizer_parse::parser::SpecialItem::ThisRow,
23983 )) => {
23984 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23985 "@ (This Row) requires table-aware context; not yet supported"
23986 .to_string(),
23987 ));
23988 }
23989 Some(formualizer_parse::parser::TableSpecifier::Row(_))
23990 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
23991 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23992 "Complex structured references not yet supported".to_string(),
23993 ));
23994 }
23995 };
23996
23997 return Ok(av);
23998 }
23999
24000 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
24001 let version = source
24002 .version
24003 .or_else(|| self.resolver.source_table_version(&tref.name));
24004 let table = self.resolve_source_table_cached(&tref.name, version)?;
24005 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
24006 }
24007
24008 let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
24010 let owned = boxed.materialise().into_owned();
24011 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
24012 }
24013 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
24014 Err(ExcelError::new(ExcelErrorKind::NImpl)
24015 .with_message("3D references are not yet supported".to_string()))
24016 }
24017 }
24018 }
24019
24020 fn resolve_cell_reference_value(
24021 &self,
24022 sheet: Option<&str>,
24023 row: u32,
24024 col: u32,
24025 current_sheet: &str,
24026 ) -> Result<LiteralValue, ExcelError> {
24027 let sheet_name = sheet.unwrap_or(current_sheet);
24028 if self.graph.sheet_id(sheet_name).is_none() {
24029 return Err(ExcelError::new(ExcelErrorKind::Ref));
24030 }
24031 Ok(self
24032 .get_cell_value(sheet_name, row, col)
24033 .unwrap_or(LiteralValue::Empty))
24034 }
24035
24036 fn build_criteria_mask(
24037 &self,
24038 view: &RangeView<'_>,
24039 col_in_view: usize,
24040 pred: &crate::args::CriteriaPredicate,
24041 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24042 if view.dims().1 == 0 {
24043 return None;
24044 }
24045 let sheet_rows = view.sheet().nrows as usize;
24048 if sheet_rows == 0 || view.start_row() >= sheet_rows {
24049 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
24050 }
24051 compute_criteria_mask(view, col_in_view, pred)
24052 }
24053
24054 fn build_row_visibility_mask(
24055 &self,
24056 view: &RangeView<'_>,
24057 mode: VisibilityMaskMode,
24058 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24059 self.build_row_visibility_mask_for_view(view, mode)
24060 }
24061}
24062
24063impl<R> Engine<R>
24064where
24065 R: EvaluationContext,
24066{
24067 fn clear_spill_projection_and_mirror(
24068 &mut self,
24069 anchor_vertex: VertexId,
24070 delta: Option<&mut DeltaCollector>,
24071 ) {
24072 let spill_cells = self
24073 .graph
24074 .spill_cells_for_anchor(anchor_vertex)
24075 .map(|cells| cells.to_vec())
24076 .unwrap_or_default();
24077 if spill_cells.is_empty() {
24078 return;
24079 }
24080
24081 if let Some(delta) = delta
24082 && delta.mode != DeltaMode::Off
24083 {
24084 let empty = LiteralValue::Empty;
24085 for cell in spill_cells.iter() {
24086 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24087 let old = self
24088 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24089 .unwrap_or(LiteralValue::Empty);
24090 if old != empty {
24091 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24092 }
24093 }
24094 }
24095
24096 self.graph.clear_spill_region(anchor_vertex);
24097 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24098 self.record_formula_plane_structural_change(scope);
24099 }
24100
24101 if self.config.arrow_storage_enabled
24102 && self.config.delta_overlay_enabled
24103 && self.config.write_formula_overlay_enabled
24104 {
24105 let empty = LiteralValue::Empty;
24106 for cell in spill_cells.iter() {
24107 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24108 self.mirror_value_to_computed_overlay(
24109 &sheet_name,
24110 cell.coord.row() + 1,
24111 cell.coord.col() + 1,
24112 &empty,
24113 );
24114 }
24115 }
24116 }
24117
24118 fn apply_cycle_outcome(
24132 &mut self,
24133 cycle: &[VertexId],
24134 mut delta: Option<&mut DeltaCollector>,
24135 dirty_filter: Option<&FxHashSet<VertexId>>,
24136 ) -> usize {
24137 let circ_error = LiteralValue::Error(
24138 ExcelError::new(ExcelErrorKind::Circ)
24139 .with_message("Circular dependency detected".to_string()),
24140 );
24141 let mut stamped = 0usize;
24142 for &vertex_id in cycle {
24143 if let Some(filter) = dirty_filter
24144 && !filter.contains(&vertex_id)
24145 {
24146 continue;
24147 }
24148 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
24149 stamped += 1;
24150 }
24151 stamped
24152 }
24153
24154 fn stamp_cycle_error(
24169 &mut self,
24170 vertex_id: VertexId,
24171 circ_error: &LiteralValue,
24172 mut delta: Option<&mut DeltaCollector>,
24173 ) {
24174 if self.graph.spill_registry_has_anchor(vertex_id) {
24176 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24177 }
24178 self.spill_mgr.release_owner(vertex_id);
24181
24182 if let Some(d) = delta
24184 && d.mode != DeltaMode::Off
24185 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24186 {
24187 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24188 let old = self
24189 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24190 .unwrap_or(LiteralValue::Empty);
24191 if old != *circ_error {
24192 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24193 }
24194 }
24195
24196 self.graph
24197 .update_vertex_value(vertex_id, circ_error.clone());
24198 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
24199 }
24200
24201 fn handle_cycle_unit(
24214 &mut self,
24215 cycle: &[VertexId],
24216 mut delta: Option<&mut DeltaCollector>,
24217 dirty_filter: Option<&FxHashSet<VertexId>>,
24218 cancel_flag: Option<&AtomicBool>,
24219 ) -> Result<usize, ExcelError> {
24220 self.resource_checkpoint(cycle.len() as u64)?;
24221 match self.config.cycle.detection {
24222 CycleDetection::Static => {
24223 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
24224 }
24225 CycleDetection::Runtime => {
24226 if let Some(filter) = dirty_filter
24227 && !cycle.iter().any(|v| filter.contains(v))
24228 {
24229 return Ok(0);
24230 }
24231 self.evaluate_scc_unit(cycle, delta, cancel_flag)
24235 }
24236 }
24237 }
24238
24239 pub(crate) fn evaluate_scc_unit(
24259 &mut self,
24260 cycle: &[VertexId],
24261 mut delta: Option<&mut DeltaCollector>,
24262 cancel_flag: Option<&AtomicBool>,
24263 ) -> Result<usize, ExcelError> {
24264 struct SccMember {
24265 vertex: VertexId,
24266 cell: Option<CellRef>,
24267 }
24268
24269 let task_start = crate::instant::FzInstant::now();
24270
24271 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
24276 let mut name_members: Vec<(VertexId, String)> = Vec::new();
24277 let mut other_members: Vec<VertexId> = Vec::new();
24278 for &v in cycle {
24279 match self.graph.get_vertex_kind(v) {
24280 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24281 match self.graph.get_cell_ref(v) {
24282 Some(cell) => cell_members.push((v, cell)),
24283 None => other_members.push(v),
24284 }
24285 }
24286 VertexKind::NamedScalar | VertexKind::NamedArray => {
24287 match self.graph.name_key_for_vertex(v) {
24288 Some(key) => name_members.push((v, key)),
24289 None => other_members.push(v),
24290 }
24291 }
24292 _ => other_members.push(v),
24293 }
24294 }
24295 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
24296 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
24297 other_members.sort_unstable();
24298
24299 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
24300 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
24301 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
24302 for (v, c) in &cell_members {
24303 members.push(SccMember {
24304 vertex: *v,
24305 cell: Some(*c),
24306 });
24307 }
24308 for (v, _) in &name_members {
24309 members.push(SccMember {
24310 vertex: *v,
24311 cell: None,
24312 });
24313 }
24314 for v in &other_members {
24315 members.push(SccMember {
24316 vertex: *v,
24317 cell: None,
24318 });
24319 }
24320 let n = members.len();
24321 let recordable = cell_refs.len() + name_keys.len();
24324
24325 let circ_error = LiteralValue::Error(
24326 ExcelError::new(ExcelErrorKind::Circ)
24327 .with_message("Circular dependency detected".to_string()),
24328 );
24329
24330 if !self.iterative_state_values.is_empty() {
24341 let restore: Vec<(VertexId, LiteralValue)> = members
24342 .iter()
24343 .filter_map(|m| {
24344 let cell = m.cell?;
24345 let persisted = self.iterative_state_values.get(&m.vertex)?;
24346 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24347 let overlay = self
24348 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24349 .unwrap_or(LiteralValue::Empty);
24350 if matches!(overlay, LiteralValue::Empty) {
24351 Some((m.vertex, persisted.clone()))
24352 } else {
24353 None
24354 }
24355 })
24356 .collect();
24357 for (vertex, value) in restore {
24358 self.mirror_vertex_value_to_overlay(vertex, &value);
24359 }
24360 }
24361
24362 let snapshot: Vec<LiteralValue> = members
24365 .iter()
24366 .map(|m| match m.cell {
24367 Some(cell) => {
24368 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24369 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24370 .unwrap_or(LiteralValue::Empty)
24371 }
24372 None => self
24373 .graph
24374 .get_value(m.vertex)
24375 .unwrap_or(LiteralValue::Empty),
24376 })
24377 .collect();
24378
24379 let mut excluded = vec![false; n];
24384 let mut last_value = snapshot.clone();
24385 let mut stamped = 0usize;
24386 for (i, m) in members.iter().enumerate() {
24387 match self.graph.get_vertex_kind(m.vertex) {
24388 VertexKind::FormulaArray => {
24389 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
24393 excluded[i] = true;
24394 last_value[i] = circ_error.clone();
24395 stamped += 1;
24396 }
24397 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
24398 _ => excluded[i] = true,
24399 }
24400 }
24401
24402 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
24403
24404 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
24406 let mut pos: Vec<i64> = vec![-1; n];
24408 let mut changed = vec![false; n];
24410
24411 macro_rules! run_member {
24414 ($i:expr) => {{
24415 let i: usize = $i;
24416 let m = &members[i];
24417 if i < recordable {
24418 collector.set_current(i as u32);
24419 }
24420 let value = {
24421 let ctx = RecordingContext::new(&*self, &collector);
24422 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
24423 Ok(v) => v,
24424 Err(e) => LiteralValue::Error(e),
24425 }
24426 };
24427 let is_cell_formula = m.cell.is_some();
24428 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
24429 self.stamp_cycle_error(m.vertex, &circ_error, None);
24434 excluded[i] = true;
24435 stamped += 1;
24436 changed[i] = last_value[i] != circ_error;
24437 last_value[i] = circ_error.clone();
24438 } else {
24439 self.graph.update_vertex_value(m.vertex, value.clone());
24440 self.mirror_vertex_value_to_overlay(m.vertex, &value);
24441 #[cfg(debug_assertions)]
24445 if let Some(cell) = m.cell {
24446 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24447 debug_assert!(
24448 self.read_delta_overlay_cell(
24449 &sheet_name,
24450 cell.coord.row() + 1,
24451 cell.coord.col() + 1
24452 )
24453 .is_none(),
24454 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
24455 cell.coord.row() + 1,
24456 cell.coord.col() + 1
24457 );
24458 }
24459 changed[i] = last_value[i] != value;
24460 last_value[i] = value;
24461 }
24462 }};
24463 }
24464
24465 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
24466 if let Some(flag) = flag
24467 && flag.load(Ordering::Relaxed)
24468 {
24469 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24470 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
24471 }
24472 Ok(())
24473 };
24474
24475 check_cancel(cancel_flag)?;
24477 let mut passes = 1usize;
24478 {
24479 let mut p = 0i64;
24480 for i in 0..n {
24481 if excluded[i] {
24482 continue;
24483 }
24484 run_member!(i);
24485 pos[i] = p;
24486 p += 1;
24487 }
24488 }
24489
24490 let policy = self.config.cycle.policy;
24506 let cap = n + 2;
24507 let mut witnessed_cycles = 0usize;
24508 let mut capped = false;
24509 let mut iterating = false;
24511 let mut converged = false;
24512 let mut prev_pass: Option<Vec<LiteralValue>> = None;
24516 let mut iter_max_delta = 0f64;
24519 let mut iter_nan_converged = 0usize;
24520 let mut settle_passes = 0usize;
24524 loop {
24525 let drained = collector.take_edges();
24528 for i in 0..n {
24529 if pos[i] >= 0 {
24530 out_edges[i].clear();
24531 }
24532 }
24533 for (from, to) in drained {
24534 debug_assert!(
24535 pos[from as usize] >= 0,
24536 "edge from a member that did not run"
24537 );
24538 out_edges[from as usize].push(to);
24539 }
24540 let mut edges: Vec<(u32, u32)> = Vec::new();
24541 for (i, outs) in out_edges.iter().enumerate() {
24542 if excluded[i] {
24543 continue;
24544 }
24545 for &t in outs {
24546 edges.push((i as u32, t));
24547 }
24548 }
24549 edges.sort_unstable();
24550 edges.dedup();
24551
24552 let analysis = analyze_live_graph(n, &edges);
24553
24554 if analysis.cycle_count > 0 {
24555 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
24559 match policy {
24560 CyclePolicy::Error => {
24561 for i in 0..n {
24567 if analysis.in_cycle[i] && !excluded[i] {
24568 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
24569 excluded[i] = true;
24570 last_value[i] = circ_error.clone();
24571 stamped += 1;
24572 }
24573 }
24574 check_cancel(cancel_flag)?;
24575 let order: Vec<usize> = analysis
24576 .topo
24577 .iter()
24578 .map(|&i| i as usize)
24579 .filter(|&i| !excluded[i])
24580 .collect();
24581 if !order.is_empty() {
24582 passes += 1;
24583 for i in order {
24584 run_member!(i);
24585 }
24586 }
24587 break;
24588 }
24589 CyclePolicy::Iterate {
24590 max_iterations,
24591 max_change,
24592 } => {
24593 iterating = true;
24595
24596 if let Some(prev) = &prev_pass {
24602 let mut round_max_delta = 0f64;
24603 let mut round_nan = 0usize;
24604 let mut all_converged = true;
24605 for i in 0..n {
24606 if excluded[i] {
24607 continue;
24611 }
24612 let out = crate::engine::convergence::values_converged(
24613 &prev[i],
24614 &last_value[i],
24615 max_change,
24616 self.config.date_system,
24617 );
24618 if out.nan_converged {
24619 round_nan += 1;
24620 }
24621 if let Some(d) = out.abs_delta {
24622 round_max_delta = round_max_delta.max(d);
24623 }
24624 if !out.converged {
24625 all_converged = false;
24626 }
24627 }
24628 iter_max_delta = round_max_delta;
24631 iter_nan_converged = round_nan;
24632 if all_converged {
24633 converged = true;
24634 break;
24635 }
24636 }
24637
24638 if passes >= max_iterations as usize {
24650 capped = true;
24651 break;
24652 }
24653
24654 check_cancel(cancel_flag)?;
24655 prev_pass = Some(last_value.clone());
24663 for x in pos.iter_mut() {
24664 *x = -1;
24665 }
24666 changed.fill(false);
24667 passes += 1;
24668 let mut p = 0i64;
24669 for i in 0..n {
24670 if excluded[i] {
24671 continue;
24672 }
24673 run_member!(i);
24674 pos[i] = p;
24675 p += 1;
24676 }
24677 continue;
24678 }
24679 }
24680 }
24681
24682 let mut stale: Vec<usize> = Vec::new();
24685 for i in 0..n {
24686 if excluded[i] {
24687 continue;
24688 }
24689 let is_stale = out_edges[i].iter().any(|&t| {
24690 let t = t as usize;
24691 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
24692 });
24693 if is_stale {
24694 stale.push(i);
24695 }
24696 }
24697 if stale.is_empty() {
24698 break; }
24700 if 1 + settle_passes >= cap {
24701 capped = true;
24703 for (i, m) in members.iter().enumerate() {
24704 if !excluded[i] {
24705 self.stamp_cycle_error(m.vertex, &circ_error, None);
24706 excluded[i] = true;
24707 last_value[i] = circ_error.clone();
24708 stamped += 1;
24709 }
24710 }
24711 break;
24712 }
24713
24714 check_cancel(cancel_flag)?;
24715 prev_pass = None;
24722 let topo_pos = analysis.topo_positions();
24723 stale.sort_unstable_by_key(|&i| topo_pos[i]);
24724 for x in pos.iter_mut() {
24725 *x = -1;
24726 }
24727 changed.fill(false);
24728 passes += 1;
24729 settle_passes += 1;
24730 for (p, i) in stale.into_iter().enumerate() {
24731 run_member!(i);
24732 pos[i] = p as i64;
24733 }
24734 }
24735
24736 if iterating && !converged && !capped {
24741 converged = true;
24742 }
24743
24744 collector.clear_current();
24747 if let Some(d) = delta
24748 && d.mode != DeltaMode::Off
24749 {
24750 for (i, m) in members.iter().enumerate() {
24751 if let Some(cell) = m.cell
24752 && last_value[i] != snapshot[i]
24753 {
24754 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24755 }
24756 }
24757 }
24758
24759 if iterating {
24766 self.pending_iterative_redirty
24767 .extend(members.iter().map(|m| m.vertex));
24768 }
24769
24770 {
24771 let t = &mut self.last_cycle_telemetry;
24772 t.static_sccs += 1;
24773 if witnessed_cycles == 0 && stamped == 0 && !capped {
24774 t.phantom_sccs += 1;
24775 }
24776 t.live_cycles_witnessed += witnessed_cycles;
24777 t.circ_cells_stamped += stamped;
24778 t.settle_passes_total += passes;
24779 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
24780 if iterating {
24781 t.iterated_sccs += 1;
24782 if converged {
24783 t.converged_sccs += 1;
24784 }
24785 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
24786 t.nan_converged += iter_nan_converged;
24787 }
24788 if capped {
24789 t.capped_sccs += 1;
24790 }
24791 t.elapsed_ms += task_start.elapsed().as_millis();
24792 }
24793
24794 Ok(stamped)
24795 }
24796
24797 fn evaluate_vertex_recorded(
24805 &self,
24806 vertex_id: VertexId,
24807 ctx: &RecordingContext<'_, R>,
24808 collector: &LiveEdgeCollector,
24809 ) -> Result<LiteralValue, ExcelError> {
24810 if !self.graph.vertex_exists(vertex_id) {
24811 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24812 .with_message(format!("Vertex not found: {vertex_id:?}")));
24813 }
24814
24815 let kind = self.graph.get_vertex_kind(vertex_id);
24816 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24817
24818 match kind {
24819 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24820 let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
24821 return Ok(LiteralValue::Number(0.0)); };
24823 let sheet_name = self.graph.sheet_name(sheet_id);
24824 let cell_ref = self
24825 .graph
24826 .get_cell_ref(vertex_id)
24827 .expect("cell ref for vertex");
24828 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24829 interpreter
24830 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24831 .map(|cv| {
24832 crate::engine::result_finalization::finalize_formula_result(
24833 cv.into_literal(),
24834 )
24835 })
24836 }
24837 VertexKind::NamedScalar | VertexKind::NamedArray => {
24838 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24839 ExcelError::new(ExcelErrorKind::Name)
24840 .with_message("Named range metadata missing".to_string())
24841 })?;
24842
24843 match &named_range.definition {
24844 NamedDefinition::Formula { ast, .. } => {
24845 let context_sheet = match named_range.scope {
24846 NameScope::Sheet(id) => id,
24847 NameScope::Workbook => sheet_id,
24848 };
24849 let sheet_name = self.graph.sheet_name(context_sheet);
24850 let cell_ref = self
24851 .graph
24852 .get_cell_ref(vertex_id)
24853 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24854 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24855 if kind == VertexKind::NamedScalar {
24856 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24857 } else {
24858 match interpreter.evaluate_ast(ast) {
24859 Ok(cv) => match cv.into_literal() {
24860 v @ LiteralValue::Array(_) => Ok(v),
24861 other => Ok(LiteralValue::Array(vec![vec![other]])),
24862 },
24863 Err(err) => Ok(LiteralValue::Error(err)),
24864 }
24865 }
24866 }
24867 NamedDefinition::Cell(cell_ref) => {
24868 collector.record_scalar(
24872 cell_ref.sheet_id,
24873 cell_ref.coord.row(),
24874 cell_ref.coord.col(),
24875 );
24876 self.evaluate_vertex_immutable(vertex_id)
24877 }
24878 NamedDefinition::Range(range_ref) => {
24879 if range_ref.start.sheet_id == range_ref.end.sheet_id {
24880 collector.record_rect(
24881 range_ref.start.sheet_id,
24882 range_ref.start.coord.row(),
24883 range_ref.start.coord.col(),
24884 range_ref.end.coord.row(),
24885 range_ref.end.coord.col(),
24886 );
24887 }
24888 self.evaluate_vertex_immutable(vertex_id)
24889 }
24890 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
24891 }
24892 }
24893 _ => self.evaluate_vertex_immutable(vertex_id),
24894 }
24895 }
24896
24897 fn commit_spill_and_mirror(
24899 &mut self,
24900 anchor_vertex: VertexId,
24901 targets: &[CellRef],
24902 rows: Vec<Vec<LiteralValue>>,
24903 delta: Option<&mut DeltaCollector>,
24904 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24905 ) -> Result<(), ExcelError> {
24906 let prev_spill_cells = self
24907 .graph
24908 .spill_cells_for_anchor(anchor_vertex)
24909 .map(|cells| cells.to_vec())
24910 .unwrap_or_default();
24911
24912 if let Some(delta) = delta
24913 && delta.mode != DeltaMode::Off
24914 {
24915 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
24916 targets.iter().copied().collect();
24917 let empty = LiteralValue::Empty;
24918
24919 for cell in prev_spill_cells.iter() {
24921 if target_set.contains(cell) {
24922 continue;
24923 }
24924 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24925 let old = self
24926 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24927 .unwrap_or(LiteralValue::Empty);
24928 if old != empty {
24929 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24930 }
24931 }
24932
24933 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
24935 let width = rows[0].len();
24936 for (idx, cell) in targets.iter().enumerate() {
24937 let r_off = idx / width;
24938 let c_off = idx % width;
24939 let new = rows
24940 .get(r_off)
24941 .and_then(|r| r.get(c_off))
24942 .cloned()
24943 .unwrap_or(LiteralValue::Empty);
24944 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24945 let old = self
24946 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24947 .unwrap_or(LiteralValue::Empty);
24948 if old != new {
24949 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24950 }
24951 }
24952 } else {
24953 for cell in targets.iter() {
24955 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24956 let old = self
24957 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24958 .unwrap_or(LiteralValue::Empty);
24959 if !matches!(old, LiteralValue::Empty) {
24960 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24961 }
24962 }
24963 }
24964 }
24965
24966 let arrow_sheets = &self.arrow_sheets;
24969 self.spill_mgr.commit_array_with_value_probe(
24970 &mut self.graph,
24971 anchor_vertex,
24972 targets,
24973 rows.clone(),
24974 overwritable_formulas,
24975 |g, cell| {
24976 let sheet_name = g.sheet_name(cell.sheet_id);
24977 let asheet = arrow_sheets.sheet(sheet_name)?;
24978 let r0 = cell.coord.row() as usize;
24979 let c0 = cell.coord.col() as usize;
24980 let v = asheet.get_cell_value(r0, c0);
24981 if matches!(v, LiteralValue::Empty) {
24982 None
24983 } else {
24984 Some(v)
24985 }
24986 },
24987 )?;
24988
24989 if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
24990 self.record_formula_plane_structural_change(scope);
24991 }
24992 if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
24993 self.record_formula_plane_structural_change(scope);
24994 }
24995
24996 if self.config.arrow_storage_enabled
24997 && self.config.delta_overlay_enabled
24998 && self.config.write_formula_overlay_enabled
24999 {
25000 if !prev_spill_cells.is_empty() {
25001 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
25002 targets.iter().copied().collect();
25003 let empty = LiteralValue::Empty;
25004 for cell in prev_spill_cells.iter() {
25005 if !target_set.contains(cell) {
25006 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25007 self.mirror_value_to_computed_overlay(
25008 &sheet_name,
25009 cell.coord.row() + 1,
25010 cell.coord.col() + 1,
25011 &empty,
25012 );
25013 }
25014 }
25015 }
25016
25017 for (idx, cell) in targets.iter().enumerate() {
25018 if rows.is_empty() || rows[0].is_empty() {
25019 break;
25020 }
25021 let width = rows[0].len();
25022 let r_off = idx / width;
25023 let c_off = idx % width;
25024 let v = rows[r_off][c_off].clone();
25025 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25026 self.mirror_value_to_computed_overlay(
25027 &sheet_name,
25028 cell.coord.row() + 1,
25029 cell.coord.col() + 1,
25030 &v,
25031 );
25032 }
25033 }
25034 Ok(())
25035 }
25036}
25037
25038use crate::engine::effects::Effect;
25043use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
25044
25045impl<R> Engine<R>
25046where
25047 R: EvaluationContext,
25048{
25049 pub(crate) fn plan_vertex_effects(
25056 &mut self,
25057 vertex_id: VertexId,
25058 computed_value: LiteralValue,
25059 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25060 ) -> Result<Vec<Effect>, ExcelError> {
25061 let kind = self.graph.get_vertex_kind(vertex_id);
25062 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
25063
25064 if !is_formula {
25068 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
25069 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
25070 && owner != vertex_id
25071 {
25072 return Ok(Vec::new());
25073 }
25074 return Ok(vec![Effect::WriteCell {
25076 vertex_id,
25077 value: computed_value,
25078 }]);
25079 }
25080
25081 match computed_value {
25082 LiteralValue::Array(rows) => {
25083 self.plan_array_effects(vertex_id, rows, overwritable_formulas)
25084 }
25085 other => self.plan_scalar_effects(vertex_id, other),
25086 }
25087 }
25088
25089 fn plan_scalar_effects(
25091 &self,
25092 vertex_id: VertexId,
25093 value: LiteralValue,
25094 ) -> Result<Vec<Effect>, ExcelError> {
25095 let has_spill = self
25096 .graph
25097 .spill_cells_for_anchor(vertex_id)
25098 .is_some_and(|c| !c.is_empty());
25099
25100 let mut effects = Vec::new();
25101 if has_spill {
25102 effects.push(Effect::SpillClear {
25103 anchor_vertex: vertex_id,
25104 });
25105 }
25106 effects.push(Effect::WriteCell { vertex_id, value });
25107 Ok(effects)
25108 }
25109
25110 fn plan_array_effects(
25112 &mut self,
25113 vertex_id: VertexId,
25114 rows: Vec<Vec<LiteralValue>>,
25115 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25116 ) -> Result<Vec<Effect>, ExcelError> {
25117 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
25119
25120 let anchor = self
25121 .graph
25122 .get_cell_ref(vertex_id)
25123 .expect("cell ref for vertex");
25124 let sheet_id = anchor.sheet_id;
25125 let h = rows.len() as u32;
25126 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
25127
25128 let spill_cells = (h as u64).saturating_mul(w as u64);
25130 if spill_cells > self.config.spill.max_spill_cells as u64 {
25131 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
25132 }
25133
25134 const PACKED_MAX_ROW: u32 = 1_048_575;
25136 const PACKED_MAX_COL: u32 = 16_383;
25137 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
25138 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
25139 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
25140 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
25141 }
25142
25143 let mut targets = Vec::new();
25144 for r in 0..h {
25145 for c in 0..w {
25146 targets.push(self.graph.make_cell_ref_internal(
25147 sheet_id,
25148 anchor.coord.row() + r,
25149 anchor.coord.col() + c,
25150 ));
25151 }
25152 }
25153
25154 match self.spill_mgr.reserve(
25156 vertex_id,
25157 anchor,
25158 SpillShape { rows: h, cols: w },
25159 SpillMeta {
25160 epoch: self.recalc_epoch,
25161 config: self.config.spill,
25162 },
25163 ) {
25164 Ok(()) => {
25165 if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
25167 vertex_id,
25168 &targets,
25169 overwritable_formulas,
25170 ) {
25171 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
25172 }
25173
25174 if !self.graph.value_cache_enabled() {
25178 let sheet_name = self.graph.sheet_name(sheet_id);
25179 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25180 for cell in targets.iter() {
25181 if *cell == anchor {
25183 continue;
25184 }
25185 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
25187 continue;
25188 }
25189 if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
25191 && vid != vertex_id
25192 {
25193 match self.graph.get_vertex_kind(vid) {
25194 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
25195 continue;
25196 }
25197 _ => {}
25198 }
25199 }
25200
25201 let v = asheet.get_cell_value(
25202 cell.coord.row() as usize,
25203 cell.coord.col() as usize,
25204 );
25205 if !matches!(v, LiteralValue::Empty) {
25206 return self.plan_spill_error_effects(
25207 vertex_id,
25208 "BlockedByValue",
25209 h,
25210 w,
25211 );
25212 }
25213 }
25214 }
25215 }
25216
25217 let top_left = rows
25218 .first()
25219 .and_then(|r| r.first())
25220 .cloned()
25221 .unwrap_or(LiteralValue::Empty);
25222
25223 let mut effects = Vec::new();
25224 let has_prev = self
25226 .graph
25227 .spill_cells_for_anchor(vertex_id)
25228 .is_some_and(|c| !c.is_empty());
25229 if has_prev {
25230 effects.push(Effect::SpillClear {
25231 anchor_vertex: vertex_id,
25232 });
25233 }
25234 effects.push(Effect::SpillCommit {
25235 anchor_vertex: vertex_id,
25236 anchor_cell: anchor,
25237 target_cells: targets,
25238 values: rows,
25239 });
25240 effects.push(Effect::WriteCell {
25241 vertex_id,
25242 value: top_left,
25243 });
25244 Ok(effects)
25245 }
25246 Err(e) => {
25247 let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
25248 self.plan_spill_error_effects(vertex_id, &msg, h, w)
25249 }
25250 }
25251 }
25252
25253 fn plan_spill_error_effects(
25255 &self,
25256 vertex_id: VertexId,
25257 message: &str,
25258 expected_rows: u32,
25259 expected_cols: u32,
25260 ) -> Result<Vec<Effect>, ExcelError> {
25261 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
25262 .with_message(message)
25263 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
25264 expected_rows,
25265 expected_cols,
25266 });
25267 let spill_val = LiteralValue::Error(spill_err);
25268
25269 let effects = vec![
25270 Effect::SpillClear {
25271 anchor_vertex: vertex_id,
25272 },
25273 Effect::WriteCell {
25274 vertex_id,
25275 value: spill_val,
25276 },
25277 ];
25278 Ok(effects)
25279 }
25280
25281 pub(crate) fn apply_effect(
25283 &mut self,
25284 effect: &Effect,
25285 delta: Option<&mut DeltaCollector>,
25286 log: Option<&mut ChangeLog>,
25287 ) -> Result<(), ExcelError> {
25288 self.apply_effect_with_computed_writes(effect, delta, log, None)
25289 }
25290
25291 fn apply_effect_with_computed_writes(
25292 &mut self,
25293 effect: &Effect,
25294 delta: Option<&mut DeltaCollector>,
25295 log: Option<&mut ChangeLog>,
25296 computed_writes: Option<&mut ComputedWriteBuffer>,
25297 ) -> Result<(), ExcelError> {
25298 match effect {
25299 Effect::WriteCell { vertex_id, value } => {
25300 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
25301 }
25302 Effect::SpillClear { anchor_vertex } => {
25303 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
25304 }
25305 Effect::SpillCommit {
25306 anchor_vertex,
25307 anchor_cell: _,
25308 target_cells,
25309 values,
25310 } => {
25311 self.apply_spill_commit(
25312 *anchor_vertex,
25313 target_cells,
25314 values.clone(),
25315 delta,
25316 log,
25317 computed_writes,
25318 )?;
25319 }
25320 }
25321 Ok(())
25322 }
25323
25324 fn apply_write_cell(
25326 &mut self,
25327 vertex_id: VertexId,
25328 value: &LiteralValue,
25329 delta: Option<&mut DeltaCollector>,
25330 mut computed_writes: Option<&mut ComputedWriteBuffer>,
25331 ) -> Result<(), ExcelError> {
25332 if let Some(d) = delta
25333 && d.mode != DeltaMode::Off
25334 {
25335 if let Some(buffer) = computed_writes.as_deref_mut() {
25336 self.flush_computed_write_buffer(buffer)?;
25337 }
25338 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
25339 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25340 let old = self
25341 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25342 .unwrap_or(LiteralValue::Empty);
25343 if old != *value {
25344 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25345 }
25346 }
25347 }
25348 self.graph.update_vertex_value(vertex_id, value.clone());
25349 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
25350 Ok(())
25351 }
25352
25353 fn apply_spill_clear(
25355 &mut self,
25356 anchor_vertex: VertexId,
25357 delta: Option<&mut DeltaCollector>,
25358 log: Option<&mut ChangeLog>,
25359 computed_writes: Option<&mut ComputedWriteBuffer>,
25360 ) -> Result<(), ExcelError> {
25361 if let Some(buffer) = computed_writes {
25362 self.flush_computed_write_buffer(buffer)?;
25363 }
25364
25365 let spill_cells = self
25366 .graph
25367 .spill_cells_for_anchor(anchor_vertex)
25368 .map(|cells| cells.to_vec())
25369 .unwrap_or_default();
25370 if spill_cells.is_empty() {
25371 return Ok(());
25372 }
25373
25374 let snapshot = if log.is_some() {
25376 self.snapshot_spill_for_anchor(anchor_vertex)
25377 } else {
25378 None
25379 };
25380
25381 if let Some(d) = delta
25383 && d.mode != DeltaMode::Off
25384 {
25385 let empty = LiteralValue::Empty;
25386 for cell in spill_cells.iter() {
25387 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25388 let old = self
25389 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25390 .unwrap_or(LiteralValue::Empty);
25391 if old != empty {
25392 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25393 }
25394 }
25395 }
25396
25397 self.graph.clear_spill_region(anchor_vertex);
25398 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
25399 self.record_formula_plane_structural_change(scope);
25400 }
25401
25402 if self.config.arrow_storage_enabled
25404 && self.config.delta_overlay_enabled
25405 && self.config.write_formula_overlay_enabled
25406 {
25407 let empty = LiteralValue::Empty;
25408 for cell in spill_cells.iter() {
25409 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25410 self.mirror_value_to_computed_overlay(
25411 &sheet_name,
25412 cell.coord.row() + 1,
25413 cell.coord.col() + 1,
25414 &empty,
25415 );
25416 }
25417 }
25418
25419 if let Some(log) = log
25421 && let Some(old) = snapshot
25422 {
25423 log.record(ChangeEvent::SpillCleared {
25424 anchor: anchor_vertex,
25425 old,
25426 });
25427 }
25428 Ok(())
25429 }
25430
25431 fn apply_spill_commit(
25433 &mut self,
25434 anchor_vertex: VertexId,
25435 target_cells: &[CellRef],
25436 values: Vec<Vec<LiteralValue>>,
25437 delta: Option<&mut DeltaCollector>,
25438 log: Option<&mut ChangeLog>,
25439 computed_writes: Option<&mut ComputedWriteBuffer>,
25440 ) -> Result<(), ExcelError> {
25441 if let Some(buffer) = computed_writes {
25442 self.flush_computed_write_buffer(buffer)?;
25443 }
25444
25445 let old_snapshot = if log.is_some() {
25447 self.snapshot_spill_for_anchor(anchor_vertex)
25448 } else {
25449 None
25450 };
25451
25452 self.commit_spill_and_mirror(
25454 anchor_vertex,
25455 target_cells,
25456 values.clone(),
25457 delta,
25458 None, )?;
25460
25461 if let Some(log) = log {
25463 log.record(ChangeEvent::SpillCommitted {
25464 anchor: anchor_vertex,
25465 old: old_snapshot,
25466 new: SpillSnapshot {
25467 target_cells: target_cells.to_vec(),
25468 values,
25469 },
25470 });
25471 }
25472 Ok(())
25473 }
25474
25475 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
25480 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
25481 if cells.is_empty() {
25482 return None;
25483 }
25484
25485 let max = self.config.spill.max_spill_cells as usize;
25486 let mut cells = cells;
25487 if cells.len() > max {
25488 cells.truncate(max);
25489 }
25490
25491 let first = *cells.first().expect("non-empty spill cells");
25492 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
25493 let row0 = first.coord.row();
25494 let col0 = first.coord.col();
25495
25496 let mut max_row = row0;
25497 let mut max_col = col0;
25498 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
25499 for cell in &cells {
25500 max_row = max_row.max(cell.coord.row());
25501 max_col = max_col.max(cell.coord.col());
25502 let v = self
25503 .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25504 .unwrap_or(LiteralValue::Empty);
25505 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
25506 }
25507
25508 let rows = (max_row - row0 + 1) as usize;
25509 let cols = (max_col - col0 + 1) as usize;
25510 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
25511 for r in 0..rows {
25512 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
25513 for c in 0..cols {
25514 row.push(
25515 by_coord
25516 .get(&(row0 + r as u32, col0 + c as u32))
25517 .cloned()
25518 .unwrap_or(LiteralValue::Empty),
25519 );
25520 }
25521 values.push(row);
25522 }
25523
25524 Some(SpillSnapshot {
25525 target_cells: cells,
25526 values,
25527 })
25528 }
25529
25530 fn flush_before_range_dependent_vertex(
25531 &mut self,
25532 vertex_id: VertexId,
25533 computed_writes: &mut ComputedWriteBuffer,
25534 ) -> Result<(), ExcelError> {
25535 if self.graph.get_range_dependencies(vertex_id).is_some() {
25536 self.flush_computed_write_buffer(computed_writes)?;
25537 }
25538 Ok(())
25539 }
25540
25541 fn plan_vertex_effects_with_computed_flush(
25542 &mut self,
25543 vertex_id: VertexId,
25544 computed_value: LiteralValue,
25545 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25546 computed_writes: &mut ComputedWriteBuffer,
25547 ) -> Result<Vec<Effect>, ExcelError> {
25548 if matches!(&computed_value, LiteralValue::Array(_)) {
25549 self.flush_computed_write_buffer(computed_writes)?;
25550 }
25551 self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
25552 }
25553
25554 fn evaluate_small_layer_direct_effects(
25557 &mut self,
25558 layer: &super::scheduler::Layer,
25559 mut delta: Option<&mut DeltaCollector>,
25560 mut log: Option<&mut ChangeLog>,
25561 cancel_flag: Option<&AtomicBool>,
25562 cancel_check_every: usize,
25563 cancel_message: &'static str,
25564 ) -> Result<usize, ExcelError> {
25565 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25566 if cancel_check_every > 0
25567 && i % cancel_check_every == 0
25568 && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
25569 {
25570 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25571 .with_message(cancel_message.to_string()));
25572 }
25573 let value = match self.evaluate_vertex_immutable(vertex_id) {
25574 Ok(v) => v,
25575 Err(e) => LiteralValue::Error(e),
25576 };
25577 let effects = self.plan_vertex_effects(vertex_id, value, None)?;
25578 for effect in &effects {
25579 self.apply_effect_with_computed_writes(
25580 effect,
25581 delta.as_deref_mut(),
25582 log.as_deref_mut(),
25583 None,
25584 )?;
25585 }
25586 }
25587 Ok(layer.vertices.len())
25588 }
25589
25590 fn evaluate_layer_sequential_effects(
25592 &mut self,
25593 layer: &super::scheduler::Layer,
25594 ) -> Result<usize, ExcelError> {
25595 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25596 return self.evaluate_small_layer_direct_effects(
25597 layer,
25598 None,
25599 None,
25600 None,
25601 0,
25602 "Evaluation cancelled within layer",
25603 );
25604 }
25605
25606 let mut computed_writes = ComputedWriteBuffer::default();
25607 for &vertex_id in &layer.vertices {
25608 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25609 let value = match self.evaluate_vertex_immutable(vertex_id) {
25610 Ok(v) => v,
25611 Err(e) => LiteralValue::Error(e),
25612 };
25613 let effects = match self.plan_vertex_effects_with_computed_flush(
25614 vertex_id,
25615 value,
25616 None,
25617 &mut computed_writes,
25618 ) {
25619 Ok(effects) => effects,
25620 Err(e) => {
25621 self.flush_computed_write_buffer(&mut computed_writes)?;
25622 return Err(e);
25623 }
25624 };
25625 for effect in &effects {
25626 if let Err(e) = self.apply_effect_with_computed_writes(
25627 effect,
25628 None,
25629 None,
25630 Some(&mut computed_writes),
25631 ) {
25632 self.flush_computed_write_buffer(&mut computed_writes)?;
25633 return Err(e);
25634 }
25635 }
25636 }
25637 self.flush_computed_write_buffer(&mut computed_writes)?;
25638 Ok(layer.vertices.len())
25639 }
25640
25641 fn evaluate_layer_sequential_with_delta_effects(
25643 &mut self,
25644 layer: &super::scheduler::Layer,
25645 delta: &mut DeltaCollector,
25646 ) -> Result<usize, ExcelError> {
25647 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25648 return self.evaluate_small_layer_direct_effects(
25649 layer,
25650 Some(delta),
25651 None,
25652 None,
25653 0,
25654 "Evaluation cancelled within layer",
25655 );
25656 }
25657
25658 let mut computed_writes = ComputedWriteBuffer::default();
25659 for &vertex_id in &layer.vertices {
25660 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25661 let value = match self.evaluate_vertex_immutable(vertex_id) {
25662 Ok(v) => v,
25663 Err(e) => LiteralValue::Error(e),
25664 };
25665 let effects = match self.plan_vertex_effects_with_computed_flush(
25666 vertex_id,
25667 value,
25668 None,
25669 &mut computed_writes,
25670 ) {
25671 Ok(effects) => effects,
25672 Err(e) => {
25673 self.flush_computed_write_buffer(&mut computed_writes)?;
25674 return Err(e);
25675 }
25676 };
25677 for effect in &effects {
25678 if let Err(e) = self.apply_effect_with_computed_writes(
25679 effect,
25680 Some(delta),
25681 None,
25682 Some(&mut computed_writes),
25683 ) {
25684 self.flush_computed_write_buffer(&mut computed_writes)?;
25685 return Err(e);
25686 }
25687 }
25688 }
25689 self.flush_computed_write_buffer(&mut computed_writes)?;
25690 Ok(layer.vertices.len())
25691 }
25692
25693 fn evaluate_layer_sequential_cancellable_effects(
25695 &mut self,
25696 layer: &super::scheduler::Layer,
25697 cancel_flag: &AtomicBool,
25698 ) -> Result<usize, ExcelError> {
25699 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25700 return self.evaluate_small_layer_direct_effects(
25701 layer,
25702 None,
25703 None,
25704 Some(cancel_flag),
25705 256,
25706 "Evaluation cancelled within layer",
25707 );
25708 }
25709
25710 let mut computed_writes = ComputedWriteBuffer::default();
25711 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25712 if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
25713 self.flush_computed_write_buffer(&mut computed_writes)?;
25714 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25715 .with_message("Evaluation cancelled within layer".to_string()));
25716 }
25717 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25718 let value = match self.evaluate_vertex_immutable(vertex_id) {
25719 Ok(v) => v,
25720 Err(e) => LiteralValue::Error(e),
25721 };
25722 let effects = match self.plan_vertex_effects_with_computed_flush(
25723 vertex_id,
25724 value,
25725 None,
25726 &mut computed_writes,
25727 ) {
25728 Ok(effects) => effects,
25729 Err(e) => {
25730 self.flush_computed_write_buffer(&mut computed_writes)?;
25731 return Err(e);
25732 }
25733 };
25734 for effect in &effects {
25735 if let Err(e) = self.apply_effect_with_computed_writes(
25736 effect,
25737 None,
25738 None,
25739 Some(&mut computed_writes),
25740 ) {
25741 self.flush_computed_write_buffer(&mut computed_writes)?;
25742 return Err(e);
25743 }
25744 }
25745 }
25746 self.flush_computed_write_buffer(&mut computed_writes)?;
25747 Ok(layer.vertices.len())
25748 }
25749
25750 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
25752 &mut self,
25753 layer: &super::scheduler::Layer,
25754 cancel_flag: &AtomicBool,
25755 ) -> Result<usize, ExcelError> {
25756 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25757 return self.evaluate_small_layer_direct_effects(
25758 layer,
25759 None,
25760 None,
25761 Some(cancel_flag),
25762 128,
25763 "Demand-driven evaluation cancelled within layer",
25764 );
25765 }
25766
25767 let mut computed_writes = ComputedWriteBuffer::default();
25768 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25769 if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
25770 self.flush_computed_write_buffer(&mut computed_writes)?;
25771 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25772 .with_message("Demand-driven evaluation cancelled within layer".to_string()));
25773 }
25774 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25775 let value = match self.evaluate_vertex_immutable(vertex_id) {
25776 Ok(v) => v,
25777 Err(e) => LiteralValue::Error(e),
25778 };
25779 let effects = match self.plan_vertex_effects_with_computed_flush(
25780 vertex_id,
25781 value,
25782 None,
25783 &mut computed_writes,
25784 ) {
25785 Ok(effects) => effects,
25786 Err(e) => {
25787 self.flush_computed_write_buffer(&mut computed_writes)?;
25788 return Err(e);
25789 }
25790 };
25791 for effect in &effects {
25792 if let Err(e) = self.apply_effect_with_computed_writes(
25793 effect,
25794 None,
25795 None,
25796 Some(&mut computed_writes),
25797 ) {
25798 self.flush_computed_write_buffer(&mut computed_writes)?;
25799 return Err(e);
25800 }
25801 }
25802 }
25803 self.flush_computed_write_buffer(&mut computed_writes)?;
25804 Ok(layer.vertices.len())
25805 }
25806
25807 fn evaluate_layer_parallel_effects(
25809 &mut self,
25810 layer: &super::scheduler::Layer,
25811 ) -> Result<usize, ExcelError> {
25812 use rayon::prelude::*;
25813
25814 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25815
25816 let mut phase1: Vec<VertexId> = Vec::new();
25817 let mut phase2: Vec<VertexId> = Vec::new();
25818 for &vid in &layer.vertices {
25819 if self.graph.get_range_dependencies(vid).is_some() {
25820 phase2.push(vid);
25821 } else {
25822 phase1.push(vid);
25823 }
25824 }
25825
25826 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25827 let mut applied = 0usize;
25828
25829 for group in [&phase1[..], &phase2[..]] {
25830 if group.is_empty() {
25831 continue;
25832 }
25833 let mut computed_writes = ComputedWriteBuffer::default();
25834
25835 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25836 thread_pool.install(|| {
25837 group
25838 .par_iter()
25839 .map(
25840 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25841 Ok(v) => Ok((vertex_id, v)),
25842 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25843 },
25844 )
25845 .collect()
25846 });
25847
25848 match results {
25849 Ok(vertex_results) => {
25850 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25853 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25854 for (vertex_id, result) in vertex_results {
25855 if matches!(result, LiteralValue::Array(_)) {
25856 arrays.push((vertex_id, result));
25857 } else {
25858 others.push((vertex_id, result));
25859 }
25860 }
25861 for (vertex_id, result) in arrays {
25862 let effects = match self.plan_vertex_effects_with_computed_flush(
25863 vertex_id,
25864 result,
25865 Some(&inflight),
25866 &mut computed_writes,
25867 ) {
25868 Ok(effects) => effects,
25869 Err(e) => {
25870 self.flush_computed_write_buffer(&mut computed_writes)?;
25871 return Err(e);
25872 }
25873 };
25874 for effect in &effects {
25875 if let Err(e) = self.apply_effect_with_computed_writes(
25876 effect,
25877 None,
25878 None,
25879 Some(&mut computed_writes),
25880 ) {
25881 self.flush_computed_write_buffer(&mut computed_writes)?;
25882 return Err(e);
25883 }
25884 }
25885 applied = applied.saturating_add(1);
25886 }
25887 self.flush_computed_write_buffer(&mut computed_writes)?;
25889 for (vertex_id, result) in others {
25890 let effects = match self.plan_vertex_effects_with_computed_flush(
25891 vertex_id,
25892 result,
25893 Some(&inflight),
25894 &mut computed_writes,
25895 ) {
25896 Ok(effects) => effects,
25897 Err(e) => {
25898 self.flush_computed_write_buffer(&mut computed_writes)?;
25899 return Err(e);
25900 }
25901 };
25902 for effect in &effects {
25903 if let Err(e) = self.apply_effect_with_computed_writes(
25904 effect,
25905 None,
25906 None,
25907 Some(&mut computed_writes),
25908 ) {
25909 self.flush_computed_write_buffer(&mut computed_writes)?;
25910 return Err(e);
25911 }
25912 }
25913 applied = applied.saturating_add(1);
25914 }
25915 self.flush_computed_write_buffer(&mut computed_writes)?;
25917 }
25918 Err(e) => {
25919 self.flush_computed_write_buffer(&mut computed_writes)?;
25920 return Err(e);
25921 }
25922 }
25923 }
25924
25925 Ok(applied)
25926 }
25927
25928 fn evaluate_layer_parallel_with_delta_effects(
25930 &mut self,
25931 layer: &super::scheduler::Layer,
25932 delta: &mut DeltaCollector,
25933 ) -> Result<usize, ExcelError> {
25934 use rayon::prelude::*;
25935
25936 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25937
25938 let mut phase1: Vec<VertexId> = Vec::new();
25939 let mut phase2: Vec<VertexId> = Vec::new();
25940 for &vid in &layer.vertices {
25941 if self.graph.get_range_dependencies(vid).is_some() {
25942 phase2.push(vid);
25943 } else {
25944 phase1.push(vid);
25945 }
25946 }
25947
25948 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25949 let mut applied = 0usize;
25950
25951 for group in [&phase1[..], &phase2[..]] {
25952 if group.is_empty() {
25953 continue;
25954 }
25955 let mut computed_writes = ComputedWriteBuffer::default();
25956 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25957 thread_pool.install(|| {
25958 group
25959 .par_iter()
25960 .map(
25961 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25962 Ok(v) => Ok((vertex_id, v)),
25963 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25964 },
25965 )
25966 .collect()
25967 });
25968
25969 match results {
25970 Ok(vertex_results) => {
25971 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25972 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25973 for (vertex_id, result) in vertex_results {
25974 if matches!(result, LiteralValue::Array(_)) {
25975 arrays.push((vertex_id, result));
25976 } else {
25977 others.push((vertex_id, result));
25978 }
25979 }
25980 for (vertex_id, result) in arrays {
25981 let effects = match self.plan_vertex_effects_with_computed_flush(
25982 vertex_id,
25983 result,
25984 Some(&inflight),
25985 &mut computed_writes,
25986 ) {
25987 Ok(effects) => effects,
25988 Err(e) => {
25989 self.flush_computed_write_buffer(&mut computed_writes)?;
25990 return Err(e);
25991 }
25992 };
25993 for effect in &effects {
25994 if let Err(e) = self.apply_effect_with_computed_writes(
25995 effect,
25996 Some(delta),
25997 None,
25998 Some(&mut computed_writes),
25999 ) {
26000 self.flush_computed_write_buffer(&mut computed_writes)?;
26001 return Err(e);
26002 }
26003 }
26004 applied = applied.saturating_add(1);
26005 }
26006 self.flush_computed_write_buffer(&mut computed_writes)?;
26007 for (vertex_id, result) in others {
26008 let effects = match self.plan_vertex_effects_with_computed_flush(
26009 vertex_id,
26010 result,
26011 Some(&inflight),
26012 &mut computed_writes,
26013 ) {
26014 Ok(effects) => effects,
26015 Err(e) => {
26016 self.flush_computed_write_buffer(&mut computed_writes)?;
26017 return Err(e);
26018 }
26019 };
26020 for effect in &effects {
26021 if let Err(e) = self.apply_effect_with_computed_writes(
26022 effect,
26023 Some(delta),
26024 None,
26025 Some(&mut computed_writes),
26026 ) {
26027 self.flush_computed_write_buffer(&mut computed_writes)?;
26028 return Err(e);
26029 }
26030 }
26031 applied = applied.saturating_add(1);
26032 }
26033 self.flush_computed_write_buffer(&mut computed_writes)?;
26034 }
26035 Err(e) => {
26036 self.flush_computed_write_buffer(&mut computed_writes)?;
26037 return Err(e);
26038 }
26039 }
26040 }
26041
26042 Ok(applied)
26043 }
26044
26045 fn evaluate_layer_parallel_cancellable_effects(
26047 &mut self,
26048 layer: &super::scheduler::Layer,
26049 cancel_flag: &AtomicBool,
26050 ) -> Result<usize, ExcelError> {
26051 use rayon::prelude::*;
26052
26053 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26054
26055 if cancel_flag.load(Ordering::Relaxed) {
26056 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26057 .with_message("Parallel evaluation cancelled before starting".to_string()));
26058 }
26059
26060 let mut phase1: Vec<VertexId> = Vec::new();
26061 let mut phase2: Vec<VertexId> = Vec::new();
26062 for &vid in &layer.vertices {
26063 if self.graph.get_range_dependencies(vid).is_some() {
26064 phase2.push(vid);
26065 } else {
26066 phase1.push(vid);
26067 }
26068 }
26069
26070 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26071 let mut applied = 0usize;
26072
26073 for group in [&phase1[..], &phase2[..]] {
26074 if group.is_empty() {
26075 continue;
26076 }
26077 let mut computed_writes = ComputedWriteBuffer::default();
26078
26079 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26080 thread_pool.install(|| {
26081 group
26082 .par_iter()
26083 .map(|&vertex_id| {
26084 if cancel_flag.load(Ordering::Relaxed) {
26085 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26086 .with_message(
26087 "Parallel evaluation cancelled during execution"
26088 .to_string(),
26089 ));
26090 }
26091 match self.evaluate_vertex_immutable(vertex_id) {
26092 Ok(v) => Ok((vertex_id, v)),
26093 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26094 }
26095 })
26096 .collect()
26097 });
26098
26099 match results {
26100 Ok(vertex_results) => {
26101 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26102 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26103 for (vertex_id, result) in vertex_results {
26104 if matches!(result, LiteralValue::Array(_)) {
26105 arrays.push((vertex_id, result));
26106 } else {
26107 others.push((vertex_id, result));
26108 }
26109 }
26110 for (vertex_id, result) in arrays {
26111 let effects = match self.plan_vertex_effects_with_computed_flush(
26112 vertex_id,
26113 result,
26114 Some(&inflight),
26115 &mut computed_writes,
26116 ) {
26117 Ok(effects) => effects,
26118 Err(e) => {
26119 self.flush_computed_write_buffer(&mut computed_writes)?;
26120 return Err(e);
26121 }
26122 };
26123 for effect in &effects {
26124 if let Err(e) = self.apply_effect_with_computed_writes(
26125 effect,
26126 None,
26127 None,
26128 Some(&mut computed_writes),
26129 ) {
26130 self.flush_computed_write_buffer(&mut computed_writes)?;
26131 return Err(e);
26132 }
26133 }
26134 applied = applied.saturating_add(1);
26135 }
26136 self.flush_computed_write_buffer(&mut computed_writes)?;
26137 for (vertex_id, result) in others {
26138 let effects = match self.plan_vertex_effects_with_computed_flush(
26139 vertex_id,
26140 result,
26141 Some(&inflight),
26142 &mut computed_writes,
26143 ) {
26144 Ok(effects) => effects,
26145 Err(e) => {
26146 self.flush_computed_write_buffer(&mut computed_writes)?;
26147 return Err(e);
26148 }
26149 };
26150 for effect in &effects {
26151 if let Err(e) = self.apply_effect_with_computed_writes(
26152 effect,
26153 None,
26154 None,
26155 Some(&mut computed_writes),
26156 ) {
26157 self.flush_computed_write_buffer(&mut computed_writes)?;
26158 return Err(e);
26159 }
26160 }
26161 applied = applied.saturating_add(1);
26162 }
26163 self.flush_computed_write_buffer(&mut computed_writes)?;
26164 }
26165 Err(e) => {
26166 self.flush_computed_write_buffer(&mut computed_writes)?;
26167 return Err(e);
26168 }
26169 }
26170 }
26171
26172 Ok(applied)
26173 }
26174
26175 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
26182 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
26183 engine.evaluate_all_logged_unobserved(log)
26184 })
26185 }
26186
26187 fn evaluate_all_logged_unobserved(
26188 &mut self,
26189 log: &mut ChangeLog,
26190 ) -> Result<EvalResult, ExcelError> {
26191 self.observe_function_semantic_epoch()?;
26192 self.begin_evaluation_request();
26193 let _source_cache = self.source_cache_session();
26194 self.validate_deterministic_mode()?;
26195 if self.config.defer_graph_building {
26196 self.build_graph_all()?;
26197 }
26198 if self.graph.formula_authority().active_span_count() > 0 {
26199 return self.evaluate_authoritative_formula_plane_all();
26200 }
26201 self.reset_virtual_dep_telemetry_if_disabled();
26202 let start = crate::instant::FzInstant::now();
26203 let mut computed_vertices = 0;
26204 let mut cycle_errors = 0;
26205
26206 let mut replan_iterations = 0;
26207 const MAX_REPLAN: usize = 5;
26208 let mut telemetry = self
26209 .config
26210 .enable_virtual_dep_telemetry
26211 .then(|| self.start_virtual_dep_telemetry());
26212
26213 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
26214
26215 let result = (|| -> Result<EvalResult, ExcelError> {
26216 loop {
26217 let to_evaluate = self.graph.get_evaluation_vertices();
26218 if to_evaluate.is_empty() {
26219 if let Some(t) = telemetry.as_mut()
26220 && t.bailout_reason.is_none()
26221 {
26222 t.bailout_reason = Some("no_work");
26223 }
26224 break;
26225 }
26226
26227 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
26228 if let Some(t) = telemetry.as_mut() {
26229 Self::accumulate_schedule_meta(t, &meta);
26230 }
26231
26232 for &unit in &schedule.units {
26235 match unit {
26236 ScheduleUnit::Cycle(i) => {
26237 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
26248 {
26249 cycle_errors += 1;
26250 }
26251 }
26252 ScheduleUnit::Layer(i) => {
26253 computed_vertices +=
26254 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
26255 }
26256 }
26257 }
26258
26259 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
26260 if let Some(t) = telemetry.as_mut() {
26261 t.changed_vdeps_total += changed_vertices.len();
26262 }
26263 self.resource_checkpoint(0)?;
26264 self.graph.clear_dirty_flags(&to_evaluate);
26265 for v in &changed_vertices {
26266 self.graph.set_dirty(*v, true);
26267 }
26268
26269 if changed_vertices.is_empty() {
26270 if let Some(t) = telemetry.as_mut() {
26271 t.bailout_reason = Some("converged");
26272 }
26273 break;
26274 }
26275 if replan_iterations >= MAX_REPLAN {
26276 if let Some(mut t) = telemetry.take() {
26277 t.bailout_reason = Some("max_replan");
26278 t.replan_iterations = replan_iterations;
26279 self.last_virtual_dep_telemetry = t;
26280 }
26281 return Err(
26282 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
26283 );
26284 }
26285 replan_iterations += 1;
26286 }
26287
26288 if let Some(mut t) = telemetry {
26289 t.replan_iterations = replan_iterations;
26290 self.last_virtual_dep_telemetry = t;
26291 }
26292
26293 self.redirty_for_next_recalc();
26294 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
26295
26296 Ok(EvalResult {
26297 computed_vertices,
26298 cycle_errors,
26299 elapsed: start.elapsed(),
26300 })
26301 })();
26302 log.end_compound();
26303 result
26304 }
26305
26306 fn evaluate_layer_logged(
26308 &mut self,
26309 layer: &super::scheduler::Layer,
26310 log: &mut ChangeLog,
26311 ) -> Result<usize, ExcelError> {
26312 self.resource_checkpoint(layer.vertices.len() as u64)?;
26313 let mut computed_writes = ComputedWriteBuffer::default();
26314 for &vertex_id in &layer.vertices {
26315 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
26316 let value = match self.evaluate_vertex_immutable(vertex_id) {
26317 Ok(v) => v,
26318 Err(e) => LiteralValue::Error(e),
26319 };
26320 let effects = match self.plan_vertex_effects_with_computed_flush(
26321 vertex_id,
26322 value,
26323 None,
26324 &mut computed_writes,
26325 ) {
26326 Ok(effects) => effects,
26327 Err(e) => {
26328 self.flush_computed_write_buffer(&mut computed_writes)?;
26329 return Err(e);
26330 }
26331 };
26332 for effect in &effects {
26333 if let Err(e) = self.apply_effect_with_computed_writes(
26334 effect,
26335 None,
26336 Some(log),
26337 Some(&mut computed_writes),
26338 ) {
26339 self.flush_computed_write_buffer(&mut computed_writes)?;
26340 return Err(e);
26341 }
26342 }
26343 }
26344 self.flush_computed_write_buffer(&mut computed_writes)?;
26345 Ok(layer.vertices.len())
26346 }
26347}