1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5 DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::editor::change_log::MutationCapture;
8use crate::engine::graph::prepared_legacy_graph::{
9 PreparedLegacyGraphError, PreparedLegacyGraphPlan,
10};
11use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
12use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
13use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
14use crate::engine::live_graph::analyze_live_graph;
15use crate::engine::lookup_index_cache::{
16 BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
17 LookupIndexKey, estimate_bytes,
18};
19use crate::engine::named_range::{NameScope, NamedDefinition};
20use crate::engine::range_view::RangeView;
21use crate::engine::row_visibility::RowVisibilityState;
22use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
23use crate::engine::target_preparation::{
24 StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
25};
26use crate::engine::used_extent::{
27 ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
28};
29use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
30use crate::engine::{
31 ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
32 EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
33 EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
34 FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
35 FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
36 FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
37 FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
38 VertexId, VertexKind, VisibilityMaskMode,
39};
40use crate::formula_plane::placement::prepare_anchor_once_fragment;
41use crate::formula_plane::placement::{
42 CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
43 place_candidate_family_with_analyses, prepare_anchor_once_family,
44 split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
45 validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
46};
47use crate::formula_plane::producer::{
48 DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
49 FormulaProducerWork, ProducerDirtyDomain, ProjectionResult, SpanReadSummary,
50};
51use crate::formula_plane::region_index::{BoundedRegionQueryResult, DirtyDomain, Region};
52use crate::formula_plane::runtime::{
53 FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
54};
55use crate::formula_plane::scheduler::{
56 MixedLayer, MixedSchedule, MixedScheduleFallback, MixedScheduleFallbackReason, MixedTopology,
57 MixedTopologyCompileResult, MixedTopologyCompileStats, MixedTopologyConfig,
58 build_demand_closure_cached, build_demand_closure_in_memory_runs, build_demand_closure_paged,
59 build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
60 schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
61 schedule_dirty_work_repeated_passes,
62};
63#[cfg(not(target_arch = "wasm32"))]
64use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
65#[cfg(not(target_arch = "wasm32"))]
66use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
67#[cfg(test)]
68use crate::formula_plane::span_eval::SpanEvalReport;
69use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
70use crate::formula_plane::structural::relocate_ast_for_template_placement;
71use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
72use crate::function::FnCaps;
73use crate::interpreter::Interpreter;
74use crate::reference::{CellRef, Coord, RangeRef};
75use crate::traits::FunctionProvider;
76use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
77use formualizer_common::{
78 CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
79};
80use formualizer_parse::parser::ReferenceType;
81use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
82use rayon::ThreadPoolBuilder;
83use rustc_hash::{FxHashMap, FxHashSet};
84use std::collections::{BTreeMap, BTreeSet, VecDeque};
85use std::sync::Arc;
86use std::sync::atomic::{AtomicBool, Ordering};
87use std::time::Instant;
88
89type StagedFormulaEntry = (u32, u32, String);
90type StagedSheetParts = (
91 Vec<StagedFormulaEntry>,
92 Option<crate::engine::DeferredFormulaPackage>,
93);
94
95#[derive(Default)]
105pub(crate) struct StagedSheet {
106 entries: Vec<StagedFormulaEntry>,
107 index: FxHashMap<(u32, u32), usize>,
108 deferred_package: Option<crate::engine::DeferredFormulaPackage>,
109}
110
111impl StagedSheet {
112 fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
113 package
114 .invalidated
115 .extend(package.families.iter().map(|family| family.source_id));
116 package.invalidated.extend(
117 package
118 .partitioned_families
119 .iter()
120 .map(|family| family.source_id),
121 );
122 }
123
124 fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
125 let Some(package) = self.deferred_package.as_mut() else {
126 return;
127 };
128 let family = package
129 .replay
130 .lock()
131 .ok()
132 .and_then(|mut replay| replay.formula_at(row, col).ok())
133 .flatten()
134 .and_then(|record| record.partition_owner.or(record.family));
135 if let Some(family) = family {
136 package.invalidated.insert(family);
137 } else {
138 Self::invalidate_all_deferred_families(package);
142 }
143 package.suppressed.insert((row, col));
144 }
145
146 fn reconcile_attached_deferred_package(&mut self) {
147 let Some(package) = self.deferred_package.as_mut() else {
148 return;
149 };
150 if self.entries.is_empty() {
151 return;
152 }
153
154 let replayed = (|| {
155 let mut disposition = crate::engine::FormulaReplayDisposition::default();
156 for partition in &package.partitioned_families {
157 disposition.register_partition(partition, false)?;
158 }
159 package
160 .replay
161 .lock()
162 .map_err(|_| "deferred formula spool lock poisoned".to_string())?
163 .replay_partitioned(&disposition, &package.partitioned_families)
164 })();
165
166 let Ok(records) = replayed else {
167 Self::invalidate_all_deferred_families(package);
168 package
169 .suppressed
170 .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
171 return;
172 };
173 let mut owners = FxHashMap::default();
174 for record in &records {
175 owners
176 .entry((record.row, record.col))
177 .or_insert(record.partition_owner.or(record.family));
178 }
179 for (row, col, _) in &self.entries {
180 if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
181 package.invalidated.insert(family);
182 } else {
183 Self::invalidate_all_deferred_families(package);
184 }
185 package.suppressed.insert((*row, *col));
186 }
187 package.reconciliation_replay = Some(records);
188 }
189
190 fn stage(&mut self, row: u32, col: u32, text: String) {
191 self.invalidate_deferred_at(row, col);
192 match self.index.entry((row, col)) {
193 std::collections::hash_map::Entry::Occupied(slot) => {
194 self.entries[*slot.get()].2 = text;
195 }
196 std::collections::hash_map::Entry::Vacant(slot) => {
197 slot.insert(self.entries.len());
198 self.entries.push((row, col, text));
199 }
200 }
201 }
202
203 fn remove(&mut self, row: u32, col: u32) -> Option<String> {
204 let deferred_text = self.get(row, col);
205 self.invalidate_deferred_at(row, col);
206 let Some(idx) = self.index.remove(&(row, col)) else {
207 return deferred_text;
208 };
209 let (_, _, text) = self.entries.remove(idx);
210 for slot in self.index.values_mut() {
212 if *slot > idx {
213 *slot -= 1;
214 }
215 }
216 Some(text)
217 }
218
219 fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
220 self.index
221 .get(&(row, col))
222 .map(|&i| self.entries[i].2.as_str())
223 }
224
225 fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
226 let idx = self.index.remove(&(row, col))?;
227 let (_, _, text) = self.entries.remove(idx);
228 for slot in self.index.values_mut() {
229 if *slot > idx {
230 *slot -= 1;
231 }
232 }
233 Some(text)
234 }
235
236 fn get(&self, row: u32, col: u32) -> Option<String> {
237 if let Some(text) = self.get_ordinary(row, col) {
238 return Some(text.to_string());
239 }
240 let package = self.deferred_package.as_ref()?;
241 if package.suppressed.contains(&(row, col)) {
242 return None;
243 }
244 package
245 .replay
246 .lock()
247 .ok()?
248 .formula_at(row, col)
249 .ok()?
250 .map(|record| record.text)
251 }
252
253 fn len(&self) -> usize {
254 self.deferred_package
255 .as_ref()
256 .map_or(self.entries.len(), |package| {
257 usize::try_from(package.report.source_formula_records_spooled)
258 .unwrap_or(usize::MAX)
259 .saturating_sub(package.suppressed.len())
260 .saturating_add(self.entries.len())
261 })
262 }
263
264 fn is_empty(&self) -> bool {
265 self.entries.is_empty() && self.deferred_package.is_none()
266 }
267
268 fn into_parts(self) -> StagedSheetParts {
269 (self.entries, self.deferred_package)
270 }
271}
272
273type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
274
275#[derive(Clone, Debug, PartialEq, Eq, Hash)]
276struct PreparationRegion {
277 sheet: String,
278 sheet_id: SheetId,
279 start_row: u32,
280 start_col: u32,
281 end_row: u32,
282 end_col: u32,
283}
284
285struct OrderedTargetProducers {
288 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
289 seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
290}
291
292impl OrderedTargetProducers {
293 fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
294 let mut ordered = Vec::new();
295 ordered.try_reserve(capacity)?;
296 let mut seen = FxHashSet::default();
297 seen.try_reserve(capacity)?;
298 Ok(Self { ordered, seen })
299 }
300
301 fn from_ordered(
302 ordered: Vec<crate::engine::target_preparation::TargetProducer>,
303 ) -> Result<Self, std::collections::TryReserveError> {
304 let mut seen = FxHashSet::default();
305 seen.try_reserve(ordered.len())?;
306 seen.extend(ordered.iter().copied());
307 Ok(Self { ordered, seen })
308 }
309
310 fn push(
311 &mut self,
312 producer: crate::engine::target_preparation::TargetProducer,
313 ) -> Result<bool, std::collections::TryReserveError> {
314 #[cfg(test)]
315 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
316 if self.seen.contains(&producer) {
317 return Ok(false);
318 }
319 self.seen.try_reserve(1)?;
320 self.ordered.try_reserve(1)?;
321 self.seen.insert(producer);
322 self.ordered.push(producer);
323 Ok(true)
324 }
325
326 fn len(&self) -> usize {
327 self.ordered.len()
328 }
329
330 fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
331 self.ordered
332 }
333}
334
335#[cfg(test)]
336thread_local! {
337 static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
338}
339
340fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
341 crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
342 reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
343 limit: u64::MAX,
344 observed: observed as u64,
345 request_id,
346 })
347 .into_excel_error()
348}
349
350#[derive(Debug)]
351struct PreparedOrdinaryStagedFormula {
352 sheet: String,
353 sheet_id: SheetId,
354 lease: StagedFormulaLease,
355 ast_id: Option<AstNodeId>,
356 plan: Option<DependencyPlanRow>,
357}
358
359struct PreparedTargetSourcePackage {
360 sheet: String,
361 sheet_id: SheetId,
362 lease: StagedPackageLease,
363 source_report: crate::engine::FormulaCompressedSourceReport,
364 replay_records: Vec<crate::engine::DeferredReplayFormula>,
365 disposition: crate::engine::FormulaReplayDisposition,
366 placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
367 legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
368 direct_families: usize,
369 direct_cells: u64,
370 direct_fragments: u64,
371 direct_complete_families: u64,
372 direct_complete_cells: u64,
373 direct_partition_families: u64,
374 direct_partition_cells: u64,
375 anchor_parses: u64,
376 anchor_asts: u64,
377 anchor_analyses: u64,
378}
379
380impl PreparedTargetSourcePackage {
381 fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
382 self.replay_records.iter().filter(|record| {
383 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
384 return true;
385 };
386 let coord = crate::engine::SourceCoord { row, col };
387 let disposition = match record.family {
388 Some(family) => self.disposition.shared_disposition(family, coord),
389 None => self.disposition.ordinary_disposition(coord).0,
390 };
391 !matches!(
392 disposition,
393 crate::engine::FormulaReplayCoordinateDisposition::Direct
394 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
395 )
396 })
397 }
398}
399
400fn producer_dirty_to_span_dirty(
401 dirty: ProducerDirtyDomain,
402 span_ref: FormulaSpanRef,
403) -> DirtyDomain {
404 match dirty {
405 ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
406 ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
407 ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
408 }
409}
410type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
411type StagedFormulaBatches = Vec<(String, StagedSheet)>;
412#[derive(Debug)]
413pub(crate) struct SourceFamilyPreparationError {
414 reason: String,
415 parse_attempted: bool,
416 ast_created: bool,
417 analysis_created: bool,
418}
419
420impl SourceFamilyPreparationError {
421 fn contract(reason: &'static str) -> Self {
422 Self {
423 reason: reason.to_string(),
424 parse_attempted: false,
425 ast_created: false,
426 analysis_created: false,
427 }
428 }
429
430 fn parse(reason: &'static str) -> Self {
431 Self {
432 reason: reason.to_string(),
433 parse_attempted: true,
434 ast_created: false,
435 analysis_created: false,
436 }
437 }
438
439 fn ast(reason: impl Into<String>) -> Self {
440 Self {
441 reason: reason.into(),
442 parse_attempted: true,
443 ast_created: true,
444 analysis_created: false,
445 }
446 }
447
448 fn analysis(reason: impl Into<String>) -> Self {
449 Self {
450 reason: reason.into(),
451 parse_attempted: true,
452 ast_created: true,
453 analysis_created: true,
454 }
455 }
456}
457
458struct PreparedPartitionShadow {
459 fragment_count: u64,
460 direct_cells: u64,
461 fallback_cells: u64,
462 function_semantics_used: bool,
463}
464
465pub(crate) fn classify_mixed_topology_incomplete(
466 observed: &MixedTopologyCompileStats,
467) -> crate::engine::EvaluationIncompleteReason {
468 if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
469 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
470 } else if observed.memory_overflow_count > 0 {
471 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
472 } else if observed.edge_overflow_count > 0 {
473 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
474 } else if observed.candidate_overflow_count > 0 {
475 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
476 } else {
477 crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
478 }
479}
480
481#[derive(Clone, Debug, Default)]
482struct LegacyIslandPlan {
483 membership: Vec<VertexId>,
484 dirty_vertices: Vec<VertexId>,
485 sheet_ids: Vec<SheetId>,
486 island_id: u64,
487 retained_bytes: usize,
488 boundary_relationships: usize,
489 omitted_relationships: usize,
490 omitted_direct_relationships: usize,
491}
492
493impl LegacyIslandPlan {
494 fn is_empty(&self) -> bool {
495 self.membership.is_empty()
496 }
497}
498
499type FormulaPlaneMixedScheduleBuild = (
500 MixedSchedule,
501 BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
502 u64,
503 Vec<VertexId>,
504 Vec<usize>,
505 LegacyIslandPlan,
506);
507
508#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
509pub(crate) enum FormulaSpanDemotionFault {
510 #[default]
511 None,
512 AstPreparation,
513 LegacyGraphPreparation,
514 FinalLegacyGraphValidation,
515 FinalAuthorityValidation,
516 AllocationReservation,
517 BeforeFirstMutation,
518}
519
520#[derive(Debug)]
521pub(crate) enum FormulaSpanDemotionError {
522 StaleAuthority,
523 InvalidSpan,
524 CountOverflow,
525 Resource(ExcelError),
526 AstPreparation(ExcelError),
527 LegacyGraph(PreparedLegacyGraphError),
528 Injected(FormulaSpanDemotionFault),
529}
530
531impl std::fmt::Display for FormulaSpanDemotionError {
532 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
533 match self {
534 Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
535 Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
536 Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
537 Self::Resource(error) => write!(f, "{error}"),
538 Self::AstPreparation(error) => {
539 write!(f, "FormulaPlane demotion analysis failed: {error}")
540 }
541 Self::LegacyGraph(error) => {
542 write!(f, "FormulaPlane demotion graph plan failed: {error}")
543 }
544 Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
545 }
546 }
547}
548
549impl std::error::Error for FormulaSpanDemotionError {}
550
551pub(crate) struct PreparedFormulaSpanDemotion {
552 span_refs: Vec<FormulaSpanRef>,
553 expected_plane_epoch: u64,
554 expected_indexes_epoch: u64,
555 expected_indexed_plane_epoch: u64,
556 placement_count: usize,
557 legacy_graph: PreparedLegacyGraphPlan,
558 fault: FormulaSpanDemotionFault,
559}
560
561#[derive(Clone, Copy, Debug, PartialEq, Eq)]
562pub(crate) struct FormulaSpanDemotionReport {
563 pub(crate) spans_demoted: usize,
564 pub(crate) placements_materialized: usize,
565}
566
567type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
568type CompressedReplayBatch = (
569 FormulaIngestBatch,
570 crate::engine::FormulaCompressedSourceBatch,
571);
572type PreparedSourceBatch = (
573 FormulaIngestBatch,
574 crate::engine::FormulaCompressedSourceReport,
575 crate::engine::FormulaCompressedPreparation,
576);
577type PreparedStagedFormulaBatches = (
578 PreparedFormulaBatches,
579 Vec<CompressedReplayBatch>,
580 Vec<PreparedSourceBatch>,
581);
582
583#[doc(hidden)]
586pub struct SourceFormulaIngress<'a, R> {
587 engine: &'a mut Engine<R>,
588}
589
590impl<R> SourceFormulaIngress<'_, R>
591where
592 R: EvaluationContext,
593{
594 pub fn prepare_families(
595 &mut self,
596 sheet_name: &str,
597 families: &[crate::engine::SourceFormulaFamily],
598 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
599 self.engine.observe_function_semantic_epoch()?;
600 Ok(self
601 .engine
602 .prepare_source_formula_families(sheet_name, families))
603 }
604
605 pub fn prepare_eager_proposals(
606 &mut self,
607 sheet_name: &str,
608 families: &[crate::engine::SourceFormulaFamily],
609 partitions: &[crate::engine::PartitionedSourceFormulaFamily],
610 formula_record_count: u64,
611 replay: Box<dyn crate::engine::DeferredFormulaReplay>,
612 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
613 self.engine.observe_function_semantic_epoch()?;
614 let replay = Arc::new(std::sync::Mutex::new(replay));
615 self.engine.prepare_source_formula_proposals(
616 sheet_name,
617 families,
618 partitions,
619 partitions,
620 formula_record_count,
621 replay,
622 &BTreeSet::new(),
623 )
624 }
625
626 pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
627 self.engine.stage_deferred_formula_package(package);
628 }
629
630 pub fn ingest_replay_batches(
631 &mut self,
632 batches: Vec<(
633 FormulaIngestBatch,
634 crate::engine::FormulaCompressedSourceBatch,
635 )>,
636 ) -> Result<FormulaIngestReport, ExcelError> {
637 self.engine
638 .ingest_compressed_formula_source_batches(batches)
639 }
640
641 pub fn finish_prepared(
642 &mut self,
643 batches: Vec<(
644 FormulaIngestBatch,
645 crate::engine::FormulaCompressedSourceReport,
646 crate::engine::FormulaCompressedPreparation,
647 )>,
648 ) -> Result<FormulaIngestReport, ExcelError> {
649 self.engine.finish_compressed_formula_sources(batches)
650 }
651}
652
653const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
657
658#[derive(Debug, Clone, PartialEq)]
659pub(crate) enum ComputedWrite {
660 Cell {
661 seq: u64,
662 sheet_id: SheetId,
663 row0: u32,
664 col0: u32,
665 value: OverlayValue,
666 format_id: Option<crate::format::FormatId>,
667 },
668 Rect {
669 seq: u64,
670 sheet_id: SheetId,
671 sr0: u32,
672 sc0: u32,
673 values: Vec<Vec<OverlayValue>>,
674 },
675}
676
677impl ComputedWrite {
678 #[inline]
679 pub(crate) fn seq(&self) -> u64 {
680 match self {
681 ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
682 }
683 }
684}
685
686#[derive(Debug, Default)]
687pub(crate) struct ComputedWriteBuffer {
688 writes: Vec<ComputedWrite>,
689 next_seq: u64,
690 estimated_bytes: usize,
691 formats_present: bool,
692}
693
694impl ComputedWriteBuffer {
695 const ENTRY_BASE_BYTES: usize = 32;
696
697 #[inline]
698 pub(crate) fn is_empty(&self) -> bool {
699 self.writes.is_empty()
700 }
701
702 #[inline]
703 pub(crate) fn len(&self) -> usize {
704 self.writes.len()
705 }
706
707 #[inline]
708 pub(crate) fn estimated_bytes(&self) -> usize {
709 self.estimated_bytes
710 }
711
712 #[inline]
713 pub(crate) fn writes(&self) -> &[ComputedWrite] {
714 &self.writes
715 }
716
717 pub(crate) fn push_cell(
718 &mut self,
719 sheet_id: SheetId,
720 row0: u32,
721 col0: u32,
722 value: OverlayValue,
723 ) {
724 self.push_cell_with_format(sheet_id, row0, col0, value, None);
725 }
726
727 pub(crate) fn push_cell_with_format(
728 &mut self,
729 sheet_id: SheetId,
730 row0: u32,
731 col0: u32,
732 value: OverlayValue,
733 format_id: Option<crate::format::FormatId>,
734 ) {
735 let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
736 self.formats_present |= format_id.is_some();
737 let seq = self.next_sequence();
738 self.estimated_bytes = self
739 .estimated_bytes
740 .saturating_add(Self::estimate_value_bytes(&value));
741 self.writes.push(ComputedWrite::Cell {
742 seq,
743 sheet_id,
744 row0,
745 col0,
746 value,
747 format_id,
748 });
749 }
750
751 pub(crate) fn push_rect(
752 &mut self,
753 sheet_id: SheetId,
754 sr0: u32,
755 sc0: u32,
756 values: Vec<Vec<OverlayValue>>,
757 ) {
758 let seq = self.next_sequence();
759 let added = values
760 .iter()
761 .flat_map(|row| row.iter())
762 .map(Self::estimate_value_bytes)
763 .fold(0usize, usize::saturating_add);
764 self.estimated_bytes = self.estimated_bytes.saturating_add(added);
765 self.writes.push(ComputedWrite::Rect {
766 seq,
767 sheet_id,
768 sr0,
769 sc0,
770 values,
771 });
772 }
773
774 pub(crate) fn clear(&mut self) {
775 self.writes.clear();
776 self.estimated_bytes = 0;
777 self.formats_present = false;
778 }
779
780 fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
781 self.estimated_bytes = 0;
782 let formats_present = std::mem::take(&mut self.formats_present);
783 (std::mem::take(&mut self.writes), formats_present)
784 }
785
786 fn next_sequence(&mut self) -> u64 {
787 let seq = self.next_seq;
788 self.next_seq = self.next_seq.wrapping_add(1);
789 seq
790 }
791
792 #[inline]
793 fn estimate_value_bytes(value: &OverlayValue) -> usize {
794 Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
795 }
796}
797
798#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
799struct ComputedWriteChunkKey {
800 sheet_id: SheetId,
801 col0: u32,
802 chunk_idx: usize,
803 chunk_start_row0: u32,
804}
805
806#[derive(Debug, Clone, PartialEq)]
807pub(crate) struct ComputedWriteChunkEntryPlan {
808 pub(crate) row_in_chunk: usize,
809 pub(crate) seq: u64,
810 pub(crate) value: OverlayValue,
811 pub(crate) format_id: Option<crate::format::FormatId>,
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteChunkPlanShape {
816 Point,
817 SparseOffsets {
818 entries: usize,
819 span_len: usize,
820 },
821 DenseRange {
822 start: usize,
823 len: usize,
824 },
825 RunRange {
826 start: usize,
827 len: usize,
828 runs: usize,
829 },
830}
831
832#[derive(Debug, Clone, PartialEq, Eq)]
833pub(crate) enum ComputedWriteFormatClear {
834 Range { start: usize, end: usize },
835 Offsets(Vec<usize>),
836}
837
838#[derive(Debug, Clone, PartialEq)]
839pub(crate) enum ComputedWriteChunkFormatEffect {
840 NoFormatWork,
841 ClearStale(ComputedWriteFormatClear),
842 SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
843}
844
845struct DerivedFormatPlaneSyncPlan {
846 plane_epoch: u64,
847 active_regions: Vec<crate::formula_plane::region_index::Region>,
848 overlay_regions: Vec<crate::formula_plane::region_index::Region>,
849}
850
851#[derive(Debug, Clone, PartialEq)]
852pub(crate) struct ComputedWriteChunkPlan {
853 pub(crate) sheet_id: SheetId,
854 pub(crate) col0: u32,
855 pub(crate) chunk_idx: usize,
856 pub(crate) chunk_start_row0: u32,
857 pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
858 pub(crate) shape: ComputedWriteChunkPlanShape,
859 pub(crate) format_effect: ComputedWriteChunkFormatEffect,
860}
861
862#[derive(Debug, Clone, Default, PartialEq)]
863pub(crate) struct ComputedWriteCoalescingPlan {
864 pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
865 pub(crate) input_cells: usize,
866 pub(crate) coalesced_cells: usize,
867 pub(crate) overwritten_cells: usize,
868}
869
870impl ComputedWriteCoalescingPlan {
871 #[inline]
872 pub(crate) fn is_empty(&self) -> bool {
873 self.chunks.is_empty()
874 }
875}
876
877impl ComputedWriteChunkPlan {
878 fn from_group(
879 key: ComputedWriteChunkKey,
880 mut entries: Vec<ComputedWriteChunkEntryPlan>,
881 formats_present: bool,
882 computed_lane_has_formats: bool,
883 ) -> (Self, usize) {
884 entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
885 let input_len = entries.len();
886 let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
887 for entry in entries {
888 if let Some(prev) = coalesced.last_mut()
889 && prev.row_in_chunk == entry.row_in_chunk
890 {
891 *prev = entry;
892 continue;
893 }
894 coalesced.push(entry);
895 }
896 let overwritten = input_len.saturating_sub(coalesced.len());
897 let shape = Self::classify_shape(&coalesced);
898 let format_effect = Self::classify_format_effect(
899 &coalesced,
900 &shape,
901 formats_present,
902 computed_lane_has_formats,
903 );
904 (
905 Self {
906 sheet_id: key.sheet_id,
907 col0: key.col0,
908 chunk_idx: key.chunk_idx,
909 chunk_start_row0: key.chunk_start_row0,
910 entries: coalesced,
911 shape,
912 format_effect,
913 },
914 overwritten,
915 )
916 }
917
918 fn classify_format_effect(
919 entries: &[ComputedWriteChunkEntryPlan],
920 shape: &ComputedWriteChunkPlanShape,
921 formats_present: bool,
922 computed_lane_has_formats: bool,
923 ) -> ComputedWriteChunkFormatEffect {
924 if !formats_present {
925 return if computed_lane_has_formats {
926 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
927 } else {
928 ComputedWriteChunkFormatEffect::NoFormatWork
929 };
930 }
931
932 if entries.iter().all(|entry| entry.format_id.is_none()) {
933 return if computed_lane_has_formats {
934 ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
935 } else {
936 ComputedWriteChunkFormatEffect::NoFormatWork
937 };
938 }
939
940 ComputedWriteChunkFormatEffect::SetExplicit(
941 entries
942 .iter()
943 .map(|entry| (entry.row_in_chunk, entry.format_id))
944 .collect(),
945 )
946 }
947
948 fn format_clear_spec(
949 entries: &[ComputedWriteChunkEntryPlan],
950 shape: &ComputedWriteChunkPlanShape,
951 ) -> ComputedWriteFormatClear {
952 match shape {
953 ComputedWriteChunkPlanShape::DenseRange { start, len }
954 | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
955 ComputedWriteFormatClear::Range {
956 start: *start,
957 end: start.saturating_add(*len),
958 }
959 }
960 ComputedWriteChunkPlanShape::Point
961 | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
962 ComputedWriteFormatClear::Offsets(
963 entries.iter().map(|entry| entry.row_in_chunk).collect(),
964 )
965 }
966 }
967 }
968
969 fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
970 debug_assert!(!entries.is_empty());
971 if entries.len() == 1 {
972 return ComputedWriteChunkPlanShape::Point;
973 }
974
975 let start = entries[0].row_in_chunk;
976 let end = entries[entries.len() - 1].row_in_chunk;
977 let span_len = end.saturating_sub(start).saturating_add(1);
978 if span_len != entries.len() {
979 return ComputedWriteChunkPlanShape::SparseOffsets {
980 entries: entries.len(),
981 span_len,
982 };
983 }
984
985 let runs = Self::run_count(entries);
986 if runs < entries.len() {
987 ComputedWriteChunkPlanShape::RunRange {
988 start,
989 len: entries.len(),
990 runs,
991 }
992 } else {
993 ComputedWriteChunkPlanShape::DenseRange {
994 start,
995 len: entries.len(),
996 }
997 }
998 }
999
1000 fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
1001 let mut runs = 0usize;
1002 let mut prev: Option<&OverlayValue> = None;
1003 for entry in entries {
1004 if prev != Some(&entry.value) {
1005 runs = runs.saturating_add(1);
1006 prev = Some(&entry.value);
1007 }
1008 }
1009 runs
1010 }
1011}
1012
1013pub struct Engine<R> {
1014 pub(crate) graph: DependencyGraph,
1015 resolver: R,
1016 pub config: EvalConfig,
1017 workbook_load_limits: crate::engine::WorkbookLoadLimits,
1018 clock: crate::timezone::SnapshotClock,
1023 thread_pool: Option<Arc<rayon::ThreadPool>>,
1024 pub recalc_epoch: u64,
1025 snapshot_id: std::sync::atomic::AtomicU64,
1026 topology_epoch: u64,
1027 legacy_island_structural_summaries_trusted: bool,
1030 cached_static_schedule: Option<CachedScheduleEntry>,
1031 #[cfg(any(test, feature = "benchmark_internal"))]
1032 recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1033 cached_mixed_topology: Option<CachedMixedTopology>,
1034 mixed_topology_cache_builds: u64,
1035 mixed_topology_cache_hits: u64,
1036 mixed_topology_cache_overflows: u64,
1037 mixed_topology_cache_skip_streak: u64,
1038 spill_mgr: ShimSpillManager,
1039 arrow_sheets: SheetStore,
1041 format_registry: crate::format::FormatRegistry,
1043 derived_format_results: std::sync::RwLock<FxHashMap<VertexId, Option<crate::format::FormatId>>>,
1045 derived_formats: std::sync::RwLock<FxHashMap<CellRef, crate::format::FormatId>>,
1047 derived_formats_plane_epoch: u64,
1050 #[cfg(test)]
1051 derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1052 #[cfg(test)]
1053 computed_overlay_set_explicit_entry_operations_for_test: u64,
1054 #[cfg(test)]
1055 computed_overlay_stale_clear_range_effects_for_test: u64,
1056 #[cfg(test)]
1057 computed_overlay_stale_clear_offset_attempts_for_test: u64,
1058 #[cfg(test)]
1059 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1060 has_edited: bool,
1062 overlay_compactions: u64,
1064
1065 computed_overlay_bytes_estimate: usize,
1067 computed_overlay_mirroring_disabled: bool,
1068 pub(crate) force_materialize_range_views: bool,
1071 row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1073 used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1075 lookup_index_cache: LookupIndexCache,
1076 source_cache: Arc<std::sync::RwLock<SourceCache>>,
1077 source_formula_token: Arc<()>,
1079 recalc_plan_token: Arc<()>,
1081 staged_formulas: StagedFormulaMap,
1083 staged_formula_index: StagedFormulaIndex,
1085 row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1087 row_visibility_mask_cache: std::sync::RwLock<
1089 FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1090 >,
1091 formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1093 last_formula_ingest_report: Option<FormulaIngestReport>,
1095 formula_ingest_report_total: FormulaIngestReport,
1097 formula_plane_cycle_member_span_demotions: u64,
1107 formula_plane_array_result_span_demotions: u64,
1112 formula_plane_capacity_bailouts: u64,
1117 formula_plane_structural_span_candidates: u64,
1119 active_cancel_flag: Option<crate::engine::CancelToken>,
1121 active_evaluation_deadline: Option<Instant>,
1123
1124 action_depth: u32,
1129
1130 last_virtual_dep_telemetry: VirtualDepTelemetry,
1132 virtual_dep_fallback_activations: u64,
1133
1134 last_cycle_telemetry: CycleTelemetry,
1136
1137 next_evaluation_resource_request_id: u64,
1139 evaluation_resource_request_depth: usize,
1140 active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1141 last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1142 evaluation_resource_baseline: EvaluationResourceBaselineStats,
1143 evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1144 evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1145 evaluation_resource_config_diagnostic:
1146 Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1147 active_resource_ledger: Option<ResourceLedger>,
1148
1149 pending_iterative_redirty: Vec<VertexId>,
1168 retained_scc_members: FxHashMap<VertexId, u64>,
1186 next_retained_scc_id: u64,
1187 retained_scc_config_fingerprint: u64,
1194 retained_scc_function_epoch_seen: u64,
1201 retained_scc_provider_revision_seen: Option<u64>,
1202 retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1209
1210 iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1223
1224 function_semantic_epoch_seen: u64,
1227 function_provider_revision_seen: Option<u64>,
1230
1231 #[cfg(test)]
1232 last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1233 #[cfg(test)]
1234 evaluation_request_begin_count_for_test: u64,
1235 #[cfg(any(test, feature = "test-support"))]
1236 before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1237 #[cfg(test)]
1238 before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1239 #[cfg(test)]
1240 before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1241 #[cfg(test)]
1242 inject_target_semantic_stale_once_for_test: bool,
1243 #[cfg(test)]
1244 force_virtual_dep_changes_remaining_for_test: usize,
1245 #[cfg(test)]
1246 fail_evaluation_commit_preflight_once_for_test: bool,
1247 #[cfg(test)]
1248 cancel_before_formula_plane_layer_commit_once_for_test: bool,
1249 #[cfg(test)]
1250 force_source_family_fallback: bool,
1251 #[cfg(test)]
1252 rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1253 #[cfg(test)]
1254 fragmented_commit_fault_for_test:
1255 Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1256 #[cfg(test)]
1257 formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1258 #[cfg(test)]
1259 target_preparation_fault_for_test:
1260 Option<crate::engine::target_preparation::TargetPreparationFault>,
1261 #[cfg(test)]
1262 force_non_cycle_schedule_fallback_for_test: bool,
1263 #[cfg(test)]
1264 mixed_topology_index_builds_for_test: u64,
1265 #[cfg(test)]
1266 before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1267 #[cfg(test)]
1268 after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1269}
1270
1271fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1274 fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1275 let node_type = match &ast.node_type {
1276 ASTNodeType::Literal(_) if !in_array => {
1277 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1278 *next = next.saturating_add(1);
1279 ASTNodeType::Literal(value)
1280 }
1281 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1282 ASTNodeType::Omitted => ASTNodeType::Omitted,
1283 ASTNodeType::Reference {
1284 original,
1285 reference,
1286 } => ASTNodeType::Reference {
1287 original: original.clone(),
1288 reference: reference.clone(),
1289 },
1290 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1291 op: op.clone(),
1292 expr: Box::new(visit(expr, binding, next, in_array)),
1293 },
1294 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1295 op: op.clone(),
1296 left: Box::new(visit(left, binding, next, in_array)),
1297 right: Box::new(visit(right, binding, next, in_array)),
1298 },
1299 ASTNodeType::Function { name, args } => ASTNodeType::Function {
1300 name: name.clone(),
1301 args: args
1302 .iter()
1303 .map(|arg| visit(arg, binding, next, in_array))
1304 .collect(),
1305 },
1306 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1307 callee: Box::new(visit(callee, binding, next, in_array)),
1308 args: args
1309 .iter()
1310 .map(|arg| visit(arg, binding, next, in_array))
1311 .collect(),
1312 },
1313 ASTNodeType::Array(rows) => ASTNodeType::Array(
1314 rows.iter()
1315 .map(|row| {
1316 row.iter()
1317 .map(|cell| visit(cell, binding, next, true))
1318 .collect()
1319 })
1320 .collect(),
1321 ),
1322 };
1323 ASTNode::new(node_type, ast.source_token.clone())
1324 }
1325
1326 let mut next = 0;
1327 visit(ast, binding, &mut next, false)
1328}
1329
1330impl<R: EvaluationContext> Engine<R> {
1333 pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1334 self.graph.ingest_pipeline(&self.resolver)
1335 }
1336}
1337
1338pub struct EngineAction<'a, R>
1339where
1340 R: EvaluationContext,
1341{
1342 engine: &'a mut Engine<R>,
1343 name: String,
1344 capture: Option<*mut MutationCapture>,
1347 arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1350 atomic_policy: bool,
1352}
1353
1354impl<'a, R> EngineAction<'a, R>
1355where
1356 R: EvaluationContext,
1357{
1358 #[inline]
1359 fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1360 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1361 let coord = crate::reference::Coord::from_excel(row, col, true, true);
1362 crate::reference::CellRef::new(sheet_id, coord)
1363 }
1364
1365 #[inline]
1366 pub fn name(&self) -> &str {
1367 &self.name
1368 }
1369
1370 #[inline]
1371 pub fn set_cell_value(
1372 &mut self,
1373 sheet: &str,
1374 row: u32,
1375 col: u32,
1376 value: LiteralValue,
1377 ) -> Result<(), crate::engine::EditorError> {
1378 if self.capture.is_some() {
1379 let old_value = self.engine.read_cell_value(sheet, row, col);
1380 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1381 let addr = self.addr_for(sheet, row, col);
1382 let Some(capture_ptr) = self.capture else {
1383 return Err(crate::engine::EditorError::TransactionFailed {
1384 reason: "action_with_logger: missing mutation capture".to_string(),
1385 });
1386 };
1387
1388 let old_comp = if self.arrow_undo.is_some() {
1391 self.engine.read_computed_overlay_cell(sheet, row, col)
1392 } else {
1393 None
1394 };
1395
1396 if self.engine.graph_admission_enabled() {
1397 let admission =
1398 self.engine
1399 .graph
1400 .preview_value_mutation(addr.sheet_id, row, col)?;
1401 self.engine.preflight_graph_admission(admission)?;
1402 }
1403 self.engine.demote_span_containing_cell_for_write(
1404 addr.sheet_id,
1405 addr.coord.row(),
1406 addr.coord.col(),
1407 )?;
1408 if old_formula.is_none() {
1409 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1410 }
1411
1412 let delta_old_sem = if old_formula.is_some() {
1413 None
1414 } else {
1415 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1416 };
1417
1418 let start_len = unsafe { (&*capture_ptr).len() };
1419
1420 let capture = unsafe { &mut *capture_ptr };
1422 self.engine.edit_with_capture(capture, |editor| {
1423 editor.set_cell_value_with_old_state(
1424 addr,
1425 value.clone(),
1426 old_value.clone(),
1427 old_formula.clone(),
1428 );
1429 })?;
1430 self.engine
1431 .record_formula_plane_structural_change(StructuralScope::Cell {
1432 sheet: addr.sheet_id,
1433 row: addr.coord.row(),
1434 col: addr.coord.col(),
1435 });
1436
1437 if let Some(undo_ptr) = self.arrow_undo {
1438 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1440 let undo = unsafe { &mut *undo_ptr };
1441 self.engine
1442 .record_spill_ops_into_arrow_undo(undo, new_events);
1443
1444 let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1446 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1447 let row0 = row.saturating_sub(1);
1448 let col0 = col.saturating_sub(1);
1449 let delta_new_sem = Some(value.clone());
1450 undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1451 undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1452 }
1453 Ok(())
1454 } else {
1455 self.engine
1456 .set_cell_value(sheet, row, col, value)
1457 .map_err(crate::engine::EditorError::from)
1458 }
1459 }
1460
1461 #[inline]
1462 pub fn set_cell_formula(
1463 &mut self,
1464 sheet: &str,
1465 row: u32,
1466 col: u32,
1467 ast: ASTNode,
1468 ) -> Result<(), crate::engine::EditorError> {
1469 if self.capture.is_some() {
1470 let old_value = self.engine.read_cell_value(sheet, row, col);
1471 let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1472 let addr = self.addr_for(sheet, row, col);
1473 let Some(capture_ptr) = self.capture else {
1474 return Err(crate::engine::EditorError::TransactionFailed {
1475 reason: "action_with_logger: missing mutation capture".to_string(),
1476 });
1477 };
1478
1479 let admitted_formula = if self.engine.graph_admission_enabled() {
1480 let placement =
1481 CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1482 let ingested = self.engine.ingest_pipeline().ingest_formula(
1483 FormulaAstInput::Tree(ast.clone()),
1484 placement,
1485 None,
1486 )?;
1487 let admission = self.engine.graph.preview_formula_mutations(&[(
1488 addr.sheet_id,
1489 row,
1490 col,
1491 ingested.dep_plan.clone(),
1492 )])?;
1493 self.engine.preflight_graph_admission(admission)?;
1494 Some((ingested.ast_id, ingested.dep_plan))
1495 } else {
1496 None
1497 };
1498 self.engine.demote_span_containing_cell_for_write(
1499 addr.sheet_id,
1500 addr.coord.row(),
1501 addr.coord.col(),
1502 )?;
1503 if old_formula.is_none() {
1504 old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1505 }
1506 let delta_old = if self.arrow_undo.is_some() {
1507 if old_formula.is_some() {
1508 None
1509 } else {
1510 Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1511 }
1512 } else {
1513 None
1514 };
1515 let start_len = unsafe { (&*capture_ptr).len() };
1516
1517 let capture = unsafe { &mut *capture_ptr };
1519 self.engine.edit_with_capture(capture, |editor| {
1520 if let Some((ast_id, plan)) = admitted_formula {
1521 editor.set_cell_formula_with_prepared_plan(
1522 addr,
1523 ast.clone(),
1524 old_value,
1525 old_formula,
1526 ast_id,
1527 plan,
1528 );
1529 } else {
1530 editor.set_cell_formula_with_old_state(
1531 addr,
1532 ast.clone(),
1533 old_value,
1534 old_formula,
1535 );
1536 }
1537 })?;
1538 self.engine
1539 .record_formula_plane_structural_change(StructuralScope::Cell {
1540 sheet: addr.sheet_id,
1541 row: addr.coord.row(),
1542 col: addr.coord.col(),
1543 });
1544
1545 if let Some(undo_ptr) = self.arrow_undo {
1546 let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1547 let undo = unsafe { &mut *undo_ptr };
1548 self.engine
1549 .record_spill_ops_into_arrow_undo(undo, new_events);
1550 let delta_new: Option<LiteralValue> = None;
1551 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1552 let row0 = row.saturating_sub(1);
1553 let col0 = col.saturating_sub(1);
1554 undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1555 }
1556 Ok(())
1557 } else {
1558 self.engine
1559 .set_cell_formula(sheet, row, col, ast)
1560 .map_err(crate::engine::EditorError::from)
1561 }
1562 }
1563
1564 #[inline]
1565 pub fn set_row_hidden(
1566 &mut self,
1567 sheet: &str,
1568 row_1based: u32,
1569 hidden: bool,
1570 source: RowVisibilitySource,
1571 ) -> Result<(), crate::engine::EditorError> {
1572 if self.capture.is_some() {
1573 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1574 let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1575 let old_hidden = self
1576 .engine
1577 .row_visibility
1578 .get(&sheet_id)
1579 .map(|state| state.is_row_hidden(row0, Some(source)))
1580 .unwrap_or(false);
1581 if old_hidden == hidden {
1582 return Ok(());
1583 }
1584
1585 let _ = self
1586 .engine
1587 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1588
1589 let Some(capture_ptr) = self.capture else {
1590 return Err(crate::engine::EditorError::TransactionFailed {
1591 reason: "action_with_logger: missing mutation capture".to_string(),
1592 });
1593 };
1594 unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1595 sheet_id,
1596 row0,
1597 source,
1598 old_hidden,
1599 new_hidden: hidden,
1600 });
1601
1602 Ok(())
1603 } else {
1604 self.engine
1605 .set_row_hidden(sheet, row_1based, hidden, source)
1606 }
1607 }
1608
1609 #[inline]
1610 pub fn set_rows_hidden(
1611 &mut self,
1612 sheet: &str,
1613 start_row_1based: u32,
1614 end_row_1based: u32,
1615 hidden: bool,
1616 source: RowVisibilitySource,
1617 ) -> Result<(), crate::engine::EditorError> {
1618 if self.capture.is_some() {
1619 let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1620 let (start_row0, end_row0) =
1621 Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1622
1623 let Some(capture_ptr) = self.capture else {
1624 return Err(crate::engine::EditorError::TransactionFailed {
1625 reason: "action_with_logger: missing mutation capture".to_string(),
1626 });
1627 };
1628 let capture = unsafe { &mut *capture_ptr };
1629
1630 for row0 in start_row0..=end_row0 {
1631 let old_hidden = self
1632 .engine
1633 .row_visibility
1634 .get(&sheet_id)
1635 .map(|state| state.is_row_hidden(row0, Some(source)))
1636 .unwrap_or(false);
1637 if old_hidden == hidden {
1638 continue;
1639 }
1640
1641 let _ = self
1642 .engine
1643 .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1644
1645 capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1646 sheet_id,
1647 row0,
1648 source,
1649 old_hidden,
1650 new_hidden: hidden,
1651 });
1652 }
1653
1654 Ok(())
1655 } else {
1656 self.engine
1657 .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1658 }
1659 }
1660
1661 #[inline]
1662 pub fn insert_rows(
1663 &mut self,
1664 sheet: &str,
1665 before: u32,
1666 count: u32,
1667 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1668 if count == 0 {
1669 return Ok(crate::engine::ShiftSummary::default());
1670 }
1671 if self.capture.is_some() {
1672 let Some(capture_ptr) = self.capture else {
1673 return Err(crate::engine::EditorError::TransactionFailed {
1674 reason: "action_atomic: missing mutation capture".to_string(),
1675 });
1676 };
1677
1678 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1679 let before0 = before.saturating_sub(1);
1680 let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1681 let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1682 self.engine
1686 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1687
1688 let summary = {
1690 let capture = unsafe { &mut *capture_ptr };
1691 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1692 Ok(crate::engine::ShiftSummary::default());
1693 self.engine.edit_with_capture(capture, |editor| {
1694 editor.set_structural_occupancy(occupancy);
1695 out = editor.insert_rows(sheet_id, before0, count);
1696 })?;
1697 out?
1698 };
1699
1700 self.engine.ensure_arrow_sheet(sheet);
1702 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1703 asheet.insert_rows(before0 as usize, count as usize);
1704 }
1705 self.engine
1706 .purge_derived_formats_after_row(sheet_id, before0);
1707 self.engine
1708 .shift_row_visibility_insert(sheet_id, before0, count);
1709 self.engine.mark_moved_formula_vertices_dirty(&summary);
1710 self.engine
1711 .clear_computed_overlay_after_row(sheet, before0 as usize);
1712 self.engine
1713 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1714 if let Some(undo_ptr) = self.arrow_undo {
1715 unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1716 }
1717 Ok(summary)
1718 } else {
1719 self.engine.insert_rows(sheet, before, count)
1720 }
1721 }
1722
1723 #[inline]
1724 pub fn delete_rows(
1725 &mut self,
1726 sheet: &str,
1727 start: u32,
1728 count: u32,
1729 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1730 if count == 0 {
1731 return Ok(crate::engine::ShiftSummary::default());
1732 }
1733 if self.atomic_policy {
1734 return Err(crate::engine::EditorError::TransactionUnsupported {
1735 reason:
1736 "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1737 .to_string(),
1738 });
1739 }
1740 self.engine.delete_rows(sheet, start, count)
1741 }
1742
1743 #[inline]
1744 pub fn insert_columns(
1745 &mut self,
1746 sheet: &str,
1747 before: u32,
1748 count: u32,
1749 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1750 if count == 0 {
1751 return Ok(crate::engine::ShiftSummary::default());
1752 }
1753 if self.capture.is_some() {
1754 let Some(capture_ptr) = self.capture else {
1755 return Err(crate::engine::EditorError::TransactionFailed {
1756 reason: "action_atomic: missing mutation capture".to_string(),
1757 });
1758 };
1759
1760 let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1761 let before0 = before.saturating_sub(1);
1762 let occupancy = self.engine.structural_column_occupancy();
1763 let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1764 self.engine
1765 .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1766
1767 let summary = {
1768 let capture = unsafe { &mut *capture_ptr };
1769 let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1770 Ok(crate::engine::ShiftSummary::default());
1771 self.engine.edit_with_capture(capture, |editor| {
1772 editor.set_structural_occupancy(occupancy);
1773 out = editor.insert_columns(sheet_id, before0, count);
1774 })?;
1775 out?
1776 };
1777
1778 self.engine.ensure_arrow_sheet(sheet);
1779 if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1780 asheet.insert_columns(before0 as usize, count as usize);
1781 }
1782 self.engine
1783 .purge_derived_formats_after_col(sheet_id, before0);
1784 self.engine.mark_moved_formula_vertices_dirty(&summary);
1785 self.engine
1786 .clear_computed_overlay_after_col(sheet, before0 as usize);
1787 self.engine
1788 .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1789 if let Some(undo_ptr) = self.arrow_undo {
1790 unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1791 }
1792 Ok(summary)
1793 } else {
1794 self.engine.insert_columns(sheet, before, count)
1795 }
1796 }
1797
1798 #[inline]
1799 pub fn delete_columns(
1800 &mut self,
1801 sheet: &str,
1802 start: u32,
1803 count: u32,
1804 ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1805 if count == 0 {
1806 return Ok(crate::engine::ShiftSummary::default());
1807 }
1808 if self.atomic_policy {
1809 return Err(crate::engine::EditorError::TransactionUnsupported {
1810 reason:
1811 "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1812 .to_string(),
1813 });
1814 }
1815 self.engine.delete_columns(sheet, start, count)
1816 }
1817
1818 #[inline]
1823 pub fn action<T>(
1824 &mut self,
1825 name: impl AsRef<str>,
1826 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1827 ) -> Result<T, crate::engine::EditorError> {
1828 self.engine.action(name, f)
1829 }
1830}
1831
1832struct ActionDepthGuard<'a, R> {
1833 engine: *mut Engine<R>,
1834 _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1835}
1836
1837impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1838 fn drop(&mut self) {
1839 unsafe {
1842 let e = &mut *self.engine;
1843 e.action_depth = e.action_depth.saturating_sub(1);
1844 }
1845 }
1846}
1847
1848#[derive(Default)]
1849struct SourceCache {
1850 scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1851 tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1852}
1853
1854#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1855struct VisibilityMaskCacheKey {
1856 sheet_id: SheetId,
1857 start_row0: u32,
1858 end_row0: u32,
1859 mode: VisibilityMaskMode,
1860 version: u64,
1861}
1862
1863#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1864enum StructuralScope {
1865 Cell { sheet: SheetId, row: u32, col: u32 },
1866 Region(Region),
1867 Sheet(SheetId),
1868 RemovedSheet(SheetId),
1869 OpaqueGlobal,
1870 AllSheets,
1871}
1872
1873#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1874enum LoggedEditImpact {
1875 NoOp,
1876 DataOnly,
1877 Topology,
1878}
1879
1880#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1881enum LoggedEditDirection {
1882 Original,
1883 InverseReplay,
1884 ForwardReplay,
1885}
1886
1887#[derive(Clone, Copy)]
1888struct InvalidationBaseline {
1889 snapshot_id: u64,
1890 topology_epoch: u64,
1891}
1892
1893struct SourceCacheSession {
1894 cache: Arc<std::sync::RwLock<SourceCache>>,
1895}
1896
1897impl Drop for SourceCacheSession {
1898 fn drop(&mut self) {
1899 if let Ok(mut g) = self.cache.write() {
1900 *g = SourceCache::default();
1901 }
1902 }
1903}
1904
1905#[derive(Debug)]
1906#[non_exhaustive]
1907pub struct EvalResult {
1908 pub computed_vertices: usize,
1909 pub cycle_errors: usize,
1910 pub elapsed: std::time::Duration,
1911}
1912
1913#[derive(Clone, Debug, PartialEq, Eq)]
1914#[non_exhaustive]
1915pub struct TableMetadata {
1916 pub name: String,
1917 pub sheet: String,
1918 pub start_row: u32,
1919 pub start_col: u32,
1920 pub end_row: u32,
1921 pub end_col: u32,
1922 pub header_row: bool,
1923 pub headers: Vec<String>,
1924 pub totals_row: bool,
1925}
1926
1927#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1932#[non_exhaustive]
1933pub struct EngineBaselineStats {
1934 pub graph_vertex_count: usize,
1935 pub graph_formula_vertex_count: usize,
1936 pub graph_edge_count: usize,
1937 pub dirty_vertex_count: usize,
1938 pub evaluation_vertex_count: usize,
1939 pub formula_ast_root_count: usize,
1940 pub formula_ast_node_count: usize,
1941 pub staged_formula_count: usize,
1942 pub formula_plane_active_span_count: usize,
1943 pub formula_plane_producer_result_entries: usize,
1944 pub formula_plane_consumer_read_entries: usize,
1945 pub formula_plane_mixed_topology_cache_builds: u64,
1946 pub formula_plane_mixed_topology_cache_hits: u64,
1947 pub formula_plane_mixed_topology_cache_overflows: u64,
1948 pub formula_plane_dirty_pending_events: usize,
1949 pub formula_plane_dirty_region_events_recorded: u64,
1950 pub formula_plane_dirty_span_region_events_recorded: u64,
1951 pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1952 pub formula_plane_dirty_global_invalidations: u64,
1953 pub formula_plane_structural_span_candidates: u64,
1954 pub formula_plane_cycle_member_span_demotions: u64,
1957 pub formula_plane_array_result_span_demotions: u64,
1961 pub retained_scc_members: usize,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1967#[non_exhaustive]
1968pub struct VirtualDepTelemetry {
1969 pub candidate_vertices_total: usize,
1970 pub vdeps_vertices_total: usize,
1971 pub vdeps_edges_total: usize,
1972 pub builder_elapsed_ms_total: u128,
1973 pub schedule_virtual_passes: usize,
1974 pub schedule_static_passes: usize,
1975 pub schedule_cache_hits: usize,
1976 pub schedule_cache_misses: usize,
1977 pub reused_schedule_vertices_total: usize,
1978 pub replan_iterations: usize,
1979 pub changed_vdeps_total: usize,
1980 pub bailout_reason: Option<&'static str>,
1981 pub fallback_mode_activations: u64,
1982}
1983
1984#[derive(Debug, Clone, Default, PartialEq)]
1993#[non_exhaustive]
1994pub struct CycleTelemetry {
1995 pub static_sccs: usize,
1997 pub phantom_sccs: usize,
1999 pub live_cycles_witnessed: usize,
2001 pub circ_cells_stamped: usize,
2003 pub settle_passes_total: usize,
2006 pub max_passes_single_scc: usize,
2008 pub iterated_sccs: usize,
2011 pub converged_sccs: usize,
2014 pub capped_sccs: usize,
2020 pub max_abs_delta_at_stop: f64,
2024 pub nan_converged: usize,
2027 pub reused_sccs: usize,
2032 pub reused_scc_members: usize,
2034 pub elapsed_ms: u128,
2036}
2037
2038#[derive(Debug, Clone, Copy)]
2039struct ScheduleBuildMeta {
2040 candidate_vertices: usize,
2041 vdeps_vertices: usize,
2042 vdeps_edges: usize,
2043 builder_elapsed_ms: u128,
2044 used_virtual_schedule: bool,
2045 schedule_cache_hit: bool,
2046 schedule_cache_eligible: bool,
2047}
2048
2049#[cfg(any(test, feature = "benchmark_internal"))]
2050#[doc(hidden)]
2051#[derive(Debug, Clone, Default)]
2052pub struct RecalcReuseProbe {
2053 pub schedule_requests: usize,
2054 pub schedule_cache_hits: usize,
2055 pub schedule_cache_misses: usize,
2056 pub schedule_cache_ineligible: usize,
2057 pub schedule_builds: usize,
2058 pub schedule_shared_handles: usize,
2059 pub schedule_retained_bytes: usize,
2060 pub legacy_target_requests: usize,
2061 pub target_schedule_builds: usize,
2062 pub demand_builds: usize,
2063 pub demand_vertices: usize,
2064 pub demand_clean_formulas: usize,
2065 pub demand_explicit_edges: usize,
2066 pub demand_virtual_builder_calls: usize,
2067}
2068
2069#[cfg(any(test, feature = "benchmark_internal"))]
2070fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2071 fn vector_bytes<T>(values: &Vec<T>) -> usize {
2072 values.capacity() * std::mem::size_of::<T>()
2073 }
2074
2075 [
2076 vector_bytes(&schedule.units),
2077 vector_bytes(&schedule.layers),
2078 vector_bytes(&schedule.cycles),
2079 ]
2080 .into_iter()
2081 .chain(
2082 schedule
2083 .layers
2084 .iter()
2085 .map(|layer| vector_bytes(&layer.vertices)),
2086 )
2087 .chain(schedule.cycles.iter().map(vector_bytes))
2088 .sum()
2089}
2090
2091#[derive(Debug, Clone)]
2092struct CachedScheduleEntry {
2093 topology_epoch: u64,
2094 candidate_vertices: Vec<VertexId>,
2095 schedule: Arc<crate::engine::scheduler::Schedule>,
2096}
2097
2098enum EvaluationSchedule {
2100 Owned(crate::engine::scheduler::Schedule),
2101 Shared(Arc<crate::engine::scheduler::Schedule>),
2102}
2103
2104impl std::ops::Deref for EvaluationSchedule {
2105 type Target = crate::engine::scheduler::Schedule;
2106
2107 fn deref(&self) -> &Self::Target {
2108 match self {
2109 Self::Owned(schedule) => schedule,
2110 Self::Shared(schedule) => schedule,
2111 }
2112 }
2113}
2114
2115#[derive(Clone, Debug, PartialEq, Eq)]
2116struct MixedTopologyCacheKey {
2117 engine_topology_epoch: u64,
2118 graph_topology_revision: u64,
2119 authority_indexes_epoch: u64,
2120 legacy_island_revision: u64,
2124 boundary_index_revision: u64,
2125 function_semantic_epoch: u64,
2126 function_provider_revision: Option<u64>,
2127 max_candidates: usize,
2128 max_edges: usize,
2129 max_memory_bytes: usize,
2130}
2131
2132#[derive(Debug)]
2133struct CachedMixedTopology {
2134 key: MixedTopologyCacheKey,
2135 topology: MixedTopology,
2136 producer_results: FormulaProducerResultIndex,
2137 consumer_reads: FormulaConsumerReadIndex,
2138 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
2139 plane_epoch: u64,
2140 island: LegacyIslandPlan,
2141}
2142
2143#[derive(Debug)]
2144struct SkippedMixedTopology {
2145 reason: MixedScheduleFallbackReason,
2146 observed: MixedTopologyCompileStats,
2147 producer_results: FormulaProducerResultIndex,
2148 consumer_reads: FormulaConsumerReadIndex,
2149 span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
2150 plane_epoch: u64,
2151 island: LegacyIslandPlan,
2152}
2153
2154#[derive(Debug)]
2155enum FormulaPlaneTopologyCompileResult {
2156 Cached(CachedMixedTopology),
2157 CacheSkipped(SkippedMixedTopology),
2158}
2159
2160#[derive(Clone, Copy, Debug)]
2161struct ExactDemandScratchEstimates {
2162 paged: u64,
2163 runs: u64,
2164 native: u64,
2165 repeated: u64,
2166}
2167
2168fn exact_demand_scratch_estimates(
2169 read_count: usize,
2170 closure_base: u64,
2171) -> ExactDemandScratchEstimates {
2172 let read_count = read_count as u64;
2173 let page_count = read_count.div_ceil(128);
2174 let run_count = read_count.div_ceil(256);
2175 let pointer_bytes = std::mem::size_of::<usize>() as u64;
2176 let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
2177 + std::mem::size_of::<Vec<usize>>()
2178 + 12 * std::mem::size_of::<usize>()) as u64;
2179 let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
2180 page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
2181 );
2182 let runs_memory = read_count
2183 .saturating_mul(pointer_bytes)
2184 .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
2185 ExactDemandScratchEstimates {
2186 paged: closure_base.saturating_add(paged_memory),
2187 runs: closure_base.saturating_add(runs_memory),
2188 native: closure_base
2189 .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
2190 repeated: closure_base.saturating_add(512),
2191 }
2192}
2193
2194fn native_exact_demand_allowed(
2195 policy: Option<crate::engine::DiskScratchPolicy>,
2196 target_supports_native: bool,
2197) -> bool {
2198 target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
2199}
2200
2201fn select_exact_demand_strategy(
2202 estimates: ExactDemandScratchEstimates,
2203 native_allowed: bool,
2204 mut can_reserve: impl FnMut(u64) -> bool,
2205) -> (FormulaPlaneTopologyStrategy, u64) {
2206 if can_reserve(estimates.paged) {
2207 (
2208 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
2209 estimates.paged,
2210 )
2211 } else if can_reserve(estimates.runs) {
2212 (
2213 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
2214 estimates.runs,
2215 )
2216 } else if native_allowed && can_reserve(estimates.native) {
2217 (
2218 FormulaPlaneTopologyStrategy::ExactNativeScratch,
2219 estimates.native,
2220 )
2221 } else {
2222 (
2223 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
2224 estimates.repeated,
2225 )
2226 }
2227}
2228
2229#[cfg(test)]
2230mod exact_demand_strategy_tests {
2231 use super::*;
2232
2233 #[test]
2234 fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
2235 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2236 assert!(estimates.paged > estimates.runs);
2237 assert!(estimates.runs > estimates.native);
2238 assert!(estimates.native > estimates.repeated);
2239 for (limit, native_allowed, expected) in [
2240 (
2241 estimates.paged,
2242 true,
2243 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
2244 ),
2245 (
2246 estimates.runs,
2247 true,
2248 FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
2249 ),
2250 (
2251 estimates.native,
2252 true,
2253 FormulaPlaneTopologyStrategy::ExactNativeScratch,
2254 ),
2255 (
2256 estimates.repeated,
2257 true,
2258 FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
2259 ),
2260 ] {
2261 assert_eq!(
2262 select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
2263 expected
2264 );
2265 }
2266 }
2267
2268 #[cfg(not(target_arch = "wasm32"))]
2269 #[test]
2270 fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
2271 let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
2272 let path = scratch.path.clone();
2273 let auxiliary_path = scratch.auxiliary_path.clone();
2274 assert!(path.exists());
2275 assert!(auxiliary_path.exists());
2276 drop(scratch);
2277 assert!(!path.exists());
2278 assert!(!auxiliary_path.exists());
2279 }
2280
2281 #[test]
2282 fn native_policy_and_wasm_target_support_gate_native_rung() {
2283 let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2284 assert!(!native_exact_demand_allowed(
2285 Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2286 true,
2287 ));
2288 assert!(!native_exact_demand_allowed(
2289 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2290 false,
2291 ));
2292 let native_allowed = native_exact_demand_allowed(
2293 Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2294 !cfg!(target_arch = "wasm32"),
2295 );
2296 let expected = if cfg!(target_arch = "wasm32") {
2297 FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2298 } else {
2299 FormulaPlaneTopologyStrategy::ExactNativeScratch
2300 };
2301 assert_eq!(
2302 select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2303 bytes <= estimates.native
2304 })
2305 .0,
2306 expected
2307 );
2308 }
2309}
2310
2311#[cfg(not(target_arch = "wasm32"))]
2312struct NativeTopologyScratch {
2313 path: std::path::PathBuf,
2314 file: Option<std::fs::File>,
2315 auxiliary_path: std::path::PathBuf,
2316 auxiliary_file: Option<std::fs::File>,
2317}
2318
2319#[cfg(not(target_arch = "wasm32"))]
2320impl NativeTopologyScratch {
2321 fn create(request_id: u64) -> std::io::Result<Self> {
2322 use std::fs::OpenOptions;
2323 for attempt in 0..32_u64 {
2324 let path = std::env::temp_dir().join(format!(
2325 "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2326 std::process::id()
2327 ));
2328 match OpenOptions::new()
2329 .read(true)
2330 .write(true)
2331 .create_new(true)
2332 .open(&path)
2333 {
2334 Ok(file) => {
2335 let auxiliary_path = path.with_extension("aux.tmp");
2336 match OpenOptions::new()
2337 .read(true)
2338 .write(true)
2339 .create_new(true)
2340 .open(&auxiliary_path)
2341 {
2342 Ok(auxiliary_file) => {
2343 return Ok(Self {
2344 path,
2345 file: Some(file),
2346 auxiliary_path,
2347 auxiliary_file: Some(auxiliary_file),
2348 });
2349 }
2350 Err(error) => {
2351 drop(file);
2352 let _ = std::fs::remove_file(&path);
2353 if error.kind() == std::io::ErrorKind::AlreadyExists {
2354 continue;
2355 }
2356 return Err(error);
2357 }
2358 }
2359 }
2360 Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2361 Err(error) => return Err(error),
2362 }
2363 }
2364 Err(std::io::Error::new(
2365 std::io::ErrorKind::AlreadyExists,
2366 "could not allocate unique topology scratch file",
2367 ))
2368 }
2369}
2370
2371#[cfg(not(target_arch = "wasm32"))]
2372impl Drop for NativeTopologyScratch {
2373 fn drop(&mut self) {
2374 drop(self.file.take());
2375 drop(self.auxiliary_file.take());
2376 let _ = std::fs::remove_file(&self.path);
2377 let _ = std::fs::remove_file(&self.auxiliary_path);
2378 }
2379}
2380
2381fn estimated_span_bindings_bytes(
2382 bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2383) -> Option<usize> {
2384 const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2385 bindings.len().checked_mul(
2386 std::mem::size_of::<FormulaSpanId>()
2387 .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2388 .checked_add(TREE_ENTRY_OVERHEAD)?,
2389 )
2390}
2391
2392type ScheduleBuildOutput = (
2393 crate::engine::scheduler::Schedule,
2394 FxHashMap<VertexId, Vec<VertexId>>,
2395 ScheduleBuildMeta,
2396);
2397
2398type EvaluationScheduleBuildOutput = (
2399 EvaluationSchedule,
2400 FxHashMap<VertexId, Vec<VertexId>>,
2401 ScheduleBuildMeta,
2402);
2403
2404#[derive(Debug)]
2406pub struct RecalcPlan {
2407 key: RecalcPlanKey,
2408 kind: RecalcPlanKind,
2409}
2410
2411#[derive(Debug)]
2412struct RecalcPlanKey {
2413 engine_token: Arc<()>,
2414 revisions: PlanningRevisionSnapshot,
2415}
2416
2417#[derive(Clone, Debug, PartialEq, Eq)]
2418struct PlanningRevisionSnapshot {
2419 engine_topology_epoch: u64,
2420 graph_topology_revision: u64,
2421 authority: u64,
2422 authority_indexes: u64,
2423 authority_indexed_plane: u64,
2424 staged: u64,
2425 symbols: u64,
2426 semantic: u64,
2427 provider: Option<u64>,
2428 formula_plane_mode: FormulaPlaneMode,
2429 deterministic_mode: crate::engine::DeterministicMode,
2430 budgets: crate::engine::EvaluationBudgets,
2431 span_refs: Vec<FormulaSpanRef>,
2432}
2433
2434#[derive(Debug)]
2435enum RecalcPlanKind {
2436 CompatibilityFull {
2437 schedule: crate::engine::Schedule,
2438 has_dynamic_refs: bool,
2439 },
2440 Target {
2441 targets: Vec<crate::engine::EvaluationTarget>,
2442 scope: crate::engine::PrepareScope,
2443 topology: RecalcTopology,
2444 dynamic_policy: DynamicPlanPolicy,
2445 },
2446}
2447
2448#[derive(Debug)]
2449enum RecalcTopology {
2450 RunLocalRecipe,
2451 Workbook,
2452}
2453
2454#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2455enum DynamicPlanPolicy {
2456 BoundedTargetReplan,
2457}
2458
2459impl RecalcPlan {
2460 pub fn layer_count(&self) -> usize {
2463 match &self.kind {
2464 RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2465 RecalcPlanKind::Target { .. } => 0,
2466 }
2467 }
2468
2469 pub fn has_dynamic_refs(&self) -> bool {
2470 match &self.kind {
2471 RecalcPlanKind::CompatibilityFull {
2472 has_dynamic_refs, ..
2473 } => *has_dynamic_refs,
2474 RecalcPlanKind::Target { .. } => false,
2475 }
2476 }
2477
2478 #[cfg(test)]
2479 pub(crate) fn force_stale_reasons_for_test(
2480 &mut self,
2481 reasons: &[formualizer_common::PlanStaleReason],
2482 ) {
2483 use formualizer_common::PlanStaleReason;
2484 for reason in reasons {
2485 match reason {
2486 PlanStaleReason::Engine => {
2487 self.key.engine_token = Arc::new(());
2488 }
2489 PlanStaleReason::Provider => {
2490 self.key.revisions.provider = Some(
2491 self.key
2492 .revisions
2493 .provider
2494 .unwrap_or_default()
2495 .wrapping_add(1),
2496 );
2497 }
2498 PlanStaleReason::Semantic => {
2499 self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2500 }
2501 PlanStaleReason::Budget => {
2502 let current = self.key.revisions.budgets.work.max_work_units;
2503 self.key.revisions.budgets.work.max_work_units =
2504 Some(current.unwrap_or_default().wrapping_add(1));
2505 }
2506 PlanStaleReason::Staged => {
2507 self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2508 }
2509 PlanStaleReason::Symbols => {
2510 self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2511 }
2512 PlanStaleReason::Authority => {
2513 self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2514 }
2515 PlanStaleReason::SpanGeneration => {
2516 self.key.revisions.span_refs.push(FormulaSpanRef {
2517 id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2518 generation: u32::MAX,
2519 version: u32::MAX,
2520 });
2521 }
2522 PlanStaleReason::Graph => {
2523 self.key.revisions.graph_topology_revision =
2524 self.key.revisions.graph_topology_revision.wrapping_add(1);
2525 }
2526 _ => {}
2527 }
2528 }
2529 }
2530}
2531
2532#[cfg(any(test, feature = "test-support"))]
2533pub(crate) mod criteria_mask_test_hooks {
2534 use std::cell::Cell;
2535
2536 thread_local! {
2537 static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2538 static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2539 static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2540 }
2541
2542 pub(crate) fn take_mask_work() -> (usize, usize) {
2543 MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2544 }
2545
2546 pub(crate) fn note_mask(rows: usize) {
2547 MASK_CALLS_ROWS.with(|c| {
2548 let (calls, work) = c.get();
2549 c.set((calls + 1, work + rows));
2550 });
2551 }
2552
2553 pub fn reset_text_segment_counters() {
2554 TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2555 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2556 }
2557
2558 pub fn text_segment_counters() -> (usize, usize) {
2559 let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2560 let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2561 (a, b)
2562 }
2563
2564 pub(crate) fn inc_total() {
2565 TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2566 }
2567 pub(crate) fn inc_all_null() {
2568 TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2569 }
2570}
2571
2572#[cfg(test)]
2573pub(crate) mod visibility_mask_test_hooks {
2574 use std::cell::Cell;
2575
2576 thread_local! {
2577 static HITS: Cell<usize> = const { Cell::new(0) };
2578 static MISSES: Cell<usize> = const { Cell::new(0) };
2579 static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2580 }
2581
2582 pub fn reset() {
2583 HITS.with(|c| c.set(0));
2584 MISSES.with(|c| c.set(0));
2585 EVICTIONS.with(|c| c.set(0));
2586 }
2587
2588 pub fn counters() -> (usize, usize, usize) {
2589 let hits = HITS.with(|c| c.get());
2590 let misses = MISSES.with(|c| c.get());
2591 let evictions = EVICTIONS.with(|c| c.get());
2592 (hits, misses, evictions)
2593 }
2594
2595 pub(crate) fn inc_hit() {
2596 HITS.with(|c| c.set(c.get() + 1));
2597 }
2598
2599 pub(crate) fn inc_miss() {
2600 MISSES.with(|c| c.set(c.get() + 1));
2601 }
2602
2603 pub(crate) fn inc_eviction() {
2604 EVICTIONS.with(|c| c.set(c.get() + 1));
2605 }
2606}
2607
2608fn compute_criteria_mask(
2609 view: &RangeView<'_>,
2610 col_in_view: usize,
2611 pred: &crate::args::CriteriaPredicate,
2612) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2613 use crate::compute_prelude::{boolean, cmp, concat_arrays};
2614 use arrow::compute::kernels::comparison::{ilike, nilike};
2615 use arrow_array::{
2616 Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2617 };
2618
2619 fn apply_numeric_pred(
2621 chunk: &Float64Array,
2622 pred: &crate::args::CriteriaPredicate,
2623 ) -> Option<BooleanArray> {
2624 match pred {
2625 crate::args::CriteriaPredicate::Gt(n) => {
2626 cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2627 }
2628 crate::args::CriteriaPredicate::Ge(n) => {
2629 cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2630 }
2631 crate::args::CriteriaPredicate::Lt(n) => {
2632 cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2633 }
2634 crate::args::CriteriaPredicate::Le(n) => {
2635 cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2636 }
2637 crate::args::CriteriaPredicate::Eq(v) => match v {
2638 formualizer_common::LiteralValue::Number(x) => {
2639 cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2640 }
2641 formualizer_common::LiteralValue::Int(i) => {
2642 cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2643 }
2644 _ => None,
2645 },
2646 crate::args::CriteriaPredicate::Ne(v) => match v {
2647 formualizer_common::LiteralValue::Number(x) => {
2648 cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2649 }
2650 formualizer_common::LiteralValue::Int(i) => {
2651 cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2652 }
2653 _ => None,
2654 },
2655 _ => None,
2656 }
2657 }
2658
2659 let is_numeric_pred = matches!(
2661 pred,
2662 crate::args::CriteriaPredicate::Gt(_)
2663 | crate::args::CriteriaPredicate::Ge(_)
2664 | crate::args::CriteriaPredicate::Lt(_)
2665 | crate::args::CriteriaPredicate::Le(_)
2666 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2667 | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2668 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2669 | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2670 );
2671
2672 if is_numeric_pred {
2676 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2677 for res in view.numbers_slices() {
2678 let (_rs, _rl, cols_seg) = res.ok()?;
2679 if col_in_view < cols_seg.len() {
2680 let chunk = cols_seg[col_in_view].as_ref();
2681 let mask = apply_numeric_pred(chunk, pred)?;
2682 bool_parts.push(mask);
2683 }
2684 }
2685
2686 if bool_parts.is_empty() {
2687 return None;
2688 } else if bool_parts.len() == 1 {
2689 return Some(std::sync::Arc::new(bool_parts.remove(0)));
2690 } else {
2691 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2693 .iter()
2694 .map(|a| a as &dyn arrow_array::Array)
2695 .collect();
2696 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2697 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2698 return Some(std::sync::Arc::new(ba));
2699 }
2700 }
2701
2702 let (text_kind, text_pat, empty_special) = match pred {
2705 crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2706 (0u8, t.to_lowercase(), t.is_empty())
2707 }
2708 crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2709 (1u8, t.to_lowercase(), false)
2710 }
2711 crate::args::CriteriaPredicate::TextLike {
2712 pattern,
2713 case_insensitive,
2714 } => {
2715 let p = if *case_insensitive {
2716 pattern.to_lowercase()
2717 } else {
2718 pattern.clone()
2719 };
2720 (2u8, p.replace('*', "%").replace('?', "_"), false)
2721 }
2722 _ => return None,
2723 };
2724
2725 let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2726 let pat = StringArray::new_scalar(text_pat);
2727 let mut bool_parts: Vec<BooleanArray> = Vec::new();
2728
2729 for res in view.iter_row_chunks() {
2730 let cs = res.ok()?;
2731 if cs.row_len == 0 {
2732 continue;
2733 }
2734 #[cfg(test)]
2735 criteria_mask_test_hooks::inc_total();
2736
2737 let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2738 if col_in_view >= slices.len() {
2739 return None;
2740 }
2741
2742 let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2743 let seg = match seg_opt {
2744 Some(s) => s,
2745 None => {
2746 #[cfg(test)]
2747 criteria_mask_test_hooks::inc_all_null();
2748 if (text_kind == 0 && empty_special) || ne_matches_blank {
2749 let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2751 bb.append_n(cs.row_len, true);
2752 bool_parts.push(bb.finish());
2753 } else {
2754 bool_parts.push(BooleanArray::new_null(cs.row_len));
2756 }
2757 continue;
2758 }
2759 };
2760
2761 let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2762 let mut m = match text_kind {
2763 0 => ilike(seg_sa, &pat).ok()?,
2764 1 => nilike(seg_sa, &pat).ok()?,
2765 2 => ilike(seg_sa, &pat).ok()?,
2766 _ => return None,
2767 };
2768
2769 if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2775 let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2777 for i in 0..seg_sa.len() {
2778 bb.append_value(seg_sa.is_null(i));
2779 }
2780 let nulls = bb.finish();
2781 m = boolean::or_kleene(&m, &nulls).ok()?;
2782 }
2783
2784 bool_parts.push(m);
2785 }
2786
2787 if bool_parts.is_empty() {
2788 None
2789 } else if bool_parts.len() == 1 {
2790 Some(std::sync::Arc::new(bool_parts.remove(0)))
2791 } else {
2792 let anys: Vec<&dyn arrow_array::Array> = bool_parts
2793 .iter()
2794 .map(|a| a as &dyn arrow_array::Array)
2795 .collect();
2796 let conc: ArrayRef = concat_arrays(&anys).ok()?;
2797 let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2798 Some(std::sync::Arc::new(ba))
2799 }
2800}
2801
2802#[derive(Debug, Clone)]
2803pub struct LayerInfo {
2804 pub vertex_count: usize,
2805 pub parallel_eligible: bool,
2806 pub sample_cells: Vec<String>, }
2808
2809#[derive(Debug, Clone)]
2810pub struct EvalPlan {
2811 pub total_vertices_to_evaluate: usize,
2812 pub layers: Vec<LayerInfo>,
2813 pub cycles_detected: usize,
2814 pub dirty_count: usize,
2815 pub volatile_count: usize,
2816 pub parallel_enabled: bool,
2817 pub estimated_parallel_layers: usize,
2818 pub target_cells: Vec<String>,
2819}
2820
2821impl<R> Engine<R>
2822where
2823 R: EvaluationContext,
2824{
2825 pub fn new(resolver: R, config: EvalConfig) -> Self {
2832 if let Err(msg) = config.cycle.validate() {
2833 panic!("invalid CycleConfig: {msg}");
2834 }
2835 crate::builtins::load_builtins();
2836 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2837 &config.evaluation_budgets,
2838 config.max_vertices,
2839 config.max_memory_mb,
2840 config.max_eval_time,
2841 );
2842
2843 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2844 #[cfg(feature = "system-clock")]
2845 {
2846 Arc::new(crate::timezone::SystemClock::new(
2847 crate::timezone::TimeZoneSpec::default(),
2848 ))
2849 }
2850 #[cfg(not(feature = "system-clock"))]
2851 {
2852 Arc::new(crate::timezone::FixedClock::new(
2853 chrono::DateTime::UNIX_EPOCH,
2854 crate::timezone::TimeZoneSpec::Utc,
2855 ))
2856 }
2857 });
2858
2859 let thread_pool = if config.enable_parallel {
2861 let mut builder = ThreadPoolBuilder::new();
2862 if let Some(max_threads) = config.max_threads {
2863 builder = builder.num_threads(max_threads);
2864 }
2865
2866 match builder.build() {
2867 Ok(pool) => Some(Arc::new(pool)),
2868 Err(_) => {
2869 None
2871 }
2872 }
2873 } else {
2874 None
2875 };
2876
2877 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2879 let function_provider_revision_seen = resolver.planning_semantic_revision();
2880 let mut engine = Self {
2881 graph: DependencyGraph::new_with_config(config.clone()),
2882 resolver,
2883 config,
2884 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2885 clock: crate::timezone::SnapshotClock::new(clock),
2886 thread_pool,
2887 recalc_epoch: 0,
2888 snapshot_id: std::sync::atomic::AtomicU64::new(1),
2889 topology_epoch: 0,
2890 legacy_island_structural_summaries_trusted: true,
2891 cached_static_schedule: None,
2892 #[cfg(any(test, feature = "benchmark_internal"))]
2893 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2894 cached_mixed_topology: None,
2895 mixed_topology_cache_builds: 0,
2896 mixed_topology_cache_hits: 0,
2897 mixed_topology_cache_overflows: 0,
2898 mixed_topology_cache_skip_streak: 0,
2899 spill_mgr: ShimSpillManager::default(),
2900 arrow_sheets: SheetStore::default(),
2901 format_registry: crate::format::FormatRegistry::default(),
2902 derived_format_results: std::sync::RwLock::new(FxHashMap::default()),
2903 derived_formats: std::sync::RwLock::new(FxHashMap::default()),
2904 derived_formats_plane_epoch: 0,
2905 #[cfg(test)]
2906 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2907 #[cfg(test)]
2908 computed_overlay_set_explicit_entry_operations_for_test: 0,
2909 #[cfg(test)]
2910 computed_overlay_stale_clear_range_effects_for_test: 0,
2911 #[cfg(test)]
2912 computed_overlay_stale_clear_offset_attempts_for_test: 0,
2913 #[cfg(test)]
2914 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2915 has_edited: false,
2916 overlay_compactions: 0,
2917 computed_overlay_bytes_estimate: 0,
2918 computed_overlay_mirroring_disabled: false,
2919 force_materialize_range_views: false,
2920 row_bounds_cache: std::sync::RwLock::new(None),
2921 used_axis_bounds_cache: std::sync::RwLock::new(None),
2922 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2923 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2924 source_formula_token: Arc::new(()),
2925 recalc_plan_token: Arc::new(()),
2926 staged_formulas: std::collections::HashMap::new(),
2927 staged_formula_index: StagedFormulaIndex::default(),
2928 row_visibility: FxHashMap::default(),
2929 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2930 formula_parse_diagnostics: Vec::new(),
2931 last_formula_ingest_report: None,
2932 formula_ingest_report_total: FormulaIngestReport::default(),
2933 formula_plane_cycle_member_span_demotions: 0,
2934 formula_plane_array_result_span_demotions: 0,
2935 formula_plane_capacity_bailouts: 0,
2936 formula_plane_structural_span_candidates: 0,
2937 active_cancel_flag: None,
2938 active_evaluation_deadline: None,
2939 action_depth: 0,
2940 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2941 virtual_dep_fallback_activations: 0,
2942 last_cycle_telemetry: CycleTelemetry::default(),
2943 next_evaluation_resource_request_id: 1,
2944 evaluation_resource_request_depth: 0,
2945 active_evaluation_resource_request: None,
2946 last_evaluation_resource_request: None,
2947 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2948 evaluation_resource_request_started_at: None,
2949 evaluation_resource_budgets: resolved_resources.budgets,
2950 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2951 active_resource_ledger: None,
2952 pending_iterative_redirty: Vec::new(),
2953 retained_scc_members: FxHashMap::default(),
2954 next_retained_scc_id: 0,
2955 retained_scc_config_fingerprint: 0,
2956 retained_scc_function_epoch_seen: 0,
2957 retained_scc_provider_revision_seen: None,
2958 retained_scc_dirty_at_begin: Vec::new(),
2959 iterative_state_values: FxHashMap::default(),
2960 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2961 function_provider_revision_seen,
2962 #[cfg(test)]
2963 last_formula_plane_span_eval_report: None,
2964 #[cfg(test)]
2965 evaluation_request_begin_count_for_test: 0,
2966 #[cfg(any(test, feature = "test-support"))]
2967 before_prepared_span_commit_hook: None,
2968 #[cfg(test)]
2969 before_target_preparation_commit_hook: None,
2970 #[cfg(test)]
2971 before_target_planning_snapshot_hook: None,
2972 #[cfg(test)]
2973 inject_target_semantic_stale_once_for_test: false,
2974 #[cfg(test)]
2975 force_virtual_dep_changes_remaining_for_test: 0,
2976 #[cfg(test)]
2977 fail_evaluation_commit_preflight_once_for_test: false,
2978 #[cfg(test)]
2979 cancel_before_formula_plane_layer_commit_once_for_test: false,
2980 #[cfg(test)]
2981 force_source_family_fallback: false,
2982 #[cfg(test)]
2983 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2984 #[cfg(test)]
2985 fragmented_commit_fault_for_test: None,
2986 #[cfg(test)]
2987 formula_span_demotion_fault_for_test: None,
2988 #[cfg(test)]
2989 target_preparation_fault_for_test: None,
2990 #[cfg(test)]
2991 force_non_cycle_schedule_fallback_for_test: false,
2992 #[cfg(test)]
2993 mixed_topology_index_builds_for_test: 0,
2994 #[cfg(test)]
2995 before_legacy_fallback_final_provider_sample_hook: None,
2996 #[cfg(test)]
2997 after_eager_proposal_commit_hook: None,
2998 };
2999 engine.config.arrow_storage_enabled = true;
3001 engine.config.delta_overlay_enabled = true;
3002 engine.config.write_formula_overlay_enabled = true;
3003 let default_sheet = engine.graph.default_sheet_name().to_string();
3004 engine.ensure_arrow_sheet(&default_sheet);
3005 engine
3006 }
3007
3008 pub fn with_thread_pool(
3013 resolver: R,
3014 config: EvalConfig,
3015 thread_pool: Arc<rayon::ThreadPool>,
3016 ) -> Self {
3017 if let Err(msg) = config.cycle.validate() {
3018 panic!("invalid CycleConfig: {msg}");
3019 }
3020 crate::builtins::load_builtins();
3021 let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3022 &config.evaluation_budgets,
3023 config.max_vertices,
3024 config.max_memory_mb,
3025 config.max_eval_time,
3026 );
3027 let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3028 #[cfg(feature = "system-clock")]
3029 {
3030 Arc::new(crate::timezone::SystemClock::new(
3031 crate::timezone::TimeZoneSpec::default(),
3032 ))
3033 }
3034 #[cfg(not(feature = "system-clock"))]
3035 {
3036 Arc::new(crate::timezone::FixedClock::new(
3037 chrono::DateTime::UNIX_EPOCH,
3038 crate::timezone::TimeZoneSpec::Utc,
3039 ))
3040 }
3041 });
3042 let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3044 let function_provider_revision_seen = resolver.planning_semantic_revision();
3045 let mut engine = Self {
3046 graph: DependencyGraph::new_with_config(config.clone()),
3047 resolver,
3048 config,
3049 workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3050 clock: crate::timezone::SnapshotClock::new(clock),
3051 thread_pool: Some(thread_pool),
3052 recalc_epoch: 0,
3053 snapshot_id: std::sync::atomic::AtomicU64::new(1),
3054 topology_epoch: 0,
3055 legacy_island_structural_summaries_trusted: true,
3056 cached_static_schedule: None,
3057 #[cfg(any(test, feature = "benchmark_internal"))]
3058 recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3059 cached_mixed_topology: None,
3060 mixed_topology_cache_builds: 0,
3061 mixed_topology_cache_hits: 0,
3062 mixed_topology_cache_overflows: 0,
3063 mixed_topology_cache_skip_streak: 0,
3064 spill_mgr: ShimSpillManager::default(),
3065 arrow_sheets: SheetStore::default(),
3066 format_registry: crate::format::FormatRegistry::default(),
3067 derived_format_results: std::sync::RwLock::new(FxHashMap::default()),
3068 derived_formats: std::sync::RwLock::new(FxHashMap::default()),
3069 derived_formats_plane_epoch: 0,
3070 #[cfg(test)]
3071 derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3072 #[cfg(test)]
3073 computed_overlay_set_explicit_entry_operations_for_test: 0,
3074 #[cfg(test)]
3075 computed_overlay_stale_clear_range_effects_for_test: 0,
3076 #[cfg(test)]
3077 computed_overlay_stale_clear_offset_attempts_for_test: 0,
3078 #[cfg(test)]
3079 computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3080 has_edited: false,
3081 overlay_compactions: 0,
3082 computed_overlay_bytes_estimate: 0,
3083 computed_overlay_mirroring_disabled: false,
3084 force_materialize_range_views: false,
3085 row_bounds_cache: std::sync::RwLock::new(None),
3086 used_axis_bounds_cache: std::sync::RwLock::new(None),
3087 lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3088 source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3089 source_formula_token: Arc::new(()),
3090 recalc_plan_token: Arc::new(()),
3091 staged_formulas: std::collections::HashMap::new(),
3092 staged_formula_index: StagedFormulaIndex::default(),
3093 row_visibility: FxHashMap::default(),
3094 row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3095 formula_parse_diagnostics: Vec::new(),
3096 last_formula_ingest_report: None,
3097 formula_ingest_report_total: FormulaIngestReport::default(),
3098 formula_plane_cycle_member_span_demotions: 0,
3099 formula_plane_array_result_span_demotions: 0,
3100 formula_plane_capacity_bailouts: 0,
3101 formula_plane_structural_span_candidates: 0,
3102 active_cancel_flag: None,
3103 active_evaluation_deadline: None,
3104 action_depth: 0,
3105 last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3106 virtual_dep_fallback_activations: 0,
3107 last_cycle_telemetry: CycleTelemetry::default(),
3108 next_evaluation_resource_request_id: 1,
3109 evaluation_resource_request_depth: 0,
3110 active_evaluation_resource_request: None,
3111 last_evaluation_resource_request: None,
3112 evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3113 evaluation_resource_request_started_at: None,
3114 evaluation_resource_budgets: resolved_resources.budgets,
3115 evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3116 active_resource_ledger: None,
3117 pending_iterative_redirty: Vec::new(),
3118 retained_scc_members: FxHashMap::default(),
3119 next_retained_scc_id: 0,
3120 retained_scc_config_fingerprint: 0,
3121 retained_scc_function_epoch_seen: 0,
3122 retained_scc_provider_revision_seen: None,
3123 retained_scc_dirty_at_begin: Vec::new(),
3124 iterative_state_values: FxHashMap::default(),
3125 function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3126 function_provider_revision_seen,
3127 #[cfg(test)]
3128 last_formula_plane_span_eval_report: None,
3129 #[cfg(test)]
3130 evaluation_request_begin_count_for_test: 0,
3131 #[cfg(any(test, feature = "test-support"))]
3132 before_prepared_span_commit_hook: None,
3133 #[cfg(test)]
3134 before_target_preparation_commit_hook: None,
3135 #[cfg(test)]
3136 before_target_planning_snapshot_hook: None,
3137 #[cfg(test)]
3138 inject_target_semantic_stale_once_for_test: false,
3139 #[cfg(test)]
3140 force_virtual_dep_changes_remaining_for_test: 0,
3141 #[cfg(test)]
3142 fail_evaluation_commit_preflight_once_for_test: false,
3143 #[cfg(test)]
3144 cancel_before_formula_plane_layer_commit_once_for_test: false,
3145 #[cfg(test)]
3146 force_source_family_fallback: false,
3147 #[cfg(test)]
3148 rerecord_cycle_retry_span_after_lease_extension_for_test: false,
3149 #[cfg(test)]
3150 fragmented_commit_fault_for_test: None,
3151 #[cfg(test)]
3152 formula_span_demotion_fault_for_test: None,
3153 #[cfg(test)]
3154 target_preparation_fault_for_test: None,
3155 #[cfg(test)]
3156 force_non_cycle_schedule_fallback_for_test: false,
3157 #[cfg(test)]
3158 mixed_topology_index_builds_for_test: 0,
3159 #[cfg(test)]
3160 before_legacy_fallback_final_provider_sample_hook: None,
3161 #[cfg(test)]
3162 after_eager_proposal_commit_hook: None,
3163 };
3164 engine.config.arrow_storage_enabled = true;
3166 engine.config.delta_overlay_enabled = true;
3167 engine.config.write_formula_overlay_enabled = true;
3168 let default_sheet = engine.graph.default_sheet_name().to_string();
3169 engine.ensure_arrow_sheet(&default_sheet);
3170 engine
3171 }
3172
3173 pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3174 &self.workbook_load_limits
3175 }
3176
3177 pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3178 self.workbook_load_limits = limits;
3179 }
3180
3181 fn clear_source_cache(&self) {
3182 if let Ok(mut g) = self.source_cache.write() {
3183 *g = SourceCache::default();
3184 }
3185 }
3186
3187 pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3188 &self.last_virtual_dep_telemetry
3189 }
3190
3191 pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3195 &self.last_cycle_telemetry
3196 }
3197
3198 pub fn last_evaluation_resource_request_stats(
3200 &self,
3201 ) -> Option<&EvaluationResourceRequestStats> {
3202 self.last_evaluation_resource_request.as_ref()
3203 }
3204
3205 pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3207 self.evaluation_resource_baseline
3208 }
3209
3210 pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3211 &self.evaluation_resource_budgets
3212 }
3213
3214 pub fn evaluation_resource_config_diagnostic(
3216 &self,
3217 ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3218 self.evaluation_resource_config_diagnostic.as_ref()
3219 }
3220
3221 pub fn reset_evaluation_resource_telemetry(&mut self) {
3223 self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3224 self.last_evaluation_resource_request = None;
3225 }
3226
3227 fn duration_ns(duration: std::time::Duration) -> u64 {
3228 u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3229 }
3230
3231 fn observe_evaluation_resource_request<T>(
3232 &mut self,
3233 kind: EvaluationRequestKind,
3234 evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3235 ) -> Result<T, ExcelError> {
3236 let outermost = self.evaluation_resource_request_depth == 0;
3237 if outermost {
3238 let request_id = self.next_evaluation_resource_request_id;
3239 self.next_evaluation_resource_request_id = request_id
3240 .checked_add(1)
3241 .expect("evaluation resource request ID exhausted");
3242 self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3243 request_id,
3244 kind,
3245 self.config.formula_plane_mode,
3246 self.staged_formula_count(),
3247 ));
3248 self.evaluation_resource_baseline.record_started(request_id);
3249 self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3250 self.active_resource_ledger = Some(ResourceLedger::new(
3251 Some(request_id),
3252 self.evaluation_resource_budgets.clone(),
3253 ));
3254 }
3255 self.evaluation_resource_request_depth =
3256 self.evaluation_resource_request_depth.saturating_add(1);
3257 let result = if outermost {
3258 self.resource_checkpoint(0).and_then(|()| evaluate(self))
3259 } else {
3260 evaluate(self)
3261 };
3262 self.evaluation_resource_request_depth =
3263 self.evaluation_resource_request_depth.saturating_sub(1);
3264
3265 if outermost {
3266 let total_ns = self
3267 .evaluation_resource_request_started_at
3268 .take()
3269 .map(|start| Self::duration_ns(start.elapsed()))
3270 .unwrap_or(0);
3271 let mut stats = self
3272 .active_evaluation_resource_request
3273 .take()
3274 .expect("outer evaluation resource request has active stats");
3275 let mut ledger = self
3276 .active_resource_ledger
3277 .take()
3278 .expect("outer evaluation resource request has active ledger");
3279 ledger.release_all_scratch();
3280 stats.ledger.update(ledger.snapshot());
3281 stats.outcome = match &result {
3282 Ok(_) => EvaluationRequestOutcome::Success,
3283 Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3284 EvaluationRequestOutcome::Cancelled
3285 }
3286 Err(_) => EvaluationRequestOutcome::Error,
3287 };
3288 if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3289 stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3290 FormulaDirtyLeaseOutcome::RetainedOnCancellation
3291 } else {
3292 FormulaDirtyLeaseOutcome::RetainedOnError
3293 };
3294 }
3295 stats.phases.total_ns = total_ns;
3296 let attributed = stats
3297 .phases
3298 .staged_prepare_ns
3299 .saturating_add(stats.phases.topology_ns)
3300 .saturating_add(stats.phases.materialization_ns);
3301 stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3302 self.evaluation_resource_baseline.record_finished(&stats);
3303 self.last_evaluation_resource_request = Some(stats);
3304 }
3305 result
3306 }
3307
3308 pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3309 self.evaluation_resource_budgets = budgets.clone();
3310 self.config.evaluation_budgets = budgets.clone();
3311 self.graph.set_evaluation_budgets(budgets);
3312 }
3313
3314 #[cfg(test)]
3315 pub(crate) fn set_evaluation_budgets_for_test(
3316 &mut self,
3317 budgets: crate::engine::EvaluationBudgets,
3318 ) {
3319 self.set_evaluation_resource_budgets(budgets);
3320 }
3321
3322 fn resource_loop_checkpoint(
3323 ledger: &mut Option<ResourceLedger>,
3324 work_units: u64,
3325 ) -> Result<(), ExcelError> {
3326 ledger
3327 .as_mut()
3328 .map_or(Ok(()), |ledger| {
3329 ledger
3330 .charge_work(work_units)
3331 .and_then(|()| ledger.checkpoint_deadline())
3332 })
3333 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3334 }
3335
3336 fn preflight_evaluation_commit_window(
3337 &mut self,
3338 bounded_writes: usize,
3339 ) -> Result<crate::instant::FzInstant, ExcelError> {
3340 #[cfg(test)]
3341 if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3342 return Err(crate::engine::ResourceLedgerError::Exhausted(
3343 formualizer_common::ResourceExhaustionDetail {
3344 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3345 limit: 0,
3346 observed: 1,
3347 request_id: self
3348 .active_evaluation_resource_request
3349 .as_ref()
3350 .map(|stats| stats.request_id),
3351 },
3352 )
3353 .into_excel_error());
3354 }
3355 let estimate = std::time::Duration::from_nanos(
3356 u64::try_from(bounded_writes)
3357 .unwrap_or(u64::MAX)
3358 .saturating_mul(100),
3359 );
3360 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3361 ledger
3362 .preflight_commit_window(estimate)
3363 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3364 }
3365 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3366 stats.evaluation_commit_preflight_count =
3367 stats.evaluation_commit_preflight_count.saturating_add(1);
3368 stats.evaluation_commit_estimated_ns = stats
3369 .evaluation_commit_estimated_ns
3370 .saturating_add(Self::duration_ns(estimate));
3371 }
3372 Ok(crate::instant::FzInstant::now())
3373 }
3374
3375 fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3376 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3377 stats.evaluation_commit_actual_ns = stats
3378 .evaluation_commit_actual_ns
3379 .saturating_add(Self::duration_ns(started.elapsed()));
3380 }
3381 }
3382
3383 fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3384 if self
3385 .active_cancel_flag
3386 .as_ref()
3387 .is_some_and(|cancel| cancel.is_cancelled())
3388 {
3389 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3390 }
3391 if self
3392 .active_evaluation_deadline
3393 .is_some_and(|deadline| Instant::now() >= deadline)
3394 {
3395 return Err(crate::engine::ResourceLedgerError::Exhausted(
3396 formualizer_common::ResourceExhaustionDetail {
3397 reason: formualizer_common::ResourceExhaustionReason::Deadline,
3398 limit: 0,
3399 observed: 1,
3400 request_id: self
3401 .active_evaluation_resource_request
3402 .as_ref()
3403 .map(|request| request.request_id),
3404 },
3405 )
3406 .into_excel_error()
3407 .with_message(message));
3408 }
3409 Ok(())
3410 }
3411
3412 fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3413 let Some(ledger) = self.active_resource_ledger.as_mut() else {
3414 return Ok(());
3415 };
3416 ledger
3417 .charge_work(work_units)
3418 .and_then(|()| ledger.checkpoint_deadline())
3419 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3420 }
3421
3422 fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3423 if work_units == 0 {
3424 return self.resource_checkpoint(0);
3425 }
3426 while work_units > 0 {
3427 let chunk = work_units.min(256);
3428 self.resource_checkpoint(chunk)?;
3429 work_units -= chunk;
3430 }
3431 Ok(())
3432 }
3433
3434 fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3435 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3436 ledger.observe_scratch(bytes);
3439 }
3440 Ok(())
3441 }
3442
3443 fn release_request_scratch(&mut self, bytes: u64) {
3444 if let Some(ledger) = self.active_resource_ledger.as_mut() {
3445 let released = ledger.release_scratch(bytes);
3446 debug_assert!(
3447 released.is_ok(),
3448 "request scratch release exceeded the outstanding reservation"
3449 );
3450 }
3451 }
3452
3453 fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3454 self.active_resource_ledger
3455 .as_ref()
3456 .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3457 }
3458
3459 fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3460 self.active_resource_ledger
3461 .as_mut()
3462 .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3463 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3464 }
3465
3466 fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3467 self.active_resource_ledger
3468 .as_mut()
3469 .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3470 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3471 }
3472
3473 fn with_request_scratch<T>(
3474 &mut self,
3475 bytes: u64,
3476 work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3477 ) -> Result<T, ExcelError> {
3478 self.reserve_request_scratch(bytes)?;
3479 let result = work(self);
3480 self.release_request_scratch(bytes);
3481 result
3482 }
3483
3484 fn observe_staged_preparation(
3485 &mut self,
3486 selected: usize,
3487 retained: usize,
3488 elapsed: std::time::Duration,
3489 ) {
3490 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3491 stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3492 stats.staged_retained = retained as u64;
3493 stats.phases.staged_prepare_ns = stats
3494 .phases
3495 .staged_prepare_ns
3496 .saturating_add(Self::duration_ns(elapsed));
3497 }
3498 }
3499
3500 fn graph_admission_enabled(&self) -> bool {
3501 crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3502 }
3503
3504 fn preflight_graph_admission(
3505 &mut self,
3506 usage: crate::engine::resource_ledger::GraphAdmission,
3507 ) -> Result<(), ExcelError> {
3508 let request_id = self
3509 .active_evaluation_resource_request
3510 .as_ref()
3511 .map(|stats| stats.request_id);
3512 crate::engine::resource_ledger::preflight_graph_admission(
3513 &self.evaluation_resource_budgets,
3514 usage,
3515 request_id,
3516 )
3517 .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3518 }
3519
3520 fn prepared_legacy_admission(
3521 &mut self,
3522 plan: &PreparedLegacyGraphPlan,
3523 materialization_cells: u64,
3524 ) -> Result<(), ExcelError> {
3525 if !self.graph_admission_enabled() {
3526 return Ok(());
3527 }
3528 let stats = self.graph.baseline_stats();
3529 let added_edges = plan.planned_edge_count().ok_or_else(|| {
3530 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3531 })?;
3532 let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3533 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3534 })?;
3535 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3536 final_vertices: stats
3537 .graph_vertex_count
3538 .checked_add(plan.new_vertex_count())
3539 .ok_or_else(|| {
3540 ExcelError::new(ExcelErrorKind::NImpl)
3541 .with_message("graph vertex count overflow")
3542 })?,
3543 final_edges: stats
3544 .graph_edge_count
3545 .checked_sub(removed_edges)
3546 .and_then(|count| count.checked_add(added_edges))
3547 .ok_or_else(|| {
3548 ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3549 })?,
3550 materialization_cells,
3551 added_vertices: plan.new_vertex_count(),
3552 added_edges,
3553 })
3554 }
3555
3556 fn observe_target_admission_failure(
3557 &mut self,
3558 reason: formualizer_common::ResourceExhaustionReason,
3559 ) {
3560 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3561 stats.target_admission_failure = Some(reason);
3562 }
3563 }
3564
3565 fn observe_target_preparation_report(
3566 &mut self,
3567 report: &crate::engine::PreparedTargetGraphReport,
3568 ) {
3569 let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3570 let index = match reason {
3571 crate::engine::OpaqueReason::DynamicReference => 0,
3572 crate::engine::OpaqueReason::RuntimeTextReference => 1,
3573 crate::engine::OpaqueReason::UnknownFunction => 2,
3574 crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3575 crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3576 crate::engine::OpaqueReason::UnresolvedName => 5,
3577 crate::engine::OpaqueReason::UnresolvedTable => 6,
3578 crate::engine::OpaqueReason::FormulaName => 7,
3579 crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3580 crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3581 crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3582 };
3583 1u64 << index
3584 };
3585 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3586 stats.staged_selected = report.selected_staged_cells as u64;
3587 stats.staged_retained = report.retained_staged_cells as u64;
3588 stats.target_requested = report.requested_targets as u64;
3589 stats.target_normalized_regions = report.normalized_regions as u64;
3590 stats.target_scope_level = match &report.widened_scope {
3591 crate::engine::PrepareScope::Exact => 0,
3592 crate::engine::PrepareScope::Sheets(_) => 1,
3593 crate::engine::PrepareScope::Workbook => 2,
3594 };
3595 stats.target_widening_reason_bits = report
3596 .widening_reasons
3597 .iter()
3598 .copied()
3599 .fold(0, |bits, reason| bits | reason_bit(reason));
3600 stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3601 stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3602 stats.target_commit_estimated_work = report.estimated_commit_work;
3603 stats.target_commit_actual_work = report.actual_commit_work;
3604 stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3605 stats.phases.staged_prepare_ns = stats
3606 .phases
3607 .staged_prepare_ns
3608 .saturating_add(Self::duration_ns(report.commit_window));
3609 }
3610 }
3611
3612 fn observe_topology(
3613 &mut self,
3614 outcome: FormulaPlaneTopologyCacheOutcome,
3615 strategy: FormulaPlaneTopologyStrategy,
3616 observed: &MixedTopologyCompileStats,
3617 elapsed: std::time::Duration,
3618 ) {
3619 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3620 if outcome.severity() > stats.topology.cache_outcome.severity() {
3621 stats.topology.cache_outcome = outcome;
3622 }
3623 if strategy.severity() > stats.topology.strategy.severity() {
3624 stats.topology.strategy = strategy;
3625 }
3626 match outcome {
3627 FormulaPlaneTopologyCacheOutcome::Hit => {
3628 stats.topology.cache_hit_events =
3629 stats.topology.cache_hit_events.saturating_add(1);
3630 }
3631 FormulaPlaneTopologyCacheOutcome::Built => {
3632 stats.topology.cache_build_events =
3633 stats.topology.cache_build_events.saturating_add(1);
3634 }
3635 FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3636 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3637 stats.topology.cache_build_events =
3638 stats.topology.cache_build_events.saturating_add(1);
3639 stats.topology.cache_skip_events =
3640 stats.topology.cache_skip_events.saturating_add(1);
3641 }
3642 FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3643 }
3644 if matches!(
3645 outcome,
3646 FormulaPlaneTopologyCacheOutcome::Built
3647 | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3648 | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3649 ) {
3650 stats.topology.producers_observed = stats
3651 .topology
3652 .producers_observed
3653 .saturating_add(observed.producers as u64);
3654 stats.topology.candidates_observed = stats
3655 .topology
3656 .candidates_observed
3657 .saturating_add(observed.candidates as u64);
3658 stats.topology.edges_observed = stats
3659 .topology
3660 .edges_observed
3661 .saturating_add(observed.relationships as u64);
3662 stats.topology.candidate_cap_hits = stats
3663 .topology
3664 .candidate_cap_hits
3665 .saturating_add(observed.candidate_overflow_count as u64);
3666 stats.topology.edge_cap_hits = stats
3667 .topology
3668 .edge_cap_hits
3669 .saturating_add(observed.edge_overflow_count as u64);
3670 stats.topology.byte_cap_hits = stats
3671 .topology
3672 .byte_cap_hits
3673 .saturating_add(observed.memory_overflow_count as u64);
3674 stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3675 Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3676 } else if stats.topology.edge_cap_hits > 0 {
3677 Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3678 } else if stats.topology.candidate_cap_hits > 0 {
3679 Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3680 } else {
3681 None
3682 };
3683 stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3684 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3685 } else if stats.topology.edge_cap_hits > 0 {
3686 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3687 } else if stats.topology.candidate_cap_hits > 0 {
3688 Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3689 } else {
3690 None
3691 };
3692 }
3693 stats.topology.retained_bytes_observed = stats
3694 .topology
3695 .retained_bytes_observed
3696 .max(observed.estimated_memory_bytes as u64);
3697 stats.phases.topology_ns = stats
3698 .phases
3699 .topology_ns
3700 .saturating_add(Self::duration_ns(elapsed));
3701 }
3702 }
3703
3704 fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3705 if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3706 && strategy.severity() > stats.topology.strategy.severity()
3707 {
3708 stats.topology.strategy = strategy;
3709 }
3710 }
3711
3712 fn observe_formula_plane_route(
3713 &mut self,
3714 plan: &LegacyIslandPlan,
3715 phase: FormulaPlaneRoutePhase,
3716 reason: FormulaPlaneRouteTransitionReason,
3717 demotion_generation: usize,
3718 replan_generation: usize,
3719 ) {
3720 if plan.is_empty() {
3721 return;
3722 }
3723 let mut sheet_ids = [0_u16;
3724 crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3725 let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3726 sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3727 let authority = self.graph.formula_authority();
3728 let event = FormulaPlaneRouteEvent {
3729 island_id: plan.island_id,
3730 phase,
3731 route: FormulaPlaneRoute::ContractedLegacyIsland,
3732 sheet_ids,
3733 sheet_count: sheet_count as u8,
3734 authority_epoch: authority.plane.epoch().0,
3735 index_epoch: authority.indexes_epoch(),
3736 transition_reason: reason,
3737 demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3738 replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3739 };
3740 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3741 let index = usize::from(stats.topology.route_event_count);
3742 if index < stats.topology.route_events.len() {
3743 stats.topology.route_events[index] = event;
3744 stats.topology.route_event_count =
3745 stats.topology.route_event_count.saturating_add(1);
3746 } else {
3747 stats.topology.route_events_dropped =
3748 stats.topology.route_events_dropped.saturating_add(1);
3749 }
3750 stats.topology.route_event_bytes_observed = stats
3751 .topology
3752 .route_event_bytes_observed
3753 .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3754 stats.topology.island_membership_vertices = plan.membership.len() as u64;
3755 stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3756 stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3757 stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3758 }
3759 }
3760
3761 fn observe_materialization(
3762 &mut self,
3763 placements: usize,
3764 cycle: bool,
3765 elapsed: std::time::Duration,
3766 ) {
3767 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3768 if cycle {
3769 stats.cycle_materialized_cells = stats
3770 .cycle_materialized_cells
3771 .saturating_add(placements as u64);
3772 } else {
3773 stats.fallback_materialized_cells = stats
3774 .fallback_materialized_cells
3775 .saturating_add(placements as u64);
3776 stats.topology.strategy =
3777 FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3778 }
3779 stats.phases.materialization_ns = stats
3780 .phases
3781 .materialization_ns
3782 .saturating_add(Self::duration_ns(elapsed));
3783 }
3784 }
3785
3786 fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3787 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3788 stats.dirty_lease = if empty {
3789 FormulaDirtyLeaseOutcome::Empty
3790 } else {
3791 FormulaDirtyLeaseOutcome::Acquired
3792 };
3793 }
3794 }
3795
3796 fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3797 let empty = lease.is_empty();
3798 let _ = self.graph.ack_formula_dirty(lease);
3799 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3800 stats.dirty_lease = if empty {
3801 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3802 } else {
3803 FormulaDirtyLeaseOutcome::Acknowledged
3804 };
3805 }
3806 }
3807
3808 fn checked_ack_formula_dirty_observed(
3809 &mut self,
3810 lease: FormulaDirtyLease,
3811 ) -> Result<(), ExcelError> {
3812 self.resource_checkpoint(0)?;
3813 self.ack_formula_dirty_observed(lease);
3814 Ok(())
3815 }
3816
3817 fn checked_ack_formula_dirty_sublease_observed(
3818 &mut self,
3819 lease: FormulaDirtyLease,
3820 owned_events: &[usize],
3821 ) -> Result<(), ExcelError> {
3822 self.resource_checkpoint(owned_events.len() as u64)?;
3823 let empty = lease.is_empty();
3824 let partial = owned_events.len() < lease.len();
3825 let sublease = lease.sublease(owned_events.iter().copied());
3826 if !self.graph.ack_formula_dirty_sublease(sublease) {
3827 return Err(ExcelError::new(ExcelErrorKind::NImpl)
3828 .with_message("FormulaPlane dirty sublease became stale"));
3829 }
3830 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3831 stats.dirty_lease = if empty {
3832 FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3833 } else if partial {
3834 FormulaDirtyLeaseOutcome::AcknowledgedPartial
3835 } else {
3836 FormulaDirtyLeaseOutcome::Acknowledged
3837 };
3838 }
3839 Ok(())
3840 }
3841
3842 fn begin_evaluation_request(&mut self) {
3846 #[cfg(test)]
3847 {
3848 self.evaluation_request_begin_count_for_test = self
3849 .evaluation_request_begin_count_for_test
3850 .saturating_add(1);
3851 }
3852 self.last_cycle_telemetry = CycleTelemetry::default();
3853 self.pending_iterative_redirty.clear();
3856 self.reconcile_retained_sccs_at_request_begin();
3857 self.clock.refresh();
3861 }
3862
3863 fn redirty_for_next_recalc(&mut self) {
3872 self.graph.redirty_volatiles();
3873 let pending = std::mem::take(&mut self.pending_iterative_redirty);
3874 let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3875 for (vertex, scc) in dirty_at_begin {
3876 if self.retained_scc_members.get(&vertex) == Some(&scc) {
3877 self.retained_scc_members.remove(&vertex);
3883 if !self.graph.is_dirty(vertex) {
3884 self.iterative_state_values.remove(&vertex);
3885 }
3886 }
3887 }
3888 if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3889 let graph = &self.graph;
3890 self.iterative_state_values
3891 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3892 self.retained_scc_members
3893 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3894 }
3895 for &vertex in &pending {
3900 if !self.graph.is_live_formula_vertex(vertex) {
3901 continue;
3902 }
3903 if let Some(cell) = self.graph.get_cell_ref(vertex) {
3904 let sheet_name = self.graph.sheet_name(cell.sheet_id);
3905 match self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3906 Some(value) if !matches!(value, LiteralValue::Empty) => {
3907 self.iterative_state_values.insert(vertex, value);
3908 }
3909 _ => {
3910 self.iterative_state_values.remove(&vertex);
3911 }
3912 }
3913 }
3914 }
3915 if !pending.is_empty() {
3916 self.graph.redirty_iterative_members(&pending);
3917 }
3918 }
3919
3920 fn retained_scc_config_fingerprint(&self) -> u64 {
3925 use std::hash::{Hash, Hasher};
3926 let mut hasher = rustc_hash::FxHasher::default();
3927 let config = &self.config;
3928 std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3929 match config.cycle.policy {
3930 CyclePolicy::Error => 0u8.hash(&mut hasher),
3931 CyclePolicy::Iterate {
3932 max_iterations,
3933 max_change,
3934 } => {
3935 1u8.hash(&mut hasher);
3936 max_iterations.hash(&mut hasher);
3937 max_change.to_bits().hash(&mut hasher);
3938 }
3939 }
3940 config.date_system.hash(&mut hasher);
3941 config.workbook_seed.hash(&mut hasher);
3942 std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3943 format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3946 config.range_expansion_limit.hash(&mut hasher);
3947 config.max_open_ended_rows.hash(&mut hasher);
3948 config.max_open_ended_cols.hash(&mut hasher);
3949 hasher.finish()
3950 }
3951
3952 fn reconcile_retained_sccs_at_request_begin(&mut self) {
3958 self.retained_scc_dirty_at_begin.clear();
3959 if self.retained_scc_members.is_empty() {
3960 return;
3961 }
3962 let graph = &self.graph;
3965 self.retained_scc_members
3966 .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3967 if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3968 let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3969 self.retained_scc_members.clear();
3970 self.graph.mark_dirty_many(&members);
3971 return;
3972 }
3973 let changes =
3974 crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3975 let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3976 let provider_revision = self.resolver.planning_semantic_revision();
3977 let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3978 if global_changed || provider_changed {
3979 let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3980 let affected: Vec<VertexId> = self
3981 .retained_scc_members
3982 .keys()
3983 .copied()
3984 .filter(|&vertex| {
3985 let Some(ast) = self.graph.get_formula(vertex) else {
3986 return true;
3987 };
3988 (global_changed
3989 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3990 || (provider_changed && Self::ast_contains_function(&ast))
3991 })
3992 .collect();
3993 for vertex in &affected {
3994 self.retained_scc_members.remove(vertex);
3995 }
3996 if !affected.is_empty() {
3997 self.graph.mark_dirty_many(&affected);
3998 }
3999 self.retained_scc_function_epoch_seen = changes.epoch;
4000 self.retained_scc_provider_revision_seen = provider_revision;
4001 }
4002 let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
4003 let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
4004 for (&vertex, &scc) in &self.retained_scc_members {
4005 all_sccs.insert(scc);
4006 if self.graph.is_dirty(vertex) {
4007 dirty_sccs.insert(scc);
4008 self.retained_scc_dirty_at_begin.push((vertex, scc));
4009 }
4010 }
4011 let reused_members = self
4012 .retained_scc_members
4013 .values()
4014 .filter(|scc| !dirty_sccs.contains(scc))
4015 .count();
4016 let t = &mut self.last_cycle_telemetry;
4017 t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4018 t.reused_scc_members = reused_members;
4019 }
4020
4021 pub fn virtual_dep_fallback_activations(&self) -> u64 {
4022 self.virtual_dep_fallback_activations
4023 }
4024
4025 pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4026 self.lookup_index_cache.report()
4027 }
4028
4029 fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4030 let start_row = view.start_row();
4031 let end_row = view.end_row();
4032 let start_col = view.start_col();
4033 let end_col = view.end_col();
4034 for row in start_row..=end_row {
4035 let Ok(row_u32) = u32::try_from(row) else {
4036 return true;
4037 };
4038 for col in start_col..=end_col {
4039 let Ok(col_u32) = u32::try_from(col) else {
4040 return true;
4041 };
4042 let cell_ref = self
4043 .graph
4044 .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4045 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4046 && self.graph.is_volatile(*vertex_id)
4047 {
4048 return true;
4049 }
4050 }
4051 }
4052 false
4053 }
4054
4055 fn build_lookup_index_impl(
4056 &self,
4057 view: &RangeView<'_>,
4058 axis: LookupAxis,
4059 ) -> Option<Arc<LookupIndex>> {
4060 let (rows, cols) = view.dims();
4061 if rows == 0 || cols == 0 {
4062 self.lookup_index_cache.note_skipped_tiny();
4063 return None;
4064 }
4065 let len = match axis {
4066 LookupAxis::ColumnInView(col) => {
4067 if col >= cols {
4068 self.lookup_index_cache.note_skipped_tiny();
4069 return None;
4070 }
4071 rows
4072 }
4073 LookupAxis::RowInView(row) => {
4074 if row >= rows {
4075 self.lookup_index_cache.note_skipped_tiny();
4076 return None;
4077 }
4078 cols
4079 }
4080 };
4081 if len < 64 {
4082 self.lookup_index_cache.note_skipped_tiny();
4083 return None;
4084 }
4085
4086 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4087 let key = LookupIndexKey {
4088 sheet_id,
4089 start_row: u32::try_from(view.start_row()).ok()?,
4090 start_col: u32::try_from(view.start_col()).ok()?,
4091 end_row: u32::try_from(view.end_row()).ok()?,
4092 end_col: u32::try_from(view.end_col()).ok()?,
4093 axis,
4094 snapshot_id: self.data_snapshot_id(),
4095 };
4096 if let Some(index) = self.lookup_index_cache.get(&key) {
4097 return Some(index);
4098 }
4099 if self
4100 .lookup_index_cache
4101 .would_exceed_cap(estimate_bytes(len, 0))
4102 {
4103 self.lookup_index_cache.note_skipped_cap();
4104 return None;
4105 }
4106 if !self.lookup_index_cache.should_build(key) {
4107 return None;
4108 }
4109 if self.lookup_index_cache.is_known_volatile(&key) {
4110 self.lookup_index_cache.note_skipped_volatile();
4111 return None;
4112 }
4113 if self.lookup_view_contains_volatile(view, sheet_id) {
4114 self.lookup_index_cache.note_volatile_key(key);
4115 self.lookup_index_cache.note_skipped_volatile();
4116 return None;
4117 }
4118 match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4119 BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4120 BuildOutcome::ErrorInLookupAxis => {
4121 self.lookup_index_cache.note_skipped_error();
4122 None
4123 }
4124 BuildOutcome::Degenerate => {
4125 self.lookup_index_cache.note_skipped_tiny();
4126 None
4127 }
4128 }
4129 }
4130
4131 fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4132 if !self.config.enable_virtual_dep_telemetry {
4133 self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4134 fallback_mode_activations: self.virtual_dep_fallback_activations,
4135 ..VirtualDepTelemetry::default()
4136 };
4137 }
4138 }
4139
4140 fn source_cache_session(&self) -> SourceCacheSession {
4141 self.clear_source_cache();
4142 SourceCacheSession {
4143 cache: self.source_cache.clone(),
4144 }
4145 }
4146
4147 fn resolve_source_scalar_cached(
4148 &self,
4149 name: &str,
4150 version: Option<u64>,
4151 ) -> Result<LiteralValue, ExcelError> {
4152 let key = (name.to_string(), version);
4153 if let Ok(mut g) = self.source_cache.write() {
4154 if let Some(v) = g.scalars.get(&key) {
4155 return Ok(v.clone());
4156 }
4157
4158 let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4159 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4160 ExcelError::new(ExcelErrorKind::Ref)
4161 .with_message(format!("Unresolved source scalar: {name}"))
4162 } else {
4163 err
4164 }
4165 })?;
4166 g.scalars.insert(key, v.clone());
4167 Ok(v)
4168 } else {
4169 self.resolver.resolve_source_scalar(name).map_err(|err| {
4170 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4171 ExcelError::new(ExcelErrorKind::Ref)
4172 .with_message(format!("Unresolved source scalar: {name}"))
4173 } else {
4174 err
4175 }
4176 })
4177 }
4178 }
4179
4180 fn resolve_source_table_cached(
4181 &self,
4182 name: &str,
4183 version: Option<u64>,
4184 ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4185 let key = (name.to_string(), version);
4186 if let Ok(mut g) = self.source_cache.write() {
4187 if let Some(t) = g.tables.get(&key) {
4188 return Ok(t.clone());
4189 }
4190
4191 let t = self.resolver.resolve_source_table(name).map_err(|err| {
4192 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4193 ExcelError::new(ExcelErrorKind::Ref)
4194 .with_message(format!("Unresolved source table: {name}"))
4195 } else {
4196 err
4197 }
4198 })?;
4199 let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4200 g.tables.insert(key, t.clone());
4201 Ok(t)
4202 } else {
4203 self.resolver
4204 .resolve_source_table(name)
4205 .map_err(|err| {
4206 if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4207 ExcelError::new(ExcelErrorKind::Ref)
4208 .with_message(format!("Unresolved source table: {name}"))
4209 } else {
4210 err
4211 }
4212 })
4213 .map(Arc::from)
4214 }
4215 }
4216
4217 fn source_table_to_range_view(
4218 &self,
4219 table: &dyn crate::traits::Table,
4220 spec: &Option<formualizer_parse::parser::TableSpecifier>,
4221 ) -> Result<RangeView<'static>, ExcelError> {
4222 use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4223
4224 let owned = match spec {
4225 Some(TableSpecifier::Column(c)) => {
4226 let c = c.trim();
4227 if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4228 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4229 "Complex structured references not yet supported".to_string(),
4230 ));
4231 }
4232 table.get_column(c)?.materialise().into_owned()
4233 }
4234 Some(TableSpecifier::ColumnRange(start, end)) => {
4235 let cols = table.columns();
4236 let start = start.trim();
4237 let end = end.trim();
4238 let start_key = start.to_lowercase();
4239 let end_key = end.to_lowercase();
4240 let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4241 let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4242 if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4243 if si > ei {
4244 std::mem::swap(&mut si, &mut ei);
4245 }
4246 let h = table.data_height();
4247 let w = ei - si + 1;
4248 let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4249 for (offset, ci) in (si..=ei).enumerate() {
4250 let cname = &cols[ci];
4251 let col_range = table.get_column(cname)?;
4252 let (rh, _) = col_range.dimensions();
4253 for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4254 row[offset] = col_range.get(r, 0)?;
4255 }
4256 }
4257 rows
4258 } else {
4259 return Err(ExcelError::new(ExcelErrorKind::Ref)
4260 .with_message("Column range refers to unknown column(s)".to_string()));
4261 }
4262 }
4263 Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4264 | Some(TableSpecifier::Headers) => table
4265 .headers_row()
4266 .map(|r| r.materialise().into_owned())
4267 .unwrap_or_default(),
4268 Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4269 | Some(TableSpecifier::Totals) => table
4270 .totals_row()
4271 .map(|r| r.materialise().into_owned())
4272 .unwrap_or_default(),
4273 Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4274 table
4275 .data_body()
4276 .map(|r| r.materialise().into_owned())
4277 .unwrap_or_default()
4278 }
4279 Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4280 let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4281 if let Some(h) = table.headers_row() {
4282 out.extend(h.iter_rows());
4283 }
4284 if let Some(body) = table.data_body() {
4285 out.extend(body.iter_rows());
4286 }
4287 if let Some(tr) = table.totals_row() {
4288 out.extend(tr.iter_rows());
4289 }
4290 out
4291 }
4292 Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4293 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4294 "@ (This Row) requires table-aware context; not yet supported".to_string(),
4295 ));
4296 }
4297 Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4298 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4299 .with_message("Complex structured references not yet supported".to_string()));
4300 }
4301 None => {
4302 return Err(ExcelError::new(ExcelErrorKind::NImpl)
4303 .with_message("Table reference without specifier is unsupported".to_string()));
4304 }
4305 };
4306
4307 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4308 }
4309
4310 pub fn default_sheet_id(&self) -> SheetId {
4311 self.graph.default_sheet_id()
4312 }
4313
4314 pub fn default_sheet_name(&self) -> &str {
4315 self.graph.default_sheet_name()
4316 }
4317
4318 pub fn set_workbook_seed(&mut self, seed: u64) {
4320 self.config.workbook_seed = seed;
4321 }
4322
4323 pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4325 self.config.volatile_level = level;
4326 }
4327
4328 pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4330 self.config.temporal_egress = policy;
4331 }
4332
4333 pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4334 self.config.temporal_egress
4335 }
4336
4337 pub fn set_deterministic_mode(
4339 &mut self,
4340 mode: crate::engine::DeterministicMode,
4341 ) -> Result<(), ExcelError> {
4342 let clock = mode.build_clock()?;
4343 self.config.deterministic_mode = mode;
4344 self.clock = crate::timezone::SnapshotClock::new(clock);
4345 Ok(())
4346 }
4347
4348 pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4356 self.clock = crate::timezone::SnapshotClock::new(clock);
4357 }
4358
4359 fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4360 self.config.deterministic_mode.validate()
4361 }
4362
4363 pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4364 self.graph.sheet_id(name)
4365 }
4366
4367 pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4368 self.add_sheet(name)
4369 .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4370 }
4371
4372 pub fn sheet_name(&self, id: SheetId) -> &str {
4373 self.graph.sheet_name(id)
4374 }
4375
4376 pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4377 let id = self.graph.add_sheet(name)?;
4378 self.ensure_arrow_sheet(name);
4379 self.mark_topology_edited();
4381 Ok(id)
4382 }
4383
4384 pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4385 let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4386 ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4387 })?;
4388 if new_name.is_empty() || new_name.len() > 255 {
4389 return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4390 }
4391 if self.graph.sheet_id(new_name).is_some() {
4392 return Err(ExcelError::new(ExcelErrorKind::Value)
4393 .with_message(format!("Sheet '{new_name}' already exists")));
4394 }
4395 self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
4398 .map_err(Self::editor_error_to_excel)?;
4399 let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4400
4401 if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4402 let mut copied_sheet = source_sheet;
4403 copied_sheet.name = Arc::<str>::from(new_name);
4404 self.arrow_sheets.sheets.push(copied_sheet);
4405 } else {
4406 self.ensure_arrow_sheet(new_name);
4407 }
4408
4409 let duplicated_formulas = self
4410 .graph
4411 .formula_vertices()
4412 .into_iter()
4413 .filter(|vertex| {
4414 self.graph
4415 .get_cell_ref(*vertex)
4416 .is_some_and(|cell| cell.sheet_id == new_id)
4417 })
4418 .collect::<Vec<_>>();
4419 self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4420 self.mark_topology_edited();
4421 Ok(new_id)
4422 }
4423
4424 fn ensure_arrow_sheet(&mut self, name: &str) {
4425 if self.arrow_sheets.sheet(name).is_some() {
4426 return;
4427 }
4428 self.arrow_sheets
4429 .sheets
4430 .push(crate::arrow_store::ArrowSheet {
4431 name: std::sync::Arc::<str>::from(name),
4432 date_system: self.config.date_system,
4433 columns: Vec::new(),
4434 nrows: 0,
4435 chunk_starts: Vec::new(),
4436 chunk_rows: 32 * 1024,
4437 });
4438 }
4439
4440 pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4441 let name = self.graph.sheet_name(sheet_id).to_string();
4442 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
4446 .map_err(Self::editor_error_to_excel)?;
4447 self.purge_derived_formats_for_sheet(sheet_id);
4448 self.graph.remove_sheet(sheet_id)?;
4449 self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4450 self.clear_all_computed_overlays();
4454 self.mark_all_formula_vertices_dirty();
4455 self.clear_staged_formulas_for_sheet(&name);
4456 if self.row_visibility.remove(&sheet_id).is_some() {
4457 self.invalidate_row_visibility_mask_cache();
4458 }
4459 self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
4460 self.mark_topology_edited();
4461 Ok(())
4462 }
4463
4464 fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4466 if let Some(asheet) = self
4467 .arrow_sheets
4468 .sheets
4469 .iter_mut()
4470 .find(|s| s.name.as_ref() == target_name)
4471 {
4472 asheet.name = std::sync::Arc::<str>::from(new_name);
4473 return true;
4474 }
4475 false
4476 }
4477
4478 pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4479 let old_name = self.graph.sheet_name(sheet_id).to_string();
4480
4481 self.rename_sheet_in_arrow_store(&old_name, new_name);
4484
4485 match self.graph.rename_sheet(sheet_id, new_name) {
4487 Ok(_) => {
4488 self.rename_staged_formula_sheet(&old_name, new_name);
4489 let sheet_vertices: Vec<VertexId> = self
4491 .graph
4492 .grid_vertices_in_sheet(sheet_id)
4493 .map(|(id, _)| id)
4494 .collect();
4495 for v_id in sheet_vertices {
4496 self.graph.mark_vertex_dirty(v_id);
4497 }
4498 self.mark_topology_edited();
4500 Ok(())
4501 }
4502 Err(e) => {
4503 self.rename_sheet_in_arrow_store(new_name, &old_name);
4505 Err(e)
4506 }
4507 }
4508 }
4509
4510 pub fn named_ranges_iter(
4511 &self,
4512 ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4513 self.graph.named_ranges_iter()
4514 }
4515
4516 pub fn sheet_named_ranges_iter(
4517 &self,
4518 ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4519 self.graph.sheet_named_ranges_iter()
4520 }
4521
4522 pub fn resolve_name_entry(
4523 &self,
4524 name: &str,
4525 current_sheet: SheetId,
4526 ) -> Option<&crate::engine::named_range::NamedRange> {
4527 self.graph.resolve_name_entry(name, current_sheet)
4528 }
4529
4530 pub(crate) fn name_query_scope(
4540 &self,
4541 scope_sheet: Option<&str>,
4542 ) -> Result<NameScope, ExcelError> {
4543 match scope_sheet {
4544 None => Ok(NameScope::Workbook),
4545 Some(sheet) => self
4546 .graph
4547 .sheet_id(sheet)
4548 .map(NameScope::Sheet)
4549 .ok_or_else(|| {
4550 ExcelError::new(ExcelErrorKind::Ref)
4551 .with_message(format!("name scope sheet not found: {sheet}"))
4552 }),
4553 }
4554 }
4555
4556 fn resolve_sheet_locator(
4569 &self,
4570 locator: &crate::reference::SharedSheetLocator<'_>,
4571 context_sheet: SheetId,
4572 ) -> Result<SheetId, ExcelError> {
4573 self.graph
4574 .sheet_reg()
4575 .resolve_locator(locator, context_sheet)
4576 }
4577
4578 pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4579 let Ok(scope) = self.name_query_scope(scope_sheet) else {
4580 return false;
4581 };
4582 self.graph
4583 .resolve_name_entry_in_scope(name, scope)
4584 .is_some()
4585 }
4586
4587 pub fn resolved_name_value(
4591 &self,
4592 name: &str,
4593 scope_sheet: Option<&str>,
4594 ) -> Option<LiteralValue> {
4595 let scope = self.name_query_scope(scope_sheet).ok()?;
4596 let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4597 self.graph.get_value(entry.vertex)
4598 }
4599
4600 pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4601 let entry = self.graph.resolve_table_entry(name)?;
4602 Some(TableMetadata {
4603 name: entry.name.clone(),
4604 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4605 start_row: entry.range.start.coord.row() + 1,
4606 start_col: entry.range.start.coord.col() + 1,
4607 end_row: entry.range.end.coord.row() + 1,
4608 end_col: entry.range.end.coord.col() + 1,
4609 header_row: entry.header_row,
4610 headers: entry.headers.clone(),
4611 totals_row: entry.totals_row,
4612 })
4613 }
4614
4615 pub fn tables(&self) -> Vec<TableMetadata> {
4617 self.graph
4618 .table_names()
4619 .into_iter()
4620 .filter_map(|name| self.table_metadata(&name))
4621 .collect()
4622 }
4623
4624 pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4625 let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4626
4627 for (name, named) in self.graph.named_ranges_iter() {
4628 out.push(crate::engine::named_range::NamedRangeSnapshot {
4629 name: name.clone(),
4630 scope: NameScope::Workbook,
4631 definition: named.definition.clone(),
4632 });
4633 }
4634
4635 for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4636 out.push(crate::engine::named_range::NamedRangeSnapshot {
4637 name: name.clone(),
4638 scope: NameScope::Sheet(*sheet_id),
4639 definition: named.definition.clone(),
4640 });
4641 }
4642
4643 out.sort_by(|a, b| {
4644 let a_scope = match a.scope {
4645 NameScope::Workbook => (0u8, 0u32),
4646 NameScope::Sheet(id) => (1u8, u32::from(id)),
4647 };
4648 let b_scope = match b.scope {
4649 NameScope::Workbook => (0u8, 0u32),
4650 NameScope::Sheet(id) => (1u8, u32::from(id)),
4651 };
4652 a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4653 });
4654
4655 out
4656 }
4657
4658 pub fn named_ranges_snapshot_for_sheet(
4659 &self,
4660 sheet_id: SheetId,
4661 ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4662 self.named_ranges_snapshot()
4663 .into_iter()
4664 .filter(|entry| match entry.scope {
4665 NameScope::Workbook => true,
4666 NameScope::Sheet(id) => id == sheet_id,
4667 })
4668 .collect()
4669 }
4670
4671 pub fn define_name(
4672 &mut self,
4673 name: &str,
4674 definition: NamedDefinition,
4675 scope: NameScope,
4676 ) -> Result<(), ExcelError> {
4677 self.graph.validate_define_name(name, scope)?;
4683 let prepared = self
4684 .prepare_name_dependent_span_demotion([name])
4685 .map_err(Self::editor_error_to_excel)?;
4686 let demoted = self
4687 .commit_name_dependent_span_demotion(prepared)
4688 .map_err(Self::editor_error_to_excel)?;
4689 self.graph.define_name(name, definition, scope)?;
4690 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4691 if !demoted {
4692 self.mark_topology_edited();
4693 }
4694 Ok(())
4695 }
4696
4697 pub fn update_name(
4698 &mut self,
4699 name: &str,
4700 definition: NamedDefinition,
4701 scope: NameScope,
4702 ) -> Result<(), ExcelError> {
4703 self.graph.validate_existing_name(name, scope)?;
4709 let prepared = self
4710 .prepare_name_dependent_span_demotion([name])
4711 .map_err(Self::editor_error_to_excel)?;
4712 let demoted = self
4713 .commit_name_dependent_span_demotion(prepared)
4714 .map_err(Self::editor_error_to_excel)?;
4715 self.graph.update_name(name, definition, scope)?;
4716 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4717 if !demoted {
4718 self.mark_topology_edited();
4719 }
4720 Ok(())
4721 }
4722
4723 pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4724 self.graph.validate_existing_name(name, scope)?;
4728 let prepared = self
4729 .prepare_name_dependent_span_demotion([name])
4730 .map_err(Self::editor_error_to_excel)?;
4731 let demoted = self
4732 .commit_name_dependent_span_demotion(prepared)
4733 .map_err(Self::editor_error_to_excel)?;
4734 self.graph.delete_name(name, scope)?;
4735 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4736 if !demoted {
4737 self.mark_topology_edited();
4738 }
4739 Ok(())
4740 }
4741
4742 fn exact_name_dependent_span_refs<'a>(
4743 &self,
4744 names: impl IntoIterator<Item = &'a str>,
4745 ) -> Vec<FormulaSpanRef> {
4746 let authority = self.graph.formula_authority();
4747 let mut refs = names
4748 .into_iter()
4749 .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4750 .collect::<Vec<_>>();
4751 refs.sort_unstable_by_key(|span_ref| {
4752 (span_ref.id.0, span_ref.generation, span_ref.version)
4753 });
4754 refs.dedup();
4755 refs
4756 }
4757
4758 fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4759 events
4760 .iter()
4761 .filter_map(|event| match event {
4762 ChangeEvent::DefineName { name, .. }
4763 | ChangeEvent::UpdateName { name, .. }
4764 | ChangeEvent::DeleteName { name, .. }
4765 | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4766 _ => None,
4767 })
4768 .collect()
4769 }
4770
4771 fn prepare_name_dependent_span_demotion<'a>(
4772 &mut self,
4773 names: impl IntoIterator<Item = &'a str>,
4774 ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4775 {
4776 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4777 return Ok(None);
4778 }
4779 let refs = self.exact_name_dependent_span_refs(names);
4780 if refs.is_empty() {
4781 return Ok(None);
4782 }
4783
4784 let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4788 let clean_cells = {
4789 let authority = self.graph.formula_authority();
4790 refs.iter()
4791 .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4792 .flat_map(|span| span.domain.iter())
4793 .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4794 .map(|coord| {
4795 CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4796 })
4797 .collect::<Vec<_>>()
4798 };
4799 self.prepare_formula_span_demotion(&refs)
4800 .map(|prepared| Some((prepared, clean_cells)))
4801 .map_err(|error| match error {
4802 FormulaSpanDemotionError::Resource(error) => error.into(),
4803 error => crate::engine::EditorError::TransactionFailed {
4804 reason: format!(
4805 "FormulaPlane name-dependent demotion preparation failed: {error}"
4806 ),
4807 },
4808 })
4809 }
4810
4811 fn commit_name_dependent_span_demotion(
4812 &mut self,
4813 prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4814 ) -> Result<bool, crate::engine::EditorError> {
4815 let Some((prepared, clean_cells)) = prepared else {
4816 return Ok(false);
4817 };
4818 self.commit_prepared_formula_span_demotion(prepared)
4819 .map_err(|error| crate::engine::EditorError::TransactionFailed {
4820 reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4821 })?;
4822 for cell in clean_cells {
4823 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4824 self.graph.set_dirty(vertex_id, false);
4825 }
4826 }
4827 Ok(true)
4828 }
4829
4830 pub fn define_table(
4831 &mut self,
4832 name: &str,
4833 range: crate::reference::RangeRef,
4834 header_row: bool,
4835 headers: Vec<String>,
4836 totals_row: bool,
4837 ) -> Result<(), ExcelError> {
4838 self.graph
4839 .define_table(name, range, header_row, headers, totals_row)?;
4840 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4841 self.mark_topology_edited();
4842 Ok(())
4843 }
4844
4845 pub fn define_source_scalar(
4846 &mut self,
4847 name: &str,
4848 version: Option<u64>,
4849 ) -> Result<(), ExcelError> {
4850 self.graph.define_source_scalar(name, version)?;
4851 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4852 self.mark_topology_edited();
4853 Ok(())
4854 }
4855
4856 pub fn define_source_table(
4857 &mut self,
4858 name: &str,
4859 version: Option<u64>,
4860 ) -> Result<(), ExcelError> {
4861 self.graph.define_source_table(name, version)?;
4862 self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4863 self.mark_topology_edited();
4864 Ok(())
4865 }
4866
4867 pub fn set_source_scalar_version(
4868 &mut self,
4869 name: &str,
4870 version: Option<u64>,
4871 ) -> Result<(), ExcelError> {
4872 self.graph.set_source_scalar_version(name, version)?;
4873 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4874 self.graph
4875 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4876 }
4877 Ok(())
4878 }
4879
4880 pub fn set_source_table_version(
4881 &mut self,
4882 name: &str,
4883 version: Option<u64>,
4884 ) -> Result<(), ExcelError> {
4885 self.graph.set_source_table_version(name, version)?;
4886 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4887 self.graph
4888 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4889 }
4890 Ok(())
4891 }
4892
4893 pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4894 self.graph.invalidate_source(name)?;
4895 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4896 self.graph
4897 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4898 }
4899 Ok(())
4900 }
4901
4902 pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4903 self.graph.get_value(vertex)
4904 }
4905
4906 pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4907 self.graph.get_cell_value(sheet, row, col)
4908 }
4909
4910 pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4911 self.graph.get_vertex_for_cell(cell)
4912 }
4913
4914 pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4915 self.graph.get_evaluation_vertices()
4916 }
4917
4918 pub fn baseline_stats(&self) -> EngineBaselineStats {
4920 let graph = self.graph.baseline_stats();
4921 let formula_authority = self.graph.formula_authority();
4922 let formula_dirty = self.graph.formula_dirty_stats();
4923 EngineBaselineStats {
4924 graph_vertex_count: graph.graph_vertex_count,
4925 graph_formula_vertex_count: graph.graph_formula_vertex_count,
4926 graph_edge_count: graph.graph_edge_count,
4927 dirty_vertex_count: graph.dirty_vertex_count,
4928 evaluation_vertex_count: graph.evaluation_vertex_count,
4929 formula_ast_root_count: graph.formula_ast_root_count,
4930 formula_ast_node_count: graph.formula_ast_node_count,
4931 staged_formula_count: self.staged_formula_count(),
4932 formula_plane_active_span_count: formula_authority.active_span_count(),
4933 formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4934 formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4935 formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4936 formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4937 formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4938 formula_plane_dirty_pending_events: formula_dirty.pending_events,
4939 formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4940 formula_plane_dirty_span_region_events_recorded: formula_dirty
4941 .span_region_events_recorded,
4942 formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4943 formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4944 formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4945 formula_plane_cycle_member_span_demotions: self
4946 .formula_plane_cycle_member_span_demotions,
4947 formula_plane_array_result_span_demotions: self
4948 .formula_plane_array_result_span_demotions,
4949 retained_scc_members: self.retained_scc_members.len(),
4950 }
4951 }
4952
4953 pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4955 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4956 }
4957
4958 #[cfg(test)]
4959 pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4960 self.used_axis_bounds_cache
4961 .read()
4962 .ok()
4963 .and_then(|guard| {
4964 guard.as_ref().map(|cache| {
4965 (
4966 cache.row_hits.load(Ordering::Relaxed),
4967 cache.row_misses.load(Ordering::Relaxed),
4968 cache.col_hits.load(Ordering::Relaxed),
4969 cache.col_misses.load(Ordering::Relaxed),
4970 )
4971 })
4972 })
4973 .unwrap_or((0, 0, 0, 0))
4974 }
4975
4976 pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4977 self.graph.set_first_load_assume_new(enabled);
4978 }
4979
4980 pub fn first_load_assume_new(&self) -> bool {
4981 self.graph.first_load_assume_new()
4982 }
4983
4984 pub fn reset_ensure_touched(&mut self) {
4985 self.graph.reset_ensure_touched();
4986 }
4987
4988 pub fn finalize_sheet_index(&mut self, sheet: &str) {
4989 self.graph.finalize_sheet_index(sheet);
4990 }
4991
4992 pub fn action<T>(
5001 &mut self,
5002 name: impl AsRef<str>,
5003 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5004 ) -> Result<T, crate::engine::EditorError> {
5005 if self.action_depth != 0 {
5006 return Err(crate::engine::EditorError::TransactionFailed {
5007 reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
5008 .to_string(),
5009 });
5010 }
5011
5012 self.action_depth = 1;
5013 let engine_ptr: *mut Engine<R> = self;
5014 let _guard = ActionDepthGuard {
5015 engine: engine_ptr,
5016 _marker: std::marker::PhantomData,
5017 };
5018
5019 let mut tx = EngineAction {
5020 engine: self,
5021 name: name.as_ref().to_string(),
5022 capture: None,
5023 arrow_undo: None,
5024 atomic_policy: false,
5025 };
5026 f(&mut tx)
5027 }
5028
5029 pub fn action_atomic<T>(
5033 &mut self,
5034 name: impl Into<String>,
5035 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5036 ) -> Result<T, crate::engine::EditorError> {
5037 let (v, _j) = self.action_atomic_journal(name, f)?;
5038 Ok(v)
5039 }
5040
5041 pub fn action_atomic_journal<T>(
5043 &mut self,
5044 name: impl Into<String>,
5045 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5046 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
5047 if self.action_depth != 0 {
5048 return Err(crate::engine::EditorError::TransactionFailed {
5049 reason: "Nested Engine::action calls are not supported (deterministic rule)"
5050 .to_string(),
5051 });
5052 }
5053
5054 self.action_depth = 1;
5055 let engine_ptr: *mut Engine<R> = self;
5056 let _guard = ActionDepthGuard {
5057 engine: engine_ptr,
5058 _marker: std::marker::PhantomData,
5059 };
5060
5061 let name_str = name.into();
5062 let mut capture = MutationCapture::new(Default::default());
5063 let start_len = capture.len();
5064 self.action_atomic_impl(&mut capture, start_len, name_str, f)
5065 }
5066
5067 fn action_atomic_impl<T>(
5068 &mut self,
5069 capture: &mut MutationCapture,
5070 start_len: usize,
5071 name: String,
5072 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5073 ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
5074 let invalidation_baseline = self.invalidation_baseline();
5075 let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
5076 let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
5077
5078 let capture_ptr: *mut MutationCapture = capture;
5079 let mut tx = EngineAction {
5080 engine: self,
5081 name: name.clone(),
5082 capture: Some(capture_ptr),
5083 arrow_undo: Some(arrow_ptr),
5084 atomic_policy: true,
5085 };
5086
5087 let res = f(&mut tx);
5088
5089 let graph_events: Vec<crate::engine::ChangeEvent> =
5091 unsafe { (&*capture_ptr).events() }[start_len..].to_vec();
5092 let graph_batch = crate::engine::GraphUndoBatch {
5093 events: graph_events,
5094 };
5095 let affected_cells = arrow_undo.ops.len();
5096 let journal = crate::engine::ActionJournal {
5097 name,
5098 graph: graph_batch,
5099 arrow: arrow_undo,
5100 affected_cells,
5101 };
5102
5103 match res {
5104 Ok(v) => {
5105 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5106 for event in &journal.graph.events {
5107 self.record_formula_plane_change_for_event(event);
5108 }
5109 self.invalidate_for_action_journal(
5110 &journal,
5111 LoggedEditDirection::Original,
5112 invalidation_baseline,
5113 );
5114 }
5115 Ok((v, journal))
5116 }
5117 Err(e) => {
5118 if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5119 {
5120 return Err(crate::engine::EditorError::TransactionFailed {
5121 reason: format!(
5122 "Engine::action_atomic rollback failed after error '{e}': {rb}"
5123 ),
5124 });
5125 }
5126 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5127 for event in &journal.graph.events {
5128 self.record_formula_plane_change_for_event(event);
5129 }
5130 }
5131 Err(e)
5132 }
5133 }
5134 }
5135
5136 pub fn action_with_logger<T>(
5143 &mut self,
5144 log: &mut crate::engine::ChangeLog,
5145 name: impl AsRef<str>,
5146 f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5147 ) -> Result<T, crate::engine::EditorError> {
5148 if self.action_depth != 0 {
5149 return Err(crate::engine::EditorError::TransactionFailed {
5150 reason: "Nested Engine::action calls are not supported (deterministic rule)"
5151 .to_string(),
5152 });
5153 }
5154
5155 self.action_depth = 1;
5156 let engine_ptr: *mut Engine<R> = self;
5157 let _guard = ActionDepthGuard {
5158 engine: engine_ptr,
5159 _marker: std::marker::PhantomData,
5160 };
5161
5162 let name_str = name.as_ref().to_string();
5163 let mut capture = MutationCapture::new(log.current_meta());
5164 let start_len = capture.len();
5165 capture.begin_compound(name_str.clone());
5166
5167 let res = self.action_atomic_impl(&mut capture, start_len, name_str, f);
5170 capture.close_compounds();
5171
5172 match res {
5173 Ok((v, _journal)) => {
5174 log.publish_capture(capture);
5175 Ok(v)
5176 }
5177 Err(e) => {
5178 log.discard_capture(capture);
5180 Err(e)
5181 }
5182 }
5183 }
5184
5185 fn rollback_from_action_journal(
5186 &mut self,
5187 journal: &crate::engine::ActionJournal,
5188 invalidation_baseline: InvalidationBaseline,
5189 ) -> Result<(), crate::engine::EditorError> {
5190 self.invalidate_for_action_journal(
5193 journal,
5194 LoggedEditDirection::InverseReplay,
5195 invalidation_baseline,
5196 );
5197 journal.graph.undo(&mut self.graph)?;
5199 self.apply_inverse_row_visibility_events(&journal.graph.events);
5201 self.apply_arrow_undo_batch(&journal.arrow, true);
5203 Ok(())
5204 }
5205
5206 fn rollback_from_change_events(
5207 &mut self,
5208 events: &[crate::engine::ChangeEvent],
5209 invalidation_baseline: InvalidationBaseline,
5210 ) -> Result<(), crate::engine::EditorError> {
5211 use crate::engine::ChangeEvent;
5212
5213 self.invalidate_for_change_events(
5216 events,
5217 LoggedEditDirection::InverseReplay,
5218 invalidation_baseline,
5219 );
5220
5221 {
5223 let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5224 let mut compound_stack: Vec<usize> = Vec::new();
5225 for ev in events.iter().rev() {
5226 match ev {
5227 ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
5228 ChangeEvent::CompoundStart { depth, .. } => {
5229 if compound_stack.last() == Some(depth) {
5230 compound_stack.pop();
5231 }
5232 }
5233 ChangeEvent::SetRowVisibility { .. } => {
5234 }
5236 _ => {
5237 editor.apply_inverse(ev.clone())?;
5238 }
5239 }
5240 }
5241 }
5242
5243 for ev in events.iter().rev() {
5245 self.apply_inverse_row_visibility_event(ev);
5246 }
5247
5248 for ev in events.iter().rev() {
5250 self.mirror_inverse_change_to_arrow(ev);
5251 }
5252
5253 Ok(())
5254 }
5255
5256 fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5257 let sheet_id = self.graph.sheet_id(sheet)?;
5258 let coord = Coord::from_excel(row, col, true, true);
5259 let cell = CellRef::new(sheet_id, coord);
5260 let vid = self.graph.get_vertex_for_cell(&cell)?;
5261 let ast_id = self.graph.get_formula_id(vid)?;
5262 self.graph
5263 .data_store()
5264 .retrieve_ast(ast_id, self.graph.sheet_reg())
5265 }
5266
5267 pub fn define_name_with_logger(
5268 &mut self,
5269 log: &mut crate::engine::ChangeLog,
5270 name: &str,
5271 definition: NamedDefinition,
5272 scope: NameScope,
5273 ) -> Result<(), crate::engine::EditorError> {
5274 self.graph
5275 .validate_define_name(name, scope)
5276 .map_err(crate::engine::EditorError::Excel)?;
5277 let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
5278 if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
5279 return Err(crate::engine::EditorError::TransactionUnsupported {
5280 reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
5281 .to_string(),
5282 });
5283 }
5284 let prepared = self.prepare_name_dependent_span_demotion([name])?;
5285 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5286 {
5287 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5288 editor.define_name(name, definition, scope)?;
5289 }
5290 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5291 if !demoted {
5292 self.mark_topology_edited();
5293 }
5294 Ok(())
5295 }
5296
5297 pub fn update_name_with_logger(
5298 &mut self,
5299 log: &mut crate::engine::ChangeLog,
5300 name: &str,
5301 definition: NamedDefinition,
5302 scope: NameScope,
5303 ) -> Result<(), crate::engine::EditorError> {
5304 self.graph
5305 .validate_existing_name(name, scope)
5306 .map_err(crate::engine::EditorError::Excel)?;
5307 let prepared = self.prepare_name_dependent_span_demotion([name])?;
5308 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5309 {
5310 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5311 editor.update_name(name, definition, scope)?;
5312 }
5313 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5314 if !demoted {
5315 self.mark_topology_edited();
5316 }
5317 Ok(())
5318 }
5319
5320 pub fn delete_name_with_logger(
5321 &mut self,
5322 log: &mut crate::engine::ChangeLog,
5323 name: &str,
5324 scope: NameScope,
5325 ) -> Result<(), crate::engine::EditorError> {
5326 self.graph
5327 .validate_existing_name(name, scope)
5328 .map_err(crate::engine::EditorError::Excel)?;
5329 let prepared = self.prepare_name_dependent_span_demotion([name])?;
5330 let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5331 {
5332 let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5333 editor.delete_name(name, scope)?;
5334 }
5335 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5336 if !demoted {
5337 self.mark_topology_edited();
5338 }
5339 Ok(())
5340 }
5341
5342 pub fn edit_with_logger<T>(
5343 &mut self,
5344 log: &mut crate::engine::ChangeLog,
5345 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5346 ) -> Result<T, crate::engine::EditorError> {
5347 let mut capture = MutationCapture::new(log.current_meta());
5348 let result = self.edit_with_capture(&mut capture, f);
5349 capture.close_compounds();
5350 match result {
5351 Ok(value) => {
5352 log.publish_capture(capture);
5353 Ok(value)
5354 }
5355 Err(error) => {
5356 log.discard_capture(capture);
5357 Err(error)
5358 }
5359 }
5360 }
5361
5362 fn edit_with_capture<T>(
5363 &mut self,
5364 capture: &mut MutationCapture,
5365 f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5366 ) -> Result<T, crate::engine::EditorError> {
5367 let invalidation_baseline = self.invalidation_baseline();
5368 let start_len = capture.len();
5369
5370 struct ArrowSpillReader<'a> {
5373 sheets: &'a crate::arrow_store::SheetStore,
5374 }
5375 impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5376 fn read_cell_value(
5377 &self,
5378 sheet: &str,
5379 row: u32,
5380 col: u32,
5381 ) -> Option<formualizer_common::LiteralValue> {
5382 use formualizer_common::LiteralValue;
5383 let asheet = self.sheets.sheet(sheet)?;
5384 let r0 = row.saturating_sub(1) as usize;
5385 let c0 = col.saturating_sub(1) as usize;
5386 let v = asheet.get_cell_value(r0, c0);
5387 if matches!(v, LiteralValue::Empty) {
5388 None
5389 } else {
5390 Some(v)
5391 }
5392 }
5393 }
5394
5395 let ret = {
5396 let spill_reader = ArrowSpillReader {
5397 sheets: &self.arrow_sheets,
5398 };
5399 let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
5400 &mut self.graph,
5401 capture,
5402 &spill_reader,
5403 );
5404 f(&mut editor)
5405 };
5406
5407 let new_events = capture.events()[start_len..].to_vec();
5408 if new_events.iter().any(|event| {
5409 matches!(
5410 event,
5411 ChangeEvent::DefineName { .. }
5412 | ChangeEvent::UpdateName { .. }
5413 | ChangeEvent::DeleteName { .. }
5414 )
5415 }) {
5416 self.rollback_from_change_events(&new_events, invalidation_baseline)?;
5417 return Err(crate::engine::EditorError::TransactionUnsupported {
5418 reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5419 });
5420 }
5421
5422 self.clear_logged_cell_format_states(&new_events);
5425 for ev in &new_events {
5426 self.mirror_forward_change_to_arrow(ev);
5427 }
5428 for ev in &new_events {
5429 self.record_formula_plane_change_for_event(ev);
5430 }
5431
5432 if self.action_depth == 0 {
5435 self.invalidate_for_change_events(
5436 &new_events,
5437 LoggedEditDirection::Original,
5438 invalidation_baseline,
5439 );
5440 }
5441
5442 Ok(ret)
5443 }
5444
5445 pub(crate) fn preflight_replay_admission(
5446 &mut self,
5447 events: &[ChangeEvent],
5448 forward: bool,
5449 ) -> Result<(), crate::engine::EditorError> {
5450 if !self.graph_admission_enabled() {
5451 return Ok(());
5452 }
5453 let mut vertex_delta = 0i128;
5454 let mut edge_delta = 0i128;
5455 let mut added_vertices = 0usize;
5456 let mut added_edges = 0usize;
5457 let mut formula_cells = BTreeSet::new();
5458 for event in events {
5459 match event {
5460 ChangeEvent::AddVertex {
5461 formula,
5462 coord,
5463 sheet_id,
5464 ..
5465 } => {
5466 let delta = if forward { 1 } else { -1 };
5467 vertex_delta += delta;
5468 if forward {
5469 added_vertices = added_vertices.saturating_add(1);
5470 if formula.is_some() {
5471 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5472 }
5473 }
5474 }
5475 ChangeEvent::RemoveVertex {
5476 old_formula,
5477 coord,
5478 sheet_id,
5479 ..
5480 } => {
5481 let delta = if forward { -1 } else { 1 };
5482 vertex_delta += delta;
5483 if !forward {
5484 added_vertices = added_vertices.saturating_add(1);
5485 if old_formula.is_some()
5486 && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5487 {
5488 formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5489 }
5490 }
5491 }
5492 ChangeEvent::EdgeAdded { .. } => {
5493 let delta = if forward { 1 } else { -1 };
5494 edge_delta += delta;
5495 if forward {
5496 added_edges = added_edges.saturating_add(1);
5497 }
5498 }
5499 ChangeEvent::EdgeRemoved { .. } => {
5500 let delta = if forward { -1 } else { 1 };
5501 edge_delta += delta;
5502 if !forward {
5503 added_edges = added_edges.saturating_add(1);
5504 }
5505 }
5506 ChangeEvent::SetFormula {
5507 addr, old_formula, ..
5508 } => {
5509 if forward || old_formula.is_some() {
5510 formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5511 }
5512 }
5513 _ => {}
5514 }
5515 }
5516 let stats = self.graph.baseline_stats();
5517 let final_vertices = i128::try_from(stats.graph_vertex_count)
5518 .ok()
5519 .and_then(|count| count.checked_add(vertex_delta))
5520 .and_then(|count| usize::try_from(count).ok())
5521 .ok_or_else(|| {
5522 crate::engine::EditorError::Excel(
5523 ExcelError::new(ExcelErrorKind::NImpl)
5524 .with_message("replay vertex count overflow"),
5525 )
5526 })?;
5527 let final_edges = i128::try_from(stats.graph_edge_count)
5528 .ok()
5529 .and_then(|count| count.checked_add(edge_delta))
5530 .and_then(|count| usize::try_from(count).ok())
5531 .ok_or_else(|| {
5532 crate::engine::EditorError::Excel(
5533 ExcelError::new(ExcelErrorKind::NImpl)
5534 .with_message("replay edge count overflow"),
5535 )
5536 })?;
5537 self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5538 final_vertices,
5539 final_edges,
5540 materialization_cells: formula_cells.len() as u64,
5541 added_vertices,
5542 added_edges,
5543 })
5544 .map_err(crate::engine::EditorError::Excel)
5545 }
5546
5547 pub fn undo_logged(
5552 &mut self,
5553 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5554 log: &mut crate::engine::ChangeLog,
5555 ) -> Result<(), crate::engine::EditorError> {
5556 let pending_events = log
5557 .last_group_indices()
5558 .into_iter()
5559 .map(|index| log.events()[index].clone())
5560 .collect::<Vec<_>>();
5561 self.preflight_replay_admission(&pending_events, false)?;
5562 let invalidation_baseline = self.invalidation_baseline();
5563 let names = Self::name_event_names(&pending_events);
5564 let prepared =
5565 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5566 self.commit_name_dependent_span_demotion(prepared)?;
5567 self.invalidate_for_change_events(
5570 &pending_events,
5571 LoggedEditDirection::InverseReplay,
5572 invalidation_baseline,
5573 );
5574 let batch = undo.undo(&mut self.graph, log)?;
5575 for item in batch.iter().rev() {
5576 self.apply_inverse_row_visibility_event(&item.event);
5577 self.apply_inverse_staged_formula_event(&item.event);
5578 }
5579 if !batch.is_empty() {
5580 let events = batch
5581 .iter()
5582 .map(|item| item.event.clone())
5583 .collect::<Vec<_>>();
5584 self.clear_logged_cell_format_states(&events);
5585 }
5586 self.mirror_undo_batch_to_arrow(&batch);
5587 if !batch.is_empty() {
5588 for item in &batch {
5589 self.record_formula_plane_change_for_event(&item.event);
5590 }
5591 }
5592 Ok(())
5593 }
5594
5595 pub fn redo_logged(
5596 &mut self,
5597 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5598 log: &mut crate::engine::ChangeLog,
5599 ) -> Result<(), crate::engine::EditorError> {
5600 let pending_events = undo.pending_redo_events();
5601 self.preflight_replay_admission(&pending_events, true)?;
5602 let invalidation_baseline = self.invalidation_baseline();
5603 let names = Self::name_event_names(&pending_events);
5604 let prepared =
5605 self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5606 self.commit_name_dependent_span_demotion(prepared)?;
5607 self.invalidate_for_change_events(
5608 &pending_events,
5609 LoggedEditDirection::ForwardReplay,
5610 invalidation_baseline,
5611 );
5612 let batch = undo.redo(&mut self.graph, log)?;
5613 for item in &batch {
5614 self.apply_forward_row_visibility_event(&item.event);
5615 self.apply_forward_staged_formula_event(&item.event);
5616 }
5617 if !batch.is_empty() {
5618 let events = batch
5619 .iter()
5620 .map(|item| item.event.clone())
5621 .collect::<Vec<_>>();
5622 self.clear_logged_cell_format_states(&events);
5623 }
5624 self.mirror_redo_batch_to_arrow(&batch);
5625 if !batch.is_empty() {
5626 for item in &batch {
5627 self.record_formula_plane_change_for_event(&item.event);
5628 }
5629 }
5630 Ok(())
5631 }
5632
5633 pub fn undo_action(
5637 &mut self,
5638 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5639 ) -> Result<(), crate::engine::EditorError> {
5640 let Some(journal) = undo.pop_undo_action() else {
5641 return Ok(());
5642 };
5643 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5644 undo.push_done_action(journal);
5645 return Err(error);
5646 }
5647 let invalidation_baseline = self.invalidation_baseline();
5648 let names = Self::name_event_names(&journal.graph.events);
5649 let prepared =
5650 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5651 Ok(prepared) => prepared,
5652 Err(error) => {
5653 undo.push_done_action(journal);
5654 return Err(error);
5655 }
5656 };
5657 if let Err(error) = self.commit_name_dependent_span_demotion(prepared) {
5658 undo.push_done_action(journal);
5659 return Err(error);
5660 }
5661
5662 self.invalidate_for_action_journal(
5663 &journal,
5664 LoggedEditDirection::InverseReplay,
5665 invalidation_baseline,
5666 );
5667 journal.graph.undo(&mut self.graph)?;
5668 self.apply_inverse_row_visibility_events(&journal.graph.events);
5669 self.apply_arrow_undo_batch(&journal.arrow, true);
5670 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5671 for event in &journal.graph.events {
5672 self.record_formula_plane_change_for_event(event);
5673 }
5674 }
5675
5676 undo.push_redo_action(journal);
5677 Ok(())
5678 }
5679
5680 pub fn redo_action(
5684 &mut self,
5685 undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5686 ) -> Result<(), crate::engine::EditorError> {
5687 let Some(journal) = undo.pop_redo_action() else {
5688 return Ok(());
5689 };
5690 if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5691 undo.push_redo_action(journal);
5692 return Err(error);
5693 }
5694 let invalidation_baseline = self.invalidation_baseline();
5695 let names = Self::name_event_names(&journal.graph.events);
5696 let prepared =
5697 match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5698 Ok(prepared) => prepared,
5699 Err(error) => {
5700 undo.push_redo_action(journal);
5701 return Err(error);
5702 }
5703 };
5704 if let Err(error) = self.commit_name_dependent_span_demotion(prepared) {
5705 undo.push_redo_action(journal);
5706 return Err(error);
5707 }
5708
5709 self.invalidate_for_action_journal(
5710 &journal,
5711 LoggedEditDirection::ForwardReplay,
5712 invalidation_baseline,
5713 );
5714 journal.graph.redo(&mut self.graph)?;
5715 self.apply_forward_row_visibility_events(&journal.graph.events);
5716 self.apply_arrow_undo_batch(&journal.arrow, false);
5717 if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5718 for event in &journal.graph.events {
5719 self.record_formula_plane_change_for_event(event);
5720 }
5721 }
5722
5723 undo.push_done_action(journal);
5724 Ok(())
5725 }
5726
5727 fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5728 let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5729 let row = addr.coord.row() + 1;
5731 let col = addr.coord.col() + 1;
5732 (sheet, row, col)
5733 }
5734
5735 fn mirror_undo_batch_to_arrow(
5736 &mut self,
5737 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5738 ) {
5739 for item in batch.iter().rev() {
5741 self.mirror_inverse_change_to_arrow(&item.event);
5742 }
5743 }
5744
5745 fn mirror_redo_batch_to_arrow(
5746 &mut self,
5747 batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5748 ) {
5749 for item in batch.iter() {
5751 self.mirror_forward_change_to_arrow(&item.event);
5752 }
5753 }
5754
5755 fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5756 use crate::engine::ChangeEvent;
5757 use formualizer_common::LiteralValue;
5758
5759 match ev {
5760 ChangeEvent::SetValue {
5761 addr,
5762 old_value,
5763 old_formula,
5764 ..
5765 } => {
5766 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5767 if old_formula.is_some() {
5768 self.clear_delta_overlay_cell(&sheet, row, col);
5769 } else {
5770 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5771 self.mirror_value_to_overlay(&sheet, row, col, &v);
5772 }
5773 }
5774 ChangeEvent::SetFormula {
5775 addr,
5776 old_value,
5777 old_formula,
5778 ..
5779 } => {
5780 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5781 if old_formula.is_some() {
5782 self.clear_delta_overlay_cell(&sheet, row, col);
5783 } else {
5784 let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5785 self.mirror_value_to_overlay(&sheet, row, col, &v);
5786 }
5787 }
5788 ChangeEvent::SpillCommitted { old, new, .. } => {
5789 self.mirror_spill_snapshot(new, true);
5791 if let Some(snap) = old {
5792 self.mirror_spill_snapshot(snap, false);
5793 }
5794 }
5795 ChangeEvent::SpillCleared { old, .. } => {
5796 self.mirror_spill_snapshot(old, false);
5798 }
5799 ChangeEvent::SetRowVisibility { .. } => {
5800 }
5802 _ => {}
5803 }
5804 }
5805
5806 fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5807 use crate::engine::ChangeEvent;
5808
5809 match ev {
5810 ChangeEvent::SetValue { addr, new, .. } => {
5811 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5812 self.mirror_value_to_overlay(&sheet, row, col, new);
5813 }
5814 ChangeEvent::SetFormula { addr, .. } => {
5815 let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5816 self.clear_delta_overlay_cell(&sheet, row, col);
5817 }
5819 ChangeEvent::SpillCommitted { old, new, .. } => {
5820 if let Some(snap) = old {
5821 self.mirror_spill_snapshot(snap, true);
5822 }
5823 self.mirror_spill_snapshot(new, false);
5824 }
5825 ChangeEvent::SpillCleared { old, .. } => {
5826 self.mirror_spill_snapshot(old, true);
5827 }
5828 ChangeEvent::SetRowVisibility { .. } => {
5829 }
5831 _ => {
5832 }
5834 }
5835 }
5836
5837 fn mirror_spill_snapshot(
5838 &mut self,
5839 snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5840 clear_only: bool,
5841 ) {
5842 use formualizer_common::LiteralValue;
5843
5844 let mut i = 0usize;
5845 for row in &snap.values {
5846 for v in row {
5847 if let Some(cell) = snap.target_cells.get(i) {
5848 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5849 let out = if clear_only {
5850 LiteralValue::Empty
5851 } else {
5852 v.clone()
5853 };
5854 self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5855 }
5856 i += 1;
5857 }
5858 }
5859 if clear_only {
5861 for cell in snap.target_cells.iter().skip(i) {
5862 let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5863 self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5864 }
5865 }
5866 }
5867
5868 pub fn set_default_sheet_by_name(&mut self, name: &str) {
5869 self.graph.set_default_sheet_by_name(name);
5870 }
5871
5872 pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5873 self.graph.set_default_sheet_by_id(id);
5874 }
5875
5876 pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5877 self.graph.set_sheet_index_mode(mode);
5878 }
5879
5880 #[cfg(feature = "test-support")]
5881 #[doc(hidden)]
5882 pub fn mark_all_formulas_dirty_for_test(&mut self) {
5883 self.mark_all_formula_vertices_dirty();
5884 self.graph
5885 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
5886 }
5887
5888 #[cfg(feature = "test-support")]
5889 #[doc(hidden)]
5890 pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5891 criteria_mask_test_hooks::take_mask_work()
5892 }
5893
5894 #[cfg(feature = "test-support")]
5895 #[doc(hidden)]
5896 pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5897 self.lookup_index_cache.report()
5898 }
5899
5900 #[cfg(any(test, feature = "benchmark_internal"))]
5901 #[doc(hidden)]
5902 pub fn reset_recalc_reuse_probe(&mut self) {
5903 *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5904 }
5905
5906 #[cfg(any(test, feature = "benchmark_internal"))]
5907 #[doc(hidden)]
5908 pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5909 let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5910 if let Some(cached) = self.cached_static_schedule.as_ref() {
5911 probe.schedule_retained_bytes = std::mem::size_of::<CachedScheduleEntry>()
5912 + cached.candidate_vertices.capacity() * std::mem::size_of::<VertexId>()
5913 + std::mem::size_of::<crate::engine::Schedule>()
5914 + 2 * std::mem::size_of::<usize>()
5915 + schedule_probe_retained_bytes(&cached.schedule);
5916 }
5917 probe
5918 }
5919
5920 #[cfg(test)]
5921 pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5922 self.cached_static_schedule
5923 .as_ref()
5924 .map(|cached| Arc::clone(&cached.schedule))
5925 }
5926
5927 fn clear_cached_static_schedule(&mut self) {
5928 self.cached_static_schedule = None;
5929 }
5930
5931 fn invalidation_baseline(&self) -> InvalidationBaseline {
5932 InvalidationBaseline {
5933 snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5934 topology_epoch: self.topology_epoch,
5935 }
5936 }
5937
5938 fn classify_change_events(
5939 events: &[crate::engine::ChangeEvent],
5940 direction: LoggedEditDirection,
5941 ) -> LoggedEditImpact {
5942 use crate::engine::ChangeEvent;
5943
5944 events
5945 .iter()
5946 .map(|event| match event {
5947 ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5948 LoggedEditImpact::NoOp
5949 }
5950 ChangeEvent::SetRowVisibility { .. }
5951 | ChangeEvent::SetValue {
5952 old_formula: None,
5953 old_value: Some(_),
5954 ..
5955 } => LoggedEditImpact::DataOnly,
5956 ChangeEvent::SetValue {
5960 old_formula: None,
5961 old_value: None,
5962 ..
5963 } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5964 ChangeEvent::SetValue { .. }
5965 | ChangeEvent::SetFormula { .. }
5966 | ChangeEvent::AddVertex { .. }
5967 | ChangeEvent::RemoveVertex { .. }
5968 | ChangeEvent::VertexMoved { .. }
5969 | ChangeEvent::FormulaAdjusted { .. }
5970 | ChangeEvent::NamedRangeAdjusted { .. }
5971 | ChangeEvent::EdgeAdded { .. }
5972 | ChangeEvent::EdgeRemoved { .. }
5973 | ChangeEvent::DefineName { .. }
5974 | ChangeEvent::UpdateName { .. }
5975 | ChangeEvent::DeleteName { .. }
5976 | ChangeEvent::SpillCommitted { .. }
5977 | ChangeEvent::SpillCleared { .. }
5978 | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5979 })
5980 .max()
5981 .unwrap_or(LoggedEditImpact::NoOp)
5982 }
5983
5984 fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5985 use crate::engine::ArrowOp;
5986
5987 arrow
5988 .ops
5989 .iter()
5990 .map(|op| match op {
5991 ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5992 LoggedEditImpact::DataOnly
5993 }
5994 ArrowOp::RestoreComputedRect { .. }
5995 | ArrowOp::InsertRows { .. }
5996 | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5997 })
5998 .max()
5999 .unwrap_or(LoggedEditImpact::NoOp)
6000 }
6001
6002 fn apply_logged_edit_impact(
6003 &mut self,
6004 impact: LoggedEditImpact,
6005 baseline: InvalidationBaseline,
6006 ) {
6007 match impact {
6008 LoggedEditImpact::NoOp => {}
6009 LoggedEditImpact::DataOnly => {
6010 if self.topology_epoch == baseline.topology_epoch
6011 && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
6012 == baseline.snapshot_id
6013 {
6014 self.mark_data_edited();
6015 }
6016 }
6017 LoggedEditImpact::Topology => {
6018 if self.topology_epoch == baseline.topology_epoch {
6021 self.mark_topology_edited();
6022 }
6023 }
6024 }
6025 }
6026
6027 fn invalidate_for_change_events(
6028 &mut self,
6029 events: &[crate::engine::ChangeEvent],
6030 direction: LoggedEditDirection,
6031 baseline: InvalidationBaseline,
6032 ) {
6033 self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6034 }
6035
6036 fn invalidate_for_action_journal(
6037 &mut self,
6038 journal: &crate::engine::ActionJournal,
6039 direction: LoggedEditDirection,
6040 baseline: InvalidationBaseline,
6041 ) {
6042 let impact = Self::classify_change_events(&journal.graph.events, direction)
6043 .max(Self::classify_arrow_undo(&journal.arrow));
6044 self.apply_logged_edit_impact(impact, baseline);
6045 }
6046
6047 pub fn mark_data_edited(&mut self) {
6050 self.lookup_index_cache.clear();
6051 self.snapshot_id
6052 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6053 self.has_edited = true;
6054 }
6055
6056 pub fn mark_topology_edited(&mut self) {
6058 self.lookup_index_cache.clear();
6059 self.snapshot_id
6060 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6061 self.topology_epoch = self.topology_epoch.wrapping_add(1);
6062 self.graph.bump_topology_revision();
6063 self.clear_cached_static_schedule();
6064 self.cached_mixed_topology = None;
6065 if let Some(ledger) = self.active_resource_ledger.as_mut() {
6066 let released = ledger.account_mixed_cache(0);
6067 debug_assert!(released.is_ok());
6068 }
6069 self.has_edited = true;
6070 }
6071
6072 fn mark_all_formula_vertices_dirty(&mut self) {
6073 let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6074 for vertex in vertices {
6075 self.graph.mark_vertex_dirty(vertex);
6076 }
6077 }
6078
6079 fn mark_moved_formula_vertices_dirty(
6080 &mut self,
6081 summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6082 ) {
6083 for vertex in &summary.vertices_moved {
6084 if self.graph.get_formula_id(*vertex).is_some() {
6085 self.graph.mark_vertex_dirty(*vertex);
6086 }
6087 }
6088 }
6089
6090 pub fn sheet_store(&self) -> &SheetStore {
6092 &self.arrow_sheets
6093 }
6094
6095 pub(crate) fn has_row_visibility_state(&self) -> bool {
6098 !self.row_visibility.is_empty()
6099 }
6100
6101 pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6103 &mut self.arrow_sheets
6104 }
6105
6106 pub fn has_staged_formulas(&self) -> bool {
6107 !self.staged_formulas.is_empty()
6108 }
6109
6110 pub fn staged_formula_count(&self) -> usize {
6111 self.staged_formulas.values().map(StagedSheet::len).sum()
6112 }
6113
6114 pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6116 self.staged_formulas
6117 .entry(sheet.to_string())
6118 .or_default()
6119 .stage(row, col, text);
6120 self.staged_formula_index.stage(sheet, row, col);
6121 }
6122
6123 fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6124 for (row, col, _) in &staged.entries {
6125 self.staged_formula_index.remove(sheet, *row, *col);
6126 }
6127 if staged.deferred_package.is_some() {
6128 self.staged_formula_index.set_package(sheet, None);
6129 }
6130 }
6131
6132 fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6133 self.staged_formulas.insert(sheet, staged);
6134 }
6135
6136 #[doc(hidden)]
6137 pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6138 SourceFormulaIngress { engine: self }
6139 }
6140
6141 #[doc(hidden)]
6142 #[doc(hidden)]
6145 #[cfg(any(test, feature = "test-support"))]
6146 pub fn set_before_prepared_span_commit_hook(
6147 &mut self,
6148 hook: impl FnOnce() + Send + Sync + 'static,
6149 ) {
6150 self.before_prepared_span_commit_hook = Some(Box::new(hook));
6151 }
6152
6153 #[doc(hidden)]
6154 #[cfg(test)]
6156 pub(crate) fn set_before_target_preparation_commit_hook(
6157 &mut self,
6158 hook: impl FnOnce() + Send + Sync + 'static,
6159 ) {
6160 self.before_target_preparation_commit_hook = Some(Box::new(hook));
6161 }
6162
6163 fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6164 let sheet = package.sheet_name.clone();
6165 let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6166 debug_assert!(staged.deferred_package.is_none());
6167 staged.deferred_package = Some(package);
6168 staged.reconcile_attached_deferred_package();
6169 self.staged_formula_index
6170 .set_package(&sheet, staged.deferred_package.as_ref());
6171 }
6172
6173 pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6174 let mut removed = None;
6175 let mut remove_sheet = false;
6176 let mut had_package = false;
6177 if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6178 had_package = entries.deferred_package.is_some();
6179 removed = entries.remove(row, col);
6180 remove_sheet = entries.is_empty();
6181 }
6182 let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6183 if !ordinary_removed && had_package {
6184 self.staged_formula_index.touch_package(sheet);
6185 }
6186 if remove_sheet {
6187 self.staged_formulas.remove(sheet);
6188 self.staged_formula_index.set_package(sheet, None);
6189 }
6190 removed
6191 }
6192
6193 pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6194 if self.staged_formulas.remove(sheet).is_some() {
6195 self.staged_formula_index.clear_sheet(sheet);
6196 }
6197 }
6198
6199 pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6200 let Some(entries) = self.staged_formulas.remove(old) else {
6201 return;
6202 };
6203 self.staged_formula_index.clear_sheet(old);
6204 let (formulas, mut package) = entries.into_parts();
6205 for (row, col, text) in formulas {
6206 self.stage_formula_text(new, row, col, text);
6207 }
6208 if let Some(package) = package.as_mut() {
6209 package.sheet_name = new.to_string();
6210 }
6211 if let Some(package) = package {
6212 self.stage_deferred_formula_package(package);
6213 }
6214 }
6215
6216 pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6218 self.staged_formulas
6219 .get(sheet)
6220 .and_then(|v| v.get(row, col))
6221 }
6222
6223 pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6224 &self.formula_parse_diagnostics
6225 }
6226
6227 pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6228 std::mem::take(&mut self.formula_parse_diagnostics)
6229 }
6230
6231 pub fn clear_formula_parse_diagnostics(&mut self) {
6232 self.formula_parse_diagnostics.clear();
6233 }
6234
6235 pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6236 self.last_formula_ingest_report.as_ref()
6237 }
6238
6239 pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6240 &self.formula_ingest_report_total
6241 }
6242
6243 #[cfg(test)]
6244 pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6245 &mut self,
6246 hook: impl FnOnce() + Send + Sync + 'static,
6247 ) {
6248 self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6249 }
6250
6251 #[cfg(test)]
6252 pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6253 self.inject_target_semantic_stale_once_for_test = true;
6254 }
6255
6256 #[cfg(test)]
6257 pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6258 self.force_virtual_dep_changes_remaining_for_test = rounds;
6259 }
6260
6261 #[cfg(test)]
6262 pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6263 self.fail_evaluation_commit_preflight_once_for_test = true;
6264 }
6265
6266 #[cfg(test)]
6267 pub(crate) fn cancel_before_formula_plane_layer_commit_once_for_test(&mut self) {
6268 self.cancel_before_formula_plane_layer_commit_once_for_test = true;
6269 }
6270
6271 #[cfg(test)]
6272 pub(crate) fn set_target_preparation_fault_for_test(
6273 &mut self,
6274 fault: crate::engine::target_preparation::TargetPreparationFault,
6275 ) {
6276 self.target_preparation_fault_for_test = Some(fault);
6277 }
6278
6279 #[cfg(test)]
6280 pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6281 self.staged_formula_index.revision()
6282 }
6283
6284 #[cfg(test)]
6285 pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6286 let ordinary_storage = self
6287 .staged_formulas
6288 .values()
6289 .map(|sheet| sheet.entries.len())
6290 .sum::<usize>();
6291 let package_storage = self
6292 .staged_formulas
6293 .values()
6294 .filter(|sheet| sheet.deferred_package.is_some())
6295 .count();
6296 ordinary_storage == self.staged_formula_index.ordinary_count()
6297 && package_storage == self.staged_formula_index.package_count()
6298 && self.staged_formulas.iter().all(|(name, sheet)| {
6299 sheet.entries.iter().all(|(row, col, _)| {
6300 let leases = self
6301 .staged_formula_index
6302 .leases_in_region(name, *row, *col, *row, *col);
6303 leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6304 })
6305 })
6306 }
6307
6308 #[cfg(test)]
6309 pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
6310 self.last_formula_plane_span_eval_report.as_ref()
6311 }
6312
6313 #[cfg(test)]
6314 pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6315 self.evaluation_request_begin_count_for_test
6316 }
6317
6318 #[cfg(test)]
6319 pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
6320 self.force_source_family_fallback = force;
6321 }
6322
6323 #[cfg(test)]
6324 pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
6325 self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
6326 }
6327
6328 #[cfg(test)]
6329 pub(crate) fn set_fragmented_commit_fault_for_test(
6330 &mut self,
6331 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
6332 ) {
6333 self.fragmented_commit_fault_for_test = Some(fault);
6334 }
6335
6336 #[cfg(test)]
6337 pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6338 &mut self,
6339 hook: impl FnOnce() + Send + Sync + 'static,
6340 ) {
6341 self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6342 }
6343
6344 #[cfg(test)]
6345 pub(crate) fn set_after_eager_proposal_commit_hook(
6346 &mut self,
6347 hook: impl FnOnce() + Send + Sync + 'static,
6348 ) {
6349 self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6350 }
6351
6352 #[cfg(test)]
6353 pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
6354 self.graph.formula_authority().indexes_epoch()
6355 }
6356
6357 #[cfg(test)]
6358 pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
6359 self.formula_plane_capacity_bailouts
6360 }
6361
6362 #[cfg(test)]
6363 pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6364 self.topology_epoch
6365 }
6366
6367 #[cfg(test)]
6368 pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6369 self.graph.topology_revision()
6370 }
6371
6372 #[cfg(test)]
6373 pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
6374 self.cached_mixed_topology.is_some()
6375 }
6376
6377 fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6378 self.formula_ingest_report_total.mode = report.mode;
6379 self.formula_ingest_report_total.accumulate(&report);
6380 self.last_formula_ingest_report = Some(report);
6381 }
6382
6383 fn prepare_source_formula_family(
6384 &mut self,
6385 sheet_id: SheetId,
6386 family: &crate::engine::SourceFormulaFamily,
6387 allow_function_closure: bool,
6388 planning_provider: Option<&dyn crate::traits::FunctionProvider>,
6389 ) -> Result<
6390 (
6391 crate::formula_plane::placement::PreparedAnchorOncePlacement,
6392 bool,
6393 ),
6394 SourceFamilyPreparationError,
6395 > {
6396 let transport = family
6397 .validated_complete_domain(&self.workbook_load_limits)
6398 .map_err(SourceFamilyPreparationError::contract)?;
6399 let domain = match transport {
6400 crate::engine::PlacementDomainTransport::RowRun {
6401 row_start,
6402 row_end,
6403 col,
6404 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
6405 crate::engine::PlacementDomainTransport::ColRun {
6406 row,
6407 col_start,
6408 col_end,
6409 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
6410 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
6411 sheet_id,
6412 rect.start.row,
6413 rect.end.row,
6414 rect.start.col,
6415 rect.end.col,
6416 ),
6417 };
6418 let formula = if family.anchor_text.starts_with('=') {
6419 family.anchor_text.to_string()
6420 } else {
6421 format!("={}", family.anchor_text)
6422 };
6423 let ast = formualizer_parse::parser::parse(&formula)
6424 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6425 let relocation = if allow_function_closure {
6426 validate_anchor_once_shadow_relocation(
6427 &ast,
6428 family.anchor_coord0.row,
6429 family.anchor_coord0.col,
6430 &domain,
6431 planning_provider.unwrap_or(&self.resolver),
6432 )
6433 } else {
6434 validate_anchor_once_syntax(
6435 &ast,
6436 family.anchor_coord0.row,
6437 family.anchor_coord0.col,
6438 &domain,
6439 )
6440 };
6441 relocation.map_err(SourceFamilyPreparationError::ast)?;
6442 let ast_id = self.intern_formula_ast(&ast);
6443 let placement = CellRef::new(
6444 sheet_id,
6445 Coord::from_excel(
6446 family.anchor_coord0.row + 1,
6447 family.anchor_coord0.col + 1,
6448 true,
6449 true,
6450 ),
6451 );
6452 let ingested = {
6453 let mut pipeline = match planning_provider {
6454 Some(provider) => self.graph.ingest_pipeline(provider),
6455 None => self.ingest_pipeline(),
6456 };
6457 if allow_function_closure {
6458 pipeline = pipeline.enable_function_semantics();
6459 }
6460 pipeline.ingest_formula(
6461 FormulaAstInput::RawArena(ast_id),
6462 placement,
6463 Some(family.anchor_text.clone()),
6464 )
6465 }
6466 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6467 let function_semantics_used = ingested
6468 .labels
6469 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6470 let candidate = FormulaPlacementCandidate::new(
6471 sheet_id,
6472 family.anchor_coord0.row,
6473 family.anchor_coord0.col,
6474 ingested.ast_id,
6475 Some(family.anchor_text.clone()),
6476 );
6477 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6478 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6479 prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
6480 .map(|prepared| (prepared, function_semantics_used))
6481 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
6482 }
6483
6484 fn placement_domain_from_transport(
6485 sheet_id: SheetId,
6486 transport: crate::engine::PlacementDomainTransport,
6487 ) -> PlacementDomain {
6488 match transport {
6489 crate::engine::PlacementDomainTransport::RowRun {
6490 row_start,
6491 row_end,
6492 col,
6493 } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
6494 crate::engine::PlacementDomainTransport::ColRun {
6495 row,
6496 col_start,
6497 col_end,
6498 } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
6499 crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
6500 sheet_id,
6501 rect.start.row,
6502 rect.end.row,
6503 rect.start.col,
6504 rect.end.col,
6505 ),
6506 }
6507 }
6508
6509 #[cfg(test)]
6510 pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
6511 &mut self,
6512 sheet_name: &str,
6513 source_id: crate::engine::SourceFamilyId,
6514 expected: crate::engine::PartitionLegacyMember,
6515 replay: crate::engine::DeferredReplayFormula,
6516 ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
6517 {
6518 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6519 self.graph
6520 .analyze_fragmented_exact_replay_record(
6521 &self.resolver,
6522 sheet_id,
6523 source_id,
6524 expected,
6525 replay,
6526 )
6527 .map_err(|error| format!("{error:?}"))
6528 }
6529
6530 pub(crate) fn prepare_fragmented_source_transaction(
6531 &mut self,
6532 source: &crate::engine::PartitionedSourceFormulaFamily,
6533 disposition: &crate::engine::FormulaReplayDisposition,
6534 prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
6535 legacy_formulas: Vec<
6536 crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
6537 >,
6538 ) -> Result<
6539 crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6540 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
6541 > {
6542 let transaction = self.graph.prepare_fragmented_source_transaction(
6543 &self.source_formula_token,
6544 source,
6545 disposition,
6546 prepared,
6547 legacy_formulas,
6548 )?;
6549 self.prepared_legacy_admission(
6550 transaction.legacy_graph(),
6551 transaction.materialization_cells(),
6552 )
6553 .map_err(
6554 crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
6555 )?;
6556 Ok(transaction)
6557 }
6558
6559 pub(crate) fn commit_fragmented_source_transaction(
6560 &mut self,
6561 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6562 disposition: &crate::engine::FormulaReplayDisposition,
6563 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6564 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6565 let provider_revision = self.resolver.planning_semantic_revision();
6566 let epoch = registry_guard.epoch();
6567 let resolver = &self.resolver;
6568 let decision = self.graph.commit_fragmented_source_transaction(
6569 prepared,
6570 &self.source_formula_token,
6571 disposition,
6572 epoch,
6573 provider_revision,
6574 || resolver.planning_semantic_revision(),
6575 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6576 );
6577 drop(registry_guard);
6578 decision
6579 }
6580
6581 #[cfg(test)]
6582 pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
6583 &mut self,
6584 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6585 disposition: &crate::engine::FormulaReplayDisposition,
6586 fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
6587 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6588 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6589 let provider_revision = self.resolver.planning_semantic_revision();
6590 let epoch = registry_guard.epoch();
6591 let resolver = &self.resolver;
6592 let decision = self.graph.commit_fragmented_source_transaction(
6593 prepared,
6594 &self.source_formula_token,
6595 disposition,
6596 epoch,
6597 provider_revision,
6598 || resolver.planning_semantic_revision(),
6599 fault,
6600 );
6601 drop(registry_guard);
6602 decision
6603 }
6604
6605 #[cfg(test)]
6606 pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
6607 &mut self,
6608 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6609 disposition: &crate::engine::FormulaReplayDisposition,
6610 before_second_sample: impl FnOnce(),
6611 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6612 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6613 let provider_revision = self.resolver.planning_semantic_revision();
6614 let epoch = registry_guard.epoch();
6615 let resolver = &self.resolver;
6616 let decision = self.graph.commit_fragmented_source_transaction(
6617 prepared,
6618 &self.source_formula_token,
6619 disposition,
6620 epoch,
6621 provider_revision,
6622 || {
6623 before_second_sample();
6624 resolver.planning_semantic_revision()
6625 },
6626 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6627 );
6628 drop(registry_guard);
6629 decision
6630 }
6631
6632 #[cfg(test)]
6633 pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
6634 &mut self,
6635 prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6636 disposition: &crate::engine::FormulaReplayDisposition,
6637 function_semantic_epoch: u64,
6638 function_provider_revision: Option<u64>,
6639 ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6640 self.graph.commit_fragmented_source_transaction(
6641 prepared,
6642 &self.source_formula_token,
6643 disposition,
6644 function_semantic_epoch,
6645 function_provider_revision,
6646 || function_provider_revision,
6647 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6648 )
6649 }
6650
6651 pub(crate) fn analyze_partitioned_source_family_for_transaction(
6652 &mut self,
6653 sheet_name: &str,
6654 family: &crate::engine::PartitionedSourceFormulaFamily,
6655 ) -> Result<
6656 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
6657 SourceFamilyPreparationError,
6658 > {
6659 family
6660 .validate(&self.workbook_load_limits)
6661 .map_err(SourceFamilyPreparationError::contract)?;
6662 let formula = if family.template_text.starts_with('=') {
6663 family.template_text.to_string()
6664 } else {
6665 format!("={}", family.template_text)
6666 };
6667 let ast = formualizer_parse::parser::parse(&formula)
6668 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6669 let mut requests = Vec::new();
6670 Self::collect_planning_function_requests(&ast, &mut requests);
6671 let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6672 &self.resolver,
6673 requests,
6674 )
6675 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
6676 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6677 let domains: Vec<_> = family
6678 .fragments
6679 .iter()
6680 .copied()
6681 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6682 .collect();
6683 for domain in &domains {
6684 validate_anchor_once_shadow_relocation(
6685 &ast,
6686 family.template_origin0.row,
6687 family.template_origin0.col,
6688 domain,
6689 &snapshot,
6690 )
6691 .map_err(SourceFamilyPreparationError::ast)?;
6692 }
6693 let ast_id = self.intern_formula_ast(&ast);
6694 let placement = CellRef::new(
6695 sheet_id,
6696 Coord::from_excel(
6697 family.template_origin0.row + 1,
6698 family.template_origin0.col + 1,
6699 true,
6700 true,
6701 ),
6702 );
6703 let ingested = self
6704 .graph
6705 .ingest_pipeline(&snapshot)
6706 .enable_function_semantics()
6707 .ingest_formula(
6708 FormulaAstInput::RawArena(ast_id),
6709 placement,
6710 Some(family.template_text.clone()),
6711 )
6712 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6713 let function_semantics_used = ingested
6714 .labels
6715 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6716 let name_assumptions = self
6717 .graph
6718 .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
6719 .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
6720 let candidate = FormulaPlacementCandidate::new(
6721 sheet_id,
6722 family.template_origin0.row,
6723 family.template_origin0.col,
6724 ingested.ast_id,
6725 Some(family.template_text.clone()),
6726 );
6727 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6728 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6729 let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
6730 total
6731 .checked_add(domain.cell_count())
6732 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
6733 })?;
6734 let mut placements = Vec::with_capacity(domains.len());
6735 for domain in domains {
6736 let member_count = domain.cell_count();
6737 placements.push(
6738 prepare_anchor_once_fragment(
6739 candidate.clone(),
6740 analysis.clone(),
6741 domain,
6742 member_count,
6743 direct_cells,
6744 )
6745 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
6746 );
6747 }
6748 let fallback_cells = family.legacy_members.shared_member_count() as u64;
6749 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6750 return Err(SourceFamilyPreparationError::analysis(
6751 "PartitionMemberCountMismatch",
6752 ));
6753 }
6754 Ok(
6755 crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
6756 engine_token: Arc::clone(&self.source_formula_token),
6757 sheet_id,
6758 sheet_name: Arc::from(sheet_name),
6759 source: family.clone(),
6760 placements,
6761 function_semantic_epoch: snapshot.epoch(),
6762 function_provider_revision: snapshot.provider_revision(),
6763 function_semantics_used,
6764 name_assumptions,
6765 direct_cells,
6766 },
6767 )
6768 }
6769
6770 fn prepare_partitioned_source_formula_family_shadow(
6771 &mut self,
6772 sheet_id: SheetId,
6773 family: &crate::engine::PartitionedSourceFormulaFamily,
6774 ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
6775 family
6776 .validate(&self.workbook_load_limits)
6777 .map_err(SourceFamilyPreparationError::contract)?;
6778 let formula = if family.template_text.starts_with('=') {
6779 family.template_text.to_string()
6780 } else {
6781 format!("={}", family.template_text)
6782 };
6783 let ast = formualizer_parse::parser::parse(&formula)
6784 .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6785 let domains: Vec<_> = family
6786 .fragments
6787 .iter()
6788 .copied()
6789 .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6790 .collect();
6791 for domain in &domains {
6792 validate_anchor_once_shadow_relocation(
6793 &ast,
6794 family.template_origin0.row,
6795 family.template_origin0.col,
6796 domain,
6797 &self.resolver,
6798 )
6799 .map_err(SourceFamilyPreparationError::ast)?;
6800 }
6801 let ast_id = self.intern_formula_ast(&ast);
6802 let placement = CellRef::new(
6803 sheet_id,
6804 Coord::from_excel(
6805 family.template_origin0.row + 1,
6806 family.template_origin0.col + 1,
6807 true,
6808 true,
6809 ),
6810 );
6811 let ingested = self
6812 .ingest_pipeline()
6813 .enable_function_semantics()
6814 .ingest_formula(
6815 FormulaAstInput::RawArena(ast_id),
6816 placement,
6817 Some(family.template_text.clone()),
6818 )
6819 .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6820 let function_semantics_used = ingested
6821 .labels
6822 .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6823 let candidate = FormulaPlacementCandidate::new(
6824 sheet_id,
6825 family.template_origin0.row,
6826 family.template_origin0.col,
6827 ingested.ast_id,
6828 Some(family.template_text.clone()),
6829 );
6830 let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6831 .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6832 let fragment_count = domains.len() as u64;
6833 let mut direct_cells = 0u64;
6834 for domain in domains {
6835 let member_count = domain.cell_count();
6836 validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6837 .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6838 direct_cells = direct_cells
6839 .checked_add(member_count)
6840 .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6841 }
6842 let fallback_cells = family.legacy_members.shared_member_count() as u64;
6843 if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6844 return Err(SourceFamilyPreparationError::analysis(
6845 "PartitionMemberCountMismatch",
6846 ));
6847 }
6848 Ok(PreparedPartitionShadow {
6849 fragment_count,
6850 direct_cells,
6851 fallback_cells,
6852 function_semantics_used,
6853 })
6854 }
6855
6856 fn analyze_partitioned_formula_plane_shadow(
6857 &mut self,
6858 batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6859 ) -> FormulaIngestReport {
6860 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6861 for (sheet_name, families) in batches {
6862 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6863 for family in families {
6864 report.shadow_candidate_cells = report
6865 .shadow_candidate_cells
6866 .saturating_add(family.surviving_member_count);
6867 match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6868 Ok(prepared) => {
6869 let fragment_count = prepared.fragment_count;
6870 let descendants = family.surviving_member_count.saturating_sub(1);
6871 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6872 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6873 report.source_anchor_analyses =
6874 report.source_anchor_analyses.saturating_add(1);
6875 report.source_partition_analyses_reused = report
6876 .source_partition_analyses_reused
6877 .saturating_add(fragment_count.saturating_sub(1));
6878 report.source_partitioned_families_prepared = report
6879 .source_partitioned_families_prepared
6880 .saturating_add(1);
6881 report.source_partition_fragments_prepared = report
6882 .source_partition_fragments_prepared
6883 .saturating_add(fragment_count);
6884 report.source_partition_span_cells_prepared = report
6885 .source_partition_span_cells_prepared
6886 .saturating_add(prepared.direct_cells);
6887 report.source_partition_fallback_cells = report
6888 .source_partition_fallback_cells
6889 .saturating_add(prepared.fallback_cells);
6890 report.shadow_accepted_span_cells = report
6891 .shadow_accepted_span_cells
6892 .saturating_add(prepared.direct_cells);
6893 report.shadow_fallback_cells = report
6894 .shadow_fallback_cells
6895 .saturating_add(prepared.fallback_cells);
6896 report.source_descendant_strings_avoided = report
6897 .source_descendant_strings_avoided
6898 .saturating_add(descendants);
6899 report.source_descendant_events_avoided = report
6900 .source_descendant_events_avoided
6901 .saturating_add(descendants);
6902 report.source_descendant_analyses_avoided = report
6903 .source_descendant_analyses_avoided
6904 .saturating_add(descendants);
6905 report.graph_formula_vertices_avoided_shadow = report
6906 .graph_formula_vertices_avoided_shadow
6907 .saturating_add(prepared.direct_cells);
6908 report.ast_roots_avoided_shadow = report
6909 .ast_roots_avoided_shadow
6910 .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6911 report.edge_rows_avoided_shadow = report
6912 .edge_rows_avoided_shadow
6913 .saturating_add(prepared.direct_cells);
6914 if prepared.function_semantics_used {
6915 report.source_partition_function_semantics =
6916 report.source_partition_function_semantics.saturating_add(1);
6917 }
6918 }
6919 Err(error) => {
6920 if error.parse_attempted {
6921 report.source_anchor_parses =
6922 report.source_anchor_parses.saturating_add(1);
6923 }
6924 if error.ast_created {
6925 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6926 }
6927 if error.analysis_created {
6928 report.source_anchor_analyses =
6929 report.source_anchor_analyses.saturating_add(1);
6930 }
6931 report.shadow_fallback_cells = report
6932 .shadow_fallback_cells
6933 .saturating_add(family.surviving_member_count);
6934 report.source_partitioned_families_rejected = report
6935 .source_partitioned_families_rejected
6936 .saturating_add(1);
6937 *report.fallback_reasons.entry(error.reason).or_default() += 1;
6938 }
6939 }
6940 }
6941 }
6942 report
6943 }
6944
6945 fn analyze_compressed_formula_plane_shadow(
6946 &mut self,
6947 batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6948 ) -> FormulaIngestReport {
6949 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6950 let _active_epoch = self.graph.formula_authority().plane.epoch();
6951
6952 for (sheet_name, families) in batches {
6953 let sheet_id = self.graph.sheet_id_mut(sheet_name);
6954 for family in families {
6955 let descendants = family.member_count.saturating_sub(1);
6956 report.shadow_candidate_cells = report
6957 .shadow_candidate_cells
6958 .saturating_add(family.member_count);
6959
6960 match self.prepare_source_formula_family(sheet_id, family, true, None) {
6961 Ok(_) => {
6962 report.source_descendant_strings_avoided = report
6963 .source_descendant_strings_avoided
6964 .saturating_add(descendants);
6965 report.source_descendant_events_avoided = report
6966 .source_descendant_events_avoided
6967 .saturating_add(descendants);
6968 report.source_descendant_analyses_avoided = report
6969 .source_descendant_analyses_avoided
6970 .saturating_add(descendants);
6971 report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6972 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6973 report.source_anchor_analyses =
6974 report.source_anchor_analyses.saturating_add(1);
6975 report.shadow_accepted_span_cells = report
6976 .shadow_accepted_span_cells
6977 .saturating_add(family.member_count);
6978 report.source_compressed_families_prepared =
6979 report.source_compressed_families_prepared.saturating_add(1);
6980 report.source_compressed_cells_prepared = report
6981 .source_compressed_cells_prepared
6982 .saturating_add(family.member_count);
6983 report.graph_formula_vertices_avoided_shadow = report
6984 .graph_formula_vertices_avoided_shadow
6985 .saturating_add(family.member_count);
6986 report.ast_roots_avoided_shadow =
6987 report.ast_roots_avoided_shadow.saturating_add(descendants);
6988 report.edge_rows_avoided_shadow = report
6989 .edge_rows_avoided_shadow
6990 .saturating_add(family.member_count);
6991 }
6992 Err(error) => {
6993 if error.parse_attempted {
6994 report.source_anchor_parses =
6995 report.source_anchor_parses.saturating_add(1);
6996 }
6997 if error.ast_created {
6998 report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6999 }
7000 if error.analysis_created {
7001 report.source_anchor_analyses =
7002 report.source_anchor_analyses.saturating_add(1);
7003 }
7004 report.shadow_fallback_cells = report
7005 .shadow_fallback_cells
7006 .saturating_add(family.member_count);
7007 *report.fallback_reasons.entry(error.reason).or_default() += 1;
7008 }
7009 }
7010 }
7011 }
7012 report
7013 }
7014
7015 fn analyze_formula_plane_shadow_candidates(
7016 &mut self,
7017 batches: &[FormulaIngestBatch],
7018 ) -> FormulaIngestReport {
7019 let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
7020 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7021
7022 let _active_epoch = self.graph.formula_authority().plane.epoch();
7025
7026 let batch_sheet_ids: Vec<SheetId> = batches
7027 .iter()
7028 .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
7029 .collect();
7030 let mut groups: BTreeMap<
7031 (SheetId, u64, u32),
7032 Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
7033 > = BTreeMap::new();
7034 {
7035 let mut pipeline = self.ingest_pipeline();
7036 for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
7037 for record in &batch.formulas {
7038 if record.row == 0 || record.col == 0 {
7039 report.shadow_candidate_cells =
7040 report.shadow_candidate_cells.saturating_add(1);
7041 report.shadow_fallback_cells =
7042 report.shadow_fallback_cells.saturating_add(1);
7043 Self::record_shadow_fallback_reason(
7044 &mut report,
7045 PlacementFallbackReason::UnsupportedShapeOrGaps,
7046 1,
7047 );
7048 continue;
7049 }
7050
7051 let placement = CellRef::new(
7052 sheet_id,
7053 Coord::from_excel(record.row, record.col, true, true),
7054 );
7055 let ingested = match pipeline.ingest_formula(
7056 FormulaAstInput::RawArena(record.ast_id),
7057 placement,
7058 record.formula_text.clone(),
7059 ) {
7060 Ok(ingested) => ingested,
7061 Err(_) => {
7062 report.shadow_candidate_cells =
7063 report.shadow_candidate_cells.saturating_add(1);
7064 report.shadow_fallback_cells =
7065 report.shadow_fallback_cells.saturating_add(1);
7066 Self::record_shadow_fallback_reason(
7067 &mut report,
7068 PlacementFallbackReason::UnsupportedCanonicalTemplate,
7069 1,
7070 );
7071 continue;
7072 }
7073 };
7074 let candidate = FormulaPlacementCandidate::new(
7075 sheet_id,
7076 record.row - 1,
7077 record.col - 1,
7078 ingested.ast_id,
7079 record.formula_text.clone(),
7080 );
7081 let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
7082 Ok(analysis) => analysis,
7083 Err(reason) => {
7084 report.shadow_candidate_cells =
7085 report.shadow_candidate_cells.saturating_add(1);
7086 report.shadow_fallback_cells =
7087 report.shadow_fallback_cells.saturating_add(1);
7088 Self::record_shadow_fallback_reason(&mut report, reason, 1);
7089 continue;
7090 }
7091 };
7092 groups
7093 .entry((
7094 sheet_id,
7095 ingested.parameterized_canonical_hash,
7096 candidate.col,
7097 ))
7098 .or_default()
7099 .push((candidate, analysis));
7100 }
7101 }
7102 }
7103
7104 let mut scratch_plane = FormulaPlane::default();
7105 for entries in groups.into_values() {
7106 let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
7107 for (component, component_analyses) in
7108 Self::split_candidate_components_with_analyses(candidates, analyses)
7109 {
7110 let placement_report = place_candidate_family_with_analyses(
7111 &mut scratch_plane,
7112 component,
7113 component_analyses,
7114 );
7115 let counters = placement_report.counters;
7116 report.shadow_candidate_cells = report
7117 .shadow_candidate_cells
7118 .saturating_add(counters.formula_cells_seen);
7119 report.shadow_accepted_span_cells = report
7120 .shadow_accepted_span_cells
7121 .saturating_add(counters.accepted_span_cells);
7122 report.shadow_fallback_cells = report
7123 .shadow_fallback_cells
7124 .saturating_add(counters.legacy_cells);
7125 report.shadow_templates_interned = report
7126 .shadow_templates_interned
7127 .saturating_add(counters.templates_interned);
7128 report.shadow_spans_created = report
7129 .shadow_spans_created
7130 .saturating_add(counters.spans_created);
7131 report.graph_formula_vertices_avoided_shadow = report
7132 .graph_formula_vertices_avoided_shadow
7133 .saturating_add(counters.formula_vertices_avoided);
7134 report.ast_roots_avoided_shadow = report
7135 .ast_roots_avoided_shadow
7136 .saturating_add(counters.ast_roots_avoided);
7137 report.edge_rows_avoided_shadow = report
7138 .edge_rows_avoided_shadow
7139 .saturating_add(counters.edge_rows_avoided);
7140 for (reason, count) in counters.fallback_reasons {
7141 Self::record_shadow_fallback_reason(&mut report, reason, count);
7142 }
7143 }
7144 }
7145 report
7146 }
7147
7148 fn record_shadow_fallback_reason(
7149 report: &mut FormulaIngestReport,
7150 reason: PlacementFallbackReason,
7151 count: u64,
7152 ) {
7153 *report
7154 .fallback_reasons
7155 .entry(format!("{reason:?}"))
7156 .or_default() += count;
7157 }
7158
7159 fn analyze_formula_plane_authoritative_ingest(
7160 &mut self,
7161 batches: &[FormulaIngestBatch],
7162 ) -> (
7163 FormulaIngestReport,
7164 Vec<FormulaIngestBatch>,
7165 PlannedFormulaMaterialize,
7166 ) {
7167 let existing_span_refs = self
7168 .graph
7169 .formula_authority()
7170 .active_span_refs()
7171 .into_iter()
7172 .collect::<rustc_hash::FxHashSet<_>>();
7173 let mut report =
7174 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7175 report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7176
7177 let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
7178 let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
7179 let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
7180
7181 for batch in batches {
7182 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7183 for record in &batch.formulas {
7184 if record.row == 0 || record.col == 0 {
7185 report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
7186 report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
7187 Self::record_shadow_fallback_reason(
7188 &mut report,
7189 PlacementFallbackReason::UnsupportedShapeOrGaps,
7190 1,
7191 );
7192 fallback
7193 .entry(batch.sheet_name.clone())
7194 .or_default()
7195 .push(record.clone());
7196 continue;
7197 }
7198
7199 pending_candidates.push((
7200 batch.sheet_name.clone(),
7201 FormulaPlacementCandidate::new(
7202 sheet_id,
7203 record.row - 1,
7204 record.col - 1,
7205 record.ast_id,
7206 record.formula_text.clone(),
7207 ),
7208 ));
7209 }
7210 }
7211
7212 let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
7213 let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
7214 (0..pending_candidates.len()).map(|_| None).collect();
7215 let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
7216 (0..pending_candidates.len()).map(|_| None).collect();
7217 {
7218 let mut pipeline = self.ingest_pipeline();
7219 for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
7220 let placement = CellRef::new(
7221 candidate.sheet_id,
7222 Coord::from_excel(
7223 candidate.row.saturating_add(1),
7224 candidate.col.saturating_add(1),
7225 true,
7226 true,
7227 ),
7228 );
7229 let ingested = pipeline.ingest_formula(
7230 FormulaAstInput::RawArena(candidate.ast_id),
7231 placement,
7232 candidate.formula_text.clone(),
7233 );
7234 match ingested {
7235 Ok(ingested) => {
7236 candidate.ast_id = ingested.ast_id;
7237 let dep_plan = ingested.dep_plan.clone();
7238 match CandidateAnalysis::from_ingested(candidate, &ingested) {
7239 Ok(analysis) => {
7240 analyses_by_index[idx] = Some(analysis);
7241 plans_by_index[idx] = Some(dep_plan);
7242 }
7243 Err(reason) => {
7244 report.shadow_candidate_cells =
7245 report.shadow_candidate_cells.saturating_add(1);
7246 report.shadow_fallback_cells =
7247 report.shadow_fallback_cells.saturating_add(1);
7248 Self::record_shadow_fallback_reason(&mut report, reason, 1);
7249 planned_fallback
7250 .entry(sheet_name.clone())
7251 .or_default()
7252 .push((
7253 candidate.row.saturating_add(1),
7254 candidate.col.saturating_add(1),
7255 candidate.ast_id,
7256 dep_plan,
7257 ));
7258 }
7259 }
7260 }
7261 Err(_) => {
7262 let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
7263 report.shadow_candidate_cells =
7264 report.shadow_candidate_cells.saturating_add(1);
7265 report.shadow_fallback_cells =
7266 report.shadow_fallback_cells.saturating_add(1);
7267 Self::record_shadow_fallback_reason(&mut report, reason, 1);
7268 fallback.entry(sheet_name.clone()).or_default().push(
7269 FormulaIngestRecord::new(
7270 candidate.row.saturating_add(1),
7271 candidate.col.saturating_add(1),
7272 candidate.ast_id,
7273 candidate.formula_text.clone(),
7274 ),
7275 );
7276 }
7277 }
7278 }
7279 }
7280
7281 for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
7282 if analyses_by_index[idx].is_none() {
7283 continue;
7284 }
7285 let canonical_hash = analyses_by_index[idx]
7286 .as_ref()
7287 .expect("remaining candidate has an analysis")
7288 .parameterized_canonical_hash();
7289 groups
7290 .entry((candidate.sheet_id, canonical_hash, candidate.col))
7291 .or_default()
7292 .push(idx);
7293 }
7294
7295 for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
7296 let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
7297 let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
7298 for idx in &candidate_indices {
7299 if let Some(plan) = plans_by_index[*idx].take() {
7302 let candidate = &pending_candidates[*idx].1;
7303 plans_by_coord
7304 .entry((candidate.row, candidate.col))
7305 .or_default()
7306 .push(plan);
7307 }
7308 }
7309 let candidates: Vec<_> = candidate_indices
7310 .iter()
7311 .map(|idx| pending_candidates[*idx].1.clone())
7312 .collect();
7313 let components = Self::split_shadow_candidate_components(candidates);
7314 let analyzed_components =
7315 if components.len() == 1 && components[0].len() == candidate_indices.len() {
7316 let component = components.into_iter().next().expect("one component");
7317 let component_analyses = candidate_indices
7318 .iter()
7319 .map(|idx| {
7320 analyses_by_index[*idx]
7321 .take()
7322 .expect("candidate analysis must be used once")
7323 })
7324 .collect();
7325 vec![(component, component_analyses)]
7326 } else {
7327 let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
7328 for idx in candidate_indices.iter().rev() {
7329 let candidate = &pending_candidates[*idx].1;
7330 indices_by_coord
7331 .entry((candidate.row, candidate.col))
7332 .or_default()
7333 .push(*idx);
7334 }
7335
7336 components
7337 .into_iter()
7338 .map(|component| {
7339 let mut component_analyses = Vec::with_capacity(component.len());
7340 for candidate in &component {
7341 let idx = indices_by_coord
7342 .get_mut(&(candidate.row, candidate.col))
7343 .and_then(Vec::pop)
7344 .expect("component candidate must have a precomputed analysis");
7345 component_analyses.push(
7346 analyses_by_index[idx]
7347 .take()
7348 .expect("candidate analysis must be used once"),
7349 );
7350 }
7351 (component, component_analyses)
7352 })
7353 .collect()
7354 };
7355
7356 for (component, component_analyses) in analyzed_components {
7357 for (component, component_analyses) in
7358 split_candidate_affine_literal_runs(component, component_analyses)
7359 {
7360 let placement_report = {
7361 let authority = self.graph.formula_authority_mut();
7362 place_candidate_family_with_analyses(
7363 &mut authority.plane,
7364 component.clone(),
7365 component_analyses,
7366 )
7367 };
7368 Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
7369
7370 let mut candidate_by_placement: FxHashMap<
7378 crate::formula_plane::runtime::PlacementCoord,
7379 &FormulaPlacementCandidate,
7380 > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
7381 for candidate in &component {
7382 candidate_by_placement
7383 .entry(candidate.placement())
7384 .or_insert(candidate);
7385 }
7386 for result in &placement_report.results {
7387 let FormulaPlacementResult::Legacy { placement, .. } = result else {
7388 continue;
7389 };
7390 if let Some(&candidate) = candidate_by_placement.get(placement) {
7391 let plan = plans_by_coord
7392 .get_mut(&(candidate.row, candidate.col))
7393 .and_then(Vec::pop);
7394 if let Some(plan) = plan {
7395 planned_fallback
7396 .entry(sheet_name.clone())
7397 .or_default()
7398 .push((
7399 candidate.row.saturating_add(1),
7400 candidate.col.saturating_add(1),
7401 candidate.ast_id,
7402 plan,
7403 ));
7404 } else {
7405 fallback.entry(sheet_name.clone()).or_default().push(
7406 FormulaIngestRecord::new(
7407 candidate.row.saturating_add(1),
7408 candidate.col.saturating_add(1),
7409 candidate.ast_id,
7410 candidate.formula_text.clone(),
7411 ),
7412 );
7413 }
7414 }
7415 }
7416 }
7417 }
7418 }
7419
7420 let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
7421 let new_span_refs = self
7422 .graph
7423 .formula_authority()
7424 .active_span_refs()
7425 .into_iter()
7426 .filter(|span_ref| !existing_span_refs.contains(span_ref))
7427 .collect::<Vec<_>>();
7428 self.graph
7429 .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
7430
7431 let fallback_batches = fallback
7432 .into_iter()
7433 .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
7434 .collect();
7435 (report, fallback_batches, planned_fallback)
7436 }
7437
7438 fn accumulate_formula_plane_placement_report(
7439 report: &mut FormulaIngestReport,
7440 placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
7441 ) {
7442 let counters = &placement_report.counters;
7443 report.shadow_candidate_cells = report
7444 .shadow_candidate_cells
7445 .saturating_add(counters.formula_cells_seen);
7446 report.shadow_accepted_span_cells = report
7447 .shadow_accepted_span_cells
7448 .saturating_add(counters.accepted_span_cells);
7449 report.shadow_fallback_cells = report
7450 .shadow_fallback_cells
7451 .saturating_add(counters.legacy_cells);
7452 report.shadow_templates_interned = report
7453 .shadow_templates_interned
7454 .saturating_add(counters.templates_interned);
7455 report.shadow_spans_created = report
7456 .shadow_spans_created
7457 .saturating_add(counters.spans_created);
7458 report.graph_formula_vertices_avoided_shadow = report
7459 .graph_formula_vertices_avoided_shadow
7460 .saturating_add(counters.formula_vertices_avoided);
7461 report.ast_roots_avoided_shadow = report
7462 .ast_roots_avoided_shadow
7463 .saturating_add(counters.ast_roots_avoided);
7464 report.edge_rows_avoided_shadow = report
7465 .edge_rows_avoided_shadow
7466 .saturating_add(counters.edge_rows_avoided);
7467 for (reason, count) in &counters.fallback_reasons {
7468 Self::record_shadow_fallback_reason(report, *reason, *count);
7469 }
7470 }
7471
7472 fn split_candidate_components_with_analyses(
7473 candidates: Vec<FormulaPlacementCandidate>,
7474 mut analyses: Vec<CandidateAnalysis>,
7475 ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
7476 let components = Self::split_shadow_candidate_components(candidates.clone());
7477 let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
7478 for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
7479 analysis_by_coord
7480 .entry((candidate.row, candidate.col))
7481 .or_default()
7482 .push(analysis);
7483 }
7484 components
7485 .into_iter()
7486 .flat_map(|component| {
7487 let mut component_analyses = Vec::with_capacity(component.len());
7488 for candidate in &component {
7489 let analysis = analysis_by_coord
7490 .get_mut(&(candidate.row, candidate.col))
7491 .and_then(Vec::pop)
7492 .expect("component candidate must have a precomputed analysis");
7493 component_analyses.push(analysis);
7494 }
7495 split_candidate_affine_literal_runs(component, component_analyses)
7496 })
7497 .collect()
7498 }
7499
7500 fn split_shadow_candidate_components(
7501 candidates: Vec<FormulaPlacementCandidate>,
7502 ) -> Vec<Vec<FormulaPlacementCandidate>> {
7503 if candidates.len() <= 1 {
7504 return vec![candidates];
7505 }
7506
7507 let is_row_run = candidates.windows(2).all(|w| {
7511 w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
7512 });
7513 let is_col_run = candidates.windows(2).all(|w| {
7514 w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
7515 });
7516 if is_row_run || is_col_run {
7517 return vec![candidates];
7518 }
7519
7520 let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
7521 for (idx, candidate) in candidates.iter().enumerate() {
7522 coord_to_indices
7523 .entry((candidate.row, candidate.col))
7524 .or_default()
7525 .push(idx);
7526 }
7527
7528 let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
7529 let mut components = Vec::new();
7530 while let Some(&start) = remaining.iter().next() {
7531 remaining.remove(&start);
7532 let mut queue = VecDeque::from([start]);
7533 let mut component_indices = Vec::new();
7534
7535 while let Some(idx) = queue.pop_front() {
7536 component_indices.push(idx);
7537 let candidate = &candidates[idx];
7538 let mut neighbor_coords = Vec::with_capacity(5);
7539 neighbor_coords.push((candidate.row, candidate.col));
7540 if let Some(row) = candidate.row.checked_sub(1) {
7541 neighbor_coords.push((row, candidate.col));
7542 }
7543 neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
7544 if let Some(col) = candidate.col.checked_sub(1) {
7545 neighbor_coords.push((candidate.row, col));
7546 }
7547 neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
7548
7549 for coord in neighbor_coords {
7550 if let Some(indices) = coord_to_indices.get(&coord) {
7551 for &neighbor in indices {
7552 if remaining.remove(&neighbor) {
7553 queue.push_back(neighbor);
7554 }
7555 }
7556 }
7557 }
7558 }
7559
7560 component_indices.sort_by_key(|idx| {
7561 let candidate = &candidates[*idx];
7562 (candidate.row, candidate.col, *idx)
7563 });
7564 components.push(
7565 component_indices
7566 .into_iter()
7567 .map(|idx| candidates[idx].clone())
7568 .collect(),
7569 );
7570 }
7571
7572 components
7573 }
7574
7575 fn collect_planning_function_requests(
7576 ast: &ASTNode,
7577 requests: &mut Vec<(String, String, usize)>,
7578 ) {
7579 match &ast.node_type {
7580 ASTNodeType::Function { name, args } => {
7581 requests.push((String::new(), name.clone(), args.len()));
7582 for arg in args {
7583 Self::collect_planning_function_requests(arg, requests);
7584 }
7585 }
7586 ASTNodeType::BinaryOp { left, right, .. } => {
7587 Self::collect_planning_function_requests(left, requests);
7588 Self::collect_planning_function_requests(right, requests);
7589 }
7590 ASTNodeType::UnaryOp { expr, .. } => {
7591 Self::collect_planning_function_requests(expr, requests);
7592 }
7593 ASTNodeType::Call { callee, args } => {
7594 Self::collect_planning_function_requests(callee, requests);
7595 for arg in args {
7596 Self::collect_planning_function_requests(arg, requests);
7597 }
7598 }
7599 ASTNodeType::Array(rows) => {
7600 for cell in rows.iter().flatten() {
7601 Self::collect_planning_function_requests(cell, requests);
7602 }
7603 }
7604 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
7605 }
7606 }
7607
7608 fn prepared_placement_function_semantics_changed(
7609 &self,
7610 semantic_epoch: u64,
7611 prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
7612 guard: &crate::function_registry::SemanticEpochReadGuard,
7613 ) -> bool {
7614 if semantic_epoch == guard.epoch() {
7615 return false;
7616 }
7617 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
7618 let Some(ast) = self
7619 .graph
7620 .data_store()
7621 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
7622 else {
7623 return true;
7625 };
7626 let mut requests = Vec::new();
7627 Self::collect_planning_function_requests(&ast, &mut requests);
7628 guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
7629 }
7630
7631 fn prepared_function_semantics_changed(
7632 &self,
7633 preparation: &crate::engine::FormulaCompressedPreparation,
7634 guard: &crate::function_registry::SemanticEpochReadGuard,
7635 ) -> bool {
7636 if preparation.function_semantic_epoch == guard.epoch() {
7637 return false;
7638 }
7639
7640 let mut requests = Vec::new();
7641 for (_, _, prepared) in &preparation.prepared {
7642 let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
7643 let Some(ast) = self
7644 .graph
7645 .data_store()
7646 .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
7647 else {
7648 return true;
7649 };
7650 Self::collect_planning_function_requests(&ast, &mut requests);
7651 }
7652 guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
7653 }
7654
7655 pub(crate) fn prepare_source_formula_families(
7656 &mut self,
7657 sheet_name: &str,
7658 families: &[crate::engine::SourceFormulaFamily],
7659 ) -> crate::engine::FormulaCompressedPreparation {
7660 let mut preparation = crate::engine::FormulaCompressedPreparation {
7661 engine_token: Arc::clone(&self.source_formula_token),
7662 function_semantic_epoch: crate::function_registry::semantic_epoch(),
7663 function_provider_revision: None,
7664 function_semantics_used: false,
7665 sheet_name: Arc::from(sheet_name),
7666 prepared: Vec::new(),
7667 rejected: BTreeMap::new(),
7668 fragmented: Vec::new(),
7669 fragmented_sources: BTreeMap::new(),
7670 eager_replay: Vec::new(),
7671 preparation_spool_replays: 0,
7672 clean_rejected_anchor_counts: [0; 3],
7673 fragmented_rejected_anchor_counts: [0; 3],
7674 exact_replay: None,
7675 replay_disposition: crate::engine::FormulaReplayDisposition::default(),
7676 };
7677 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7678 return preparation;
7679 }
7680
7681 let mut requests = Vec::new();
7682 for family in families {
7683 let formula = if family.anchor_text.starts_with('=') {
7684 family.anchor_text.to_string()
7685 } else {
7686 format!("={}", family.anchor_text)
7687 };
7688 if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
7689 Self::collect_planning_function_requests(&ast, &mut requests);
7690 }
7691 }
7692 requests.sort();
7693 requests.dedup();
7694 let snapshot =
7695 match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7696 &self.resolver,
7697 requests,
7698 ) {
7699 Ok(snapshot) => snapshot,
7700 Err(error) => {
7701 let reason = match error {
7702 crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
7703 "FunctionSemanticSnapshotUnavailable"
7704 }
7705 crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
7706 "FunctionProviderRevisionUnavailable"
7707 }
7708 };
7709 for family in families {
7710 preparation
7711 .rejected
7712 .insert(family.source_id, reason.to_string());
7713 }
7714 return preparation;
7715 }
7716 };
7717 preparation.function_semantic_epoch = snapshot.epoch();
7718 preparation.function_provider_revision = snapshot.provider_revision();
7719
7720 let sheet_id = self.graph.sheet_id_mut(sheet_name);
7721 for family in families {
7722 #[cfg(test)]
7723 let force_replay = self.force_source_family_fallback
7724 || cfg!(feature = "benchmark_internal")
7725 && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7726 #[cfg(all(feature = "benchmark_internal", not(test)))]
7727 let force_replay =
7728 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7729 #[cfg(any(test, feature = "benchmark_internal"))]
7730 if force_replay {
7731 preparation
7732 .rejected
7733 .insert(family.source_id, "ForcedReplay".to_string());
7734 continue;
7735 }
7736 match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
7737 Ok((prepared, function_semantics_used)) => {
7738 preparation.function_semantics_used |= function_semantics_used;
7739 preparation
7740 .replay_disposition
7741 .set_family_direct(family.source_id);
7742 preparation
7743 .prepared
7744 .push((family.source_id, family.source_order, prepared));
7745 }
7746 Err(error) => {
7747 preparation.clean_rejected_anchor_counts[0] = preparation
7748 .clean_rejected_anchor_counts[0]
7749 .saturating_add(u64::from(error.parse_attempted));
7750 preparation.clean_rejected_anchor_counts[1] = preparation
7751 .clean_rejected_anchor_counts[1]
7752 .saturating_add(u64::from(error.ast_created));
7753 preparation.clean_rejected_anchor_counts[2] = preparation
7754 .clean_rejected_anchor_counts[2]
7755 .saturating_add(u64::from(error.analysis_created));
7756 preparation.rejected.insert(family.source_id, error.reason);
7757 }
7758 }
7759 }
7760 preparation
7761 }
7762
7763 fn prepare_source_formula_proposals(
7764 &mut self,
7765 sheet_name: &str,
7766 families: &[crate::engine::SourceFormulaFamily],
7767 authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7768 replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7769 formula_record_count: u64,
7770 replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
7771 suppressed: &BTreeSet<(u32, u32)>,
7772 ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
7773 let mut preparation = self.prepare_source_formula_families(sheet_name, families);
7774 preparation.exact_replay = Some(Arc::clone(&replay));
7775 preparation
7776 .replay_disposition
7777 .extend_suppressed_excel_coords(suppressed.iter().copied());
7778 if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7779 return Ok(preparation);
7780 }
7781 #[cfg(feature = "benchmark_internal")]
7782 let benchmark_forced_replay =
7783 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7784 #[cfg(not(feature = "benchmark_internal"))]
7785 let benchmark_forced_replay = false;
7786 let authority_partitions = if benchmark_forced_replay {
7787 for partition in authority_partitions {
7788 preparation
7789 .rejected
7790 .insert(partition.source_id, "ForcedReplay".to_string());
7791 }
7792 &[][..]
7793 } else {
7794 authority_partitions
7795 };
7796 let authority_ids: BTreeSet<_> = authority_partitions
7797 .iter()
7798 .map(|partition| partition.source_id)
7799 .collect();
7800 for partition in replay_partitions {
7801 if !authority_ids.contains(&partition.source_id) {
7802 preparation
7803 .replay_disposition
7804 .register_partition(partition, false)
7805 .map_err(|reason| {
7806 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7807 })?;
7808 }
7809 }
7810 if replay_partitions.is_empty()
7811 && preparation.rejected.is_empty()
7812 && preparation.direct_cell_count() == formula_record_count
7813 {
7814 return Ok(preparation);
7815 }
7816
7817 let mut analyzed = Vec::new();
7818 for source in authority_partitions {
7819 preparation
7820 .fragmented_sources
7821 .insert(source.source_id, source.clone());
7822 match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7823 Ok(prepared) => {
7824 let mut candidate = preparation.replay_disposition.clone();
7825 match candidate.register_partition(source, true) {
7826 Ok(()) => {
7827 preparation.replay_disposition = candidate;
7828 analyzed.push((source.clone(), prepared));
7829 }
7830 Err(reason) => {
7831 preparation
7832 .rejected
7833 .insert(source.source_id, reason.to_string());
7834 preparation
7835 .replay_disposition
7836 .register_partition(source, false)
7837 .map_err(|reason| {
7838 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7839 })?;
7840 }
7841 }
7842 }
7843 Err(error) => {
7844 preparation.fragmented_rejected_anchor_counts[0] = preparation
7845 .fragmented_rejected_anchor_counts[0]
7846 .saturating_add(u64::from(error.parse_attempted));
7847 preparation.fragmented_rejected_anchor_counts[1] = preparation
7848 .fragmented_rejected_anchor_counts[1]
7849 .saturating_add(u64::from(error.ast_created));
7850 preparation.fragmented_rejected_anchor_counts[2] = preparation
7851 .fragmented_rejected_anchor_counts[2]
7852 .saturating_add(u64::from(error.analysis_created));
7853 preparation.rejected.insert(source.source_id, error.reason);
7854 preparation
7855 .replay_disposition
7856 .register_partition(source, false)
7857 .map_err(|reason| {
7858 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7859 })?;
7860 }
7861 }
7862 }
7863
7864 let initial_replay = replay
7865 .lock()
7866 .map_err(|_| {
7867 ExcelError::new(ExcelErrorKind::Value)
7868 .with_message("compressed formula exact replay lock poisoned")
7869 })?
7870 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7871 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7872 preparation.preparation_spool_replays = 1;
7873 let sheet_id = self.graph.sheet_id_mut(sheet_name);
7874 let mut preparation_failed = false;
7875 for (source, prepared_family) in analyzed {
7876 let result = (|| {
7877 let expected: BTreeMap<_, _> = source
7878 .legacy_members
7879 .as_slice()
7880 .iter()
7881 .map(|member| (member.coord, member.kind))
7882 .collect();
7883 let mut actual = BTreeMap::new();
7884 for record in initial_replay
7885 .iter()
7886 .filter(|record| record.partition_owner == Some(source.source_id))
7887 {
7888 let coord = record
7889 .row
7890 .checked_sub(1)
7891 .zip(record.col.checked_sub(1))
7892 .map(|(row, col)| crate::engine::SourceCoord { row, col })
7893 .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7894 let kind = match record.family {
7895 Some(family) if family == source.source_id => {
7896 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7897 }
7898 None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7899 Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7900 };
7901 if actual.insert(coord, (kind, record.clone())).is_some() {
7902 return Err("ExactReplayRecordDuplicate".to_string());
7903 }
7904 }
7905 let actual_kinds: BTreeMap<_, _> = actual
7906 .iter()
7907 .map(|(coord, (kind, _))| (*coord, *kind))
7908 .collect();
7909 if actual_kinds != expected {
7910 return Err("ExactReplayLegacySetMismatch".to_string());
7911 }
7912 let mut legacy = Vec::with_capacity(source.legacy_members.len());
7913 for expected in source.legacy_members.as_slice() {
7914 let record = actual
7915 .remove(&expected.coord)
7916 .map(|(_, record)| record)
7917 .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7918 legacy.push(
7919 self.graph
7920 .analyze_fragmented_exact_replay_record(
7921 &self.resolver,
7922 sheet_id,
7923 source.source_id,
7924 *expected,
7925 record,
7926 )
7927 .map_err(|error| format!("{error:?}"))?,
7928 );
7929 }
7930 Ok::<_, String>((prepared_family, legacy))
7931 })();
7932 match result {
7933 Ok((prepared_family, legacy)) => preparation.fragmented.push(
7934 crate::engine::formula_source::PreparedFragmentedSourceProposal {
7935 source: source.clone(),
7936 prepared: prepared_family,
7937 legacy,
7938 replay: Arc::clone(&replay),
7939 },
7940 ),
7941 Err(reason) => {
7942 preparation_failed = true;
7943 for count in &mut preparation.fragmented_rejected_anchor_counts {
7944 *count = count.saturating_add(1);
7945 }
7946 preparation.rejected.insert(source.source_id, reason);
7947 preparation
7948 .replay_disposition
7949 .force_family_legacy(source.source_id);
7950 }
7951 }
7952 }
7953
7954 preparation.eager_replay = if preparation_failed {
7955 preparation.preparation_spool_replays = 2;
7956 replay
7957 .lock()
7958 .map_err(|_| {
7959 ExcelError::new(ExcelErrorKind::Value)
7960 .with_message("compressed formula exact replay lock poisoned")
7961 })?
7962 .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7963 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7964 } else {
7965 initial_replay
7966 };
7967 Ok(preparation)
7968 }
7969
7970 fn formula_batch_from_exact_replay(
7971 &mut self,
7972 sheet_name: &str,
7973 replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7974 ) -> Result<FormulaIngestBatch, ExcelError> {
7975 let mut cache = rustc_hash::FxHashMap::default();
7976 let mut formulas = Vec::new();
7977 for record in replayed {
7978 let key = if record.text.starts_with('=') {
7979 record.text
7980 } else {
7981 format!("={}", record.text)
7982 };
7983 let ast_id = if let Some(cached) = cache.get(&key) {
7984 *cached
7985 } else {
7986 let parsed = match formualizer_parse::parser::parse(&key) {
7987 Ok(parsed) => parsed,
7988 Err(error) => {
7989 let Some(parsed) = self.handle_formula_parse_error(
7990 sheet_name,
7991 record.row,
7992 record.col,
7993 &key,
7994 error.to_string(),
7995 )?
7996 else {
7997 continue;
7998 };
7999 parsed
8000 }
8001 };
8002 let ast_id = self.intern_formula_ast(&parsed);
8003 cache.insert(key.clone(), ast_id);
8004 ast_id
8005 };
8006 formulas.push(
8007 FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
8008 .with_source_proof(record.source_order, record.family, record.partition_owner),
8009 );
8010 }
8011 Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
8012 }
8013
8014 fn replay_prepared_families_exact_records(
8015 &mut self,
8016 preparation: &crate::engine::FormulaCompressedPreparation,
8017 ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
8018 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
8019 ExcelError::new(ExcelErrorKind::Value)
8020 .with_message("stale compressed preparation has no exact replay owner")
8021 })?;
8022 let direct: BTreeSet<_> = preparation
8023 .prepared
8024 .iter()
8025 .map(|(family, _, _)| *family)
8026 .collect();
8027 let mut replay_disposition = preparation.replay_disposition.clone();
8028 for family in &direct {
8029 replay_disposition.force_family_legacy(*family);
8030 }
8031 let replayed = replay
8032 .lock()
8033 .map_err(|_| {
8034 ExcelError::new(ExcelErrorKind::Value)
8035 .with_message("compressed formula exact replay lock poisoned")
8036 })?
8037 .replay(&replay_disposition)
8038 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
8039 let mut records: Vec<_> = replayed
8040 .into_iter()
8041 .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
8042 .collect();
8043 records.sort_by_key(|record| record.source_order);
8044 let expected: u64 = preparation
8045 .prepared
8046 .iter()
8047 .map(|(_, _, prepared)| prepared.member_count)
8048 .sum();
8049 if records.len() as u64 != expected {
8050 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
8051 "compressed formula exact replay incomplete: expected {expected}, got {}",
8052 records.len()
8053 )));
8054 }
8055 Ok(records)
8056 }
8057
8058 fn replay_one_prepared_family_exact(
8059 &mut self,
8060 preparation: &crate::engine::FormulaCompressedPreparation,
8061 family: crate::engine::SourceFamilyId,
8062 member_count: u64,
8063 ) -> Result<FormulaIngestBatch, ExcelError> {
8064 let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
8065 ExcelError::new(ExcelErrorKind::Value)
8066 .with_message("stale compressed preparation has no exact replay owner")
8067 })?;
8068 let mut replay_disposition = preparation.replay_disposition.clone();
8069 replay_disposition.force_family_legacy(family);
8070 let replayed = replay
8071 .lock()
8072 .map_err(|_| {
8073 ExcelError::new(ExcelErrorKind::Value)
8074 .with_message("compressed formula exact replay lock poisoned")
8075 })?
8076 .replay(&replay_disposition)
8077 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
8078 .into_iter()
8079 .filter(|record| record.family == Some(family));
8080 let batch =
8081 self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
8082 if batch.formulas.len() as u64 != member_count {
8083 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
8084 "compressed formula exact replay incomplete: expected {member_count}, got {}",
8085 batch.formulas.len()
8086 )));
8087 }
8088 Ok(batch)
8089 }
8090
8091 fn exact_replay_record_is_prepared_partition_legacy(
8092 source: &crate::engine::PartitionedSourceFormulaFamily,
8093 record: &crate::engine::DeferredReplayFormula,
8094 ) -> bool {
8095 if record.partition_owner != Some(source.source_id) {
8096 return false;
8097 }
8098 let Some(coord) = record
8099 .row
8100 .checked_sub(1)
8101 .zip(record.col.checked_sub(1))
8102 .map(|(row, col)| crate::engine::SourceCoord { row, col })
8103 else {
8104 return false;
8105 };
8106 source.legacy_members.as_slice().iter().any(|member| {
8107 member.coord == coord
8108 && match member.kind {
8109 crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
8110 record.family == Some(source.source_id)
8111 }
8112 crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
8113 record.family.is_none()
8114 }
8115 }
8116 })
8117 }
8118
8119 fn validate_whole_partition_replay(
8120 source: &crate::engine::PartitionedSourceFormulaFamily,
8121 records: &[crate::engine::DeferredReplayFormula],
8122 ) -> Result<(), ExcelError> {
8123 let mut coordinates = BTreeSet::new();
8124 let mut shared = 0u64;
8125 let mut ordinary = 0u64;
8126 for record in records {
8127 let coord = record
8128 .row
8129 .checked_sub(1)
8130 .zip(record.col.checked_sub(1))
8131 .map(|(row, col)| crate::engine::SourceCoord { row, col })
8132 .ok_or_else(|| {
8133 ExcelError::new(ExcelErrorKind::Value)
8134 .with_message("whole-family replay contains an invalid coordinate")
8135 })?;
8136 if !coordinates.insert(coord) {
8137 return Err(ExcelError::new(ExcelErrorKind::Value)
8138 .with_message("whole-family replay contains a duplicate coordinate"));
8139 }
8140 match (record.family, record.partition_owner) {
8141 (Some(family), Some(owner))
8142 if family == source.source_id && owner == source.source_id =>
8143 {
8144 let direct = source.fragments.iter().any(|fragment| {
8145 let rect = fragment.rect();
8146 coord.row >= rect.start.row
8147 && coord.row <= rect.end.row
8148 && coord.col >= rect.start.col
8149 && coord.col <= rect.end.col
8150 });
8151 let legacy = source.legacy_members.as_slice().iter().any(|member| {
8152 member.coord == coord
8153 && member.kind
8154 == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
8155 });
8156 if !direct && !legacy {
8157 return Err(ExcelError::new(ExcelErrorKind::Value)
8158 .with_message("whole-family replay contains an extra shared member"));
8159 }
8160 shared = shared.saturating_add(1);
8161 }
8162 (None, Some(owner)) if owner == source.source_id => {
8163 if !source.legacy_members.as_slice().iter().any(|member| {
8164 member.coord == coord
8165 && member.kind
8166 == crate::engine::PartitionLegacyMemberKind::OrdinaryException
8167 }) {
8168 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8169 "whole-family replay contains an extra ordinary exception",
8170 ));
8171 }
8172 ordinary = ordinary.saturating_add(1);
8173 }
8174 _ => {
8175 return Err(ExcelError::new(ExcelErrorKind::Value)
8176 .with_message("whole-family replay ownership mismatch"));
8177 }
8178 }
8179 }
8180 if shared != source.surviving_member_count
8181 || ordinary != source.reconciliation.ordinary_exceptions
8182 {
8183 return Err(ExcelError::new(ExcelErrorKind::Value)
8184 .with_message("whole-family replay is incomplete"));
8185 }
8186 Ok(())
8187 }
8188
8189 fn prepare_target_combined_legacy_graph(
8190 &self,
8191 packages: &[PreparedTargetSourcePackage],
8192 ordinary: &[PreparedOrdinaryStagedFormula],
8193 ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
8194 let mut planned_by_coord = BTreeMap::new();
8195 for package in packages {
8196 for (row, col, ast_id, plan) in &package.legacy {
8197 planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
8198 }
8199 }
8200 for formula in ordinary {
8201 if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
8202 planned_by_coord.insert(
8203 (formula.sheet_id, formula.lease.row, formula.lease.col),
8204 (ast_id, plan),
8205 );
8206 }
8207 }
8208 let planned = planned_by_coord
8209 .into_iter()
8210 .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
8211 .collect::<Vec<_>>();
8212 let formula_count = planned.len();
8213 let graph = self
8214 .graph
8215 .prepare_legacy_graph_plan_multi_sheet(planned)
8216 .map_err(|error| {
8217 ExcelError::new(ExcelErrorKind::Value)
8218 .with_message(format!("target graph preparation failed: {error}"))
8219 })?;
8220 Ok((graph, formula_count))
8221 }
8222
8223 fn materialize_target_package_direct_records(
8224 &mut self,
8225 package: &mut PreparedTargetSourcePackage,
8226 assumptions: &crate::engine::PreparationRevision,
8227 planning_requests: &mut BTreeSet<(String, String, usize)>,
8228 deadline: Option<std::time::Instant>,
8229 ) -> Result<(), ExcelError> {
8230 let existing = package
8231 .legacy
8232 .iter()
8233 .map(|(row, col, _, _)| (*row, *col))
8234 .collect::<BTreeSet<_>>();
8235 let mut missing = BTreeMap::new();
8236 for record in &package.replay_records {
8237 if !existing.contains(&(record.row, record.col)) {
8238 missing.insert((record.row, record.col), record.clone());
8239 }
8240 }
8241 let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
8242 for record in batch.formulas {
8243 self.target_preparation_checkpoint(deadline, 1)?;
8244 let ast = self
8245 .graph
8246 .data_store()
8247 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8248 .ok_or_else(|| {
8249 ExcelError::new(ExcelErrorKind::Value)
8250 .with_message("target fallback AST is unavailable")
8251 })?;
8252 let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
8253 if let Some(reason) =
8254 Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
8255 {
8256 return Err(Self::preparation_stale(
8257 reason,
8258 "target fallback planning snapshot became stale during materialization",
8259 ));
8260 }
8261 let placement = CellRef::new(
8262 package.sheet_id,
8263 Coord::from_excel(record.row, record.col, true, true),
8264 );
8265 let ingested = self
8266 .graph
8267 .ingest_pipeline(&snapshot)
8268 .enable_function_semantics()
8269 .ingest_formula(
8270 FormulaAstInput::RawArena(record.ast_id),
8271 placement,
8272 record.formula_text,
8273 )?;
8274 package
8275 .legacy
8276 .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
8277 }
8278 package.direct_families = 0;
8279 package.direct_cells = 0;
8280 package.direct_fragments = 0;
8281 package.direct_complete_families = 0;
8282 package.direct_complete_cells = 0;
8283 package.direct_partition_families = 0;
8284 package.direct_partition_cells = 0;
8285 Ok(())
8286 }
8287
8288 fn prepare_target_source_package(
8289 &mut self,
8290 sheet: &str,
8291 lease: StagedPackageLease,
8292 deadline: Option<std::time::Instant>,
8293 ) -> Result<PreparedTargetSourcePackage, ExcelError> {
8294 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8295 ExcelError::new(ExcelErrorKind::Ref)
8296 .with_message(format!("deferred source sheet not found: {sheet}"))
8297 })?;
8298 let (
8299 mut source_report,
8300 families,
8301 partitions,
8302 replay,
8303 invalidated,
8304 suppressed,
8305 reconciliation_replay,
8306 ) = {
8307 let package = self
8308 .staged_formulas
8309 .get(sheet)
8310 .and_then(|staged| staged.deferred_package.as_ref())
8311 .ok_or_else(|| {
8312 ExcelError::new(ExcelErrorKind::Value)
8313 .with_message("staged deferred source package is unavailable")
8314 })?;
8315 if package.sheet_name != sheet {
8316 return Err(ExcelError::new(ExcelErrorKind::Value)
8317 .with_message("deferred formula package sheet mismatch"));
8318 }
8319 (
8320 package.report.clone(),
8321 package.families.clone(),
8322 package.partitioned_families.clone(),
8323 Arc::clone(&package.replay),
8324 package.invalidated.clone(),
8325 package.suppressed.clone(),
8326 package.reconciliation_replay.clone(),
8327 )
8328 };
8329
8330 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
8331 for partition in &partitions {
8332 replay_disposition
8333 .register_partition(partition, false)
8334 .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
8335 }
8336 replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
8337 self.target_preparation_checkpoint(deadline, 1)?;
8338 let mut replay_records = if let Some(mut records) = reconciliation_replay {
8339 records.retain(|record| {
8340 let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
8341 else {
8342 return true;
8343 };
8344 let coord = crate::engine::SourceCoord { row, col };
8345 let disposition = record.family.map_or_else(
8346 || replay_disposition.ordinary_disposition(coord).0,
8347 |family| replay_disposition.shared_disposition(family, coord),
8348 );
8349 !matches!(
8350 disposition,
8351 crate::engine::FormulaReplayCoordinateDisposition::Direct
8352 | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
8353 )
8354 });
8355 records
8356 } else {
8357 replay
8358 .lock()
8359 .map_err(|_| {
8360 ExcelError::new(ExcelErrorKind::Value)
8361 .with_message("deferred formula spool lock poisoned")
8362 })?
8363 .replay_partitioned(&replay_disposition, &partitions)
8364 .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
8365 };
8366 replay_records.sort_by_key(|record| record.source_order);
8367 for chunk in replay_records.chunks(256) {
8368 self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
8369 }
8370 if replay_records
8371 .windows(2)
8372 .any(|records| records[0].source_order == records[1].source_order)
8373 {
8374 return Err(ExcelError::new(ExcelErrorKind::Value)
8375 .with_message("duplicate deferred source-order proof"));
8376 }
8377
8378 let mut disposition = replay_disposition;
8379 let mut ordered_placements = Vec::new();
8380 let mut direct_families = 0usize;
8381 let mut direct_cells = 0u64;
8382 let mut direct_fragments = 0u64;
8383 let mut direct_complete_families = 0u64;
8384 let mut direct_complete_cells = 0u64;
8385 let mut direct_partition_families = 0u64;
8386 let mut direct_partition_cells = 0u64;
8387 let mut anchor_parses = 0u64;
8388 let mut anchor_asts = 0u64;
8389 let mut anchor_analyses = 0u64;
8390 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
8391 for family in &families {
8392 self.target_preparation_checkpoint(deadline, 1)?;
8393 if invalidated.contains(&family.source_id) {
8394 continue;
8395 }
8396 match self.prepare_source_formula_family(sheet_id, family, true, None) {
8397 Ok((prepared, _)) => {
8398 anchor_parses = anchor_parses.saturating_add(1);
8399 anchor_asts = anchor_asts.saturating_add(1);
8400 anchor_analyses = anchor_analyses.saturating_add(1);
8401 direct_families = direct_families.saturating_add(1);
8402 direct_cells = direct_cells.saturating_add(prepared.member_count);
8403 direct_complete_families = direct_complete_families.saturating_add(1);
8404 direct_complete_cells =
8405 direct_complete_cells.saturating_add(prepared.member_count);
8406 if self.config.formula_plane_mode
8407 == FormulaPlaneMode::AuthoritativeExperimental
8408 {
8409 disposition.set_family_direct(family.source_id);
8410 }
8411 ordered_placements.push((family.source_order, vec![prepared]));
8412 }
8413 Err(error) => {
8414 anchor_parses =
8415 anchor_parses.saturating_add(u64::from(error.parse_attempted));
8416 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
8417 anchor_analyses =
8418 anchor_analyses.saturating_add(u64::from(error.analysis_created));
8419 *source_report
8420 .fallback_reasons
8421 .entry(error.reason)
8422 .or_default() += 1;
8423 }
8424 }
8425 }
8426
8427 for source in &partitions {
8428 self.target_preparation_checkpoint(deadline, 1)?;
8429 if invalidated.contains(&source.source_id) {
8430 continue;
8431 }
8432 let prepared =
8433 match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
8434 Ok(prepared) => {
8435 anchor_parses = anchor_parses.saturating_add(1);
8436 anchor_asts = anchor_asts.saturating_add(1);
8437 anchor_analyses = anchor_analyses.saturating_add(1);
8438 prepared
8439 }
8440 Err(error) => {
8441 anchor_parses =
8442 anchor_parses.saturating_add(u64::from(error.parse_attempted));
8443 anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
8444 anchor_analyses =
8445 anchor_analyses.saturating_add(u64::from(error.analysis_created));
8446 *source_report
8447 .fallback_reasons
8448 .entry(error.reason)
8449 .or_default() += 1;
8450 continue;
8451 }
8452 };
8453 let fragment_regions: Vec<_> = prepared
8454 .placements
8455 .iter()
8456 .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
8457 .collect();
8458 let crosses_fragment =
8459 prepared
8460 .placements
8461 .iter()
8462 .enumerate()
8463 .any(|(fragment_index, placement)| {
8464 placement
8465 .fragment_dependency_proof()
8466 .1
8467 .dependencies
8468 .iter()
8469 .any(|dependency| {
8470 fragment_regions.iter().enumerate().any(
8471 |(candidate_index, candidate)| {
8472 candidate_index != fragment_index
8473 && dependency.read_region.intersects(candidate)
8474 },
8475 )
8476 })
8477 });
8478 let family_records = replay_records
8479 .iter()
8480 .filter(|record| {
8481 record.family == Some(source.source_id)
8482 || record.partition_owner == Some(source.source_id)
8483 })
8484 .cloned()
8485 .collect::<Vec<_>>();
8486 if crosses_fragment {
8487 *source_report
8488 .fallback_reasons
8489 .entry("CrossFragmentDependency".to_string())
8490 .or_default() += 1;
8491 continue;
8492 }
8493 if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
8494 *source_report
8495 .fallback_reasons
8496 .entry(format!("TargetPartitionReplay:{error}"))
8497 .or_default() += 1;
8498 continue;
8499 }
8500 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
8501 let mut candidate = disposition.clone();
8502 if let Err(reason) = candidate.register_partition(source, true) {
8503 *source_report
8504 .fallback_reasons
8505 .entry(reason.to_string())
8506 .or_default() += 1;
8507 continue;
8508 }
8509 disposition = candidate;
8510 }
8511 direct_families = direct_families.saturating_add(1);
8512 direct_cells = direct_cells.saturating_add(prepared.direct_cells);
8513 direct_partition_families = direct_partition_families.saturating_add(1);
8514 direct_partition_cells =
8515 direct_partition_cells.saturating_add(prepared.direct_cells);
8516 direct_fragments =
8517 direct_fragments.saturating_add(prepared.placements.len() as u64);
8518 ordered_placements.push((source.source_order, prepared.placements));
8519 }
8520 }
8521 ordered_placements.sort_by_key(|(source_order, _)| *source_order);
8522 let placements = ordered_placements
8523 .into_iter()
8524 .flat_map(|(_, placements)| placements)
8525 .collect();
8526
8527 Ok(PreparedTargetSourcePackage {
8528 sheet: sheet.to_string(),
8529 sheet_id,
8530 lease,
8531 source_report,
8532 replay_records,
8533 disposition,
8534 placements,
8535 legacy: Vec::new(),
8536 direct_families,
8537 direct_cells,
8538 direct_fragments,
8539 direct_complete_families,
8540 direct_complete_cells,
8541 direct_partition_families,
8542 direct_partition_cells,
8543 anchor_parses,
8544 anchor_asts,
8545 anchor_analyses,
8546 })
8547 }
8548
8549 fn fallback_planning_snapshot(
8550 &self,
8551 batch: &FormulaIngestBatch,
8552 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
8553 let mut requests = Vec::new();
8554 for formula in &batch.formulas {
8555 let ast = self
8556 .graph
8557 .data_store()
8558 .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
8559 .ok_or_else(|| {
8560 ExcelError::new(ExcelErrorKind::Value)
8561 .with_message("ordered fallback AST is unavailable")
8562 })?;
8563 Self::collect_planning_function_requests(&ast, &mut requests);
8564 }
8565 requests.sort();
8566 requests.dedup();
8567 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
8568 &self.resolver,
8569 requests,
8570 )
8571 .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
8572 }
8573
8574 fn prepare_legacy_batch_fallback(
8575 &mut self,
8576 batch: FormulaIngestBatch,
8577 function_provider: &dyn crate::traits::FunctionProvider,
8578 ) -> Result<
8579 (
8580 crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
8581 u64,
8582 ),
8583 ExcelError,
8584 > {
8585 let formula_count = batch.formulas.len() as u64;
8586 let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
8587 let mut planned = Vec::with_capacity(batch.formulas.len());
8588 for record in batch.formulas {
8589 let placement = CellRef::new(
8590 sheet_id,
8591 Coord::from_excel(record.row, record.col, true, true),
8592 );
8593 let ingested = self
8594 .graph
8595 .ingest_pipeline(function_provider)
8596 .enable_function_semantics()
8597 .ingest_formula(
8598 FormulaAstInput::RawArena(record.ast_id),
8599 placement,
8600 record.formula_text,
8601 )
8602 .map_err(|error| {
8603 ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
8604 })?;
8605 planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
8606 }
8607 let plan = self
8608 .graph
8609 .prepare_legacy_graph_plan(sheet_id, planned)
8610 .map_err(|error| {
8611 ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
8612 })?;
8613 Ok((plan, formula_count))
8614 }
8615
8616 fn publish_compressed_partial_report(
8617 &mut self,
8618 report: &FormulaIngestReport,
8619 direct_report: &FormulaIngestReport,
8620 ) {
8621 if direct_report.source_family_promoted == 0
8622 && direct_report.graph_formula_cells_materialized == 0
8623 {
8624 return;
8625 }
8626 let mut published = report.clone();
8627 published.accumulate(direct_report);
8628 self.record_formula_ingest_report(published);
8629 }
8630
8631 fn finish_compressed_formula_sources(
8632 &mut self,
8633 batches: Vec<(
8634 FormulaIngestBatch,
8635 crate::engine::FormulaCompressedSourceReport,
8636 crate::engine::FormulaCompressedPreparation,
8637 )>,
8638 ) -> Result<FormulaIngestReport, ExcelError> {
8639 self.observe_function_semantic_epoch()?;
8640 if batches.iter().any(|(_, _, preparation)| {
8641 !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
8642 }) {
8643 return Err(ExcelError::new(ExcelErrorKind::Value)
8644 .with_message("compressed source preparation belongs to another engine"));
8645 }
8646 if batches.iter().any(|(fallback, _, preparation)| {
8647 preparation.sheet_name.as_ref() != fallback.sheet_name
8648 }) {
8649 return Err(ExcelError::new(ExcelErrorKind::Value)
8650 .with_message("compressed source preparation sheet mismatch"));
8651 }
8652 let initial_guard = crate::function_registry::semantic_epoch_read_guard();
8653 let initial_provider_revision = self.resolver.planning_semantic_revision();
8654 let mut fallback_batches = Vec::with_capacity(batches.len());
8655 let mut stale_fallback_batches = Vec::new();
8656 let mut pending_preparations = Vec::new();
8657 for (mut fallback, mut source, mut preparation) in batches {
8658 for formula in fallback.formulas.drain(..) {
8659 let source_order = formula.source_order.ok_or_else(|| {
8660 ExcelError::new(ExcelErrorKind::Value).with_message(
8661 "compressed source supplied formulas without source-order proof",
8662 )
8663 })?;
8664 let text = formula.formula_text.ok_or_else(|| {
8665 ExcelError::new(ExcelErrorKind::Value).with_message(
8666 "ordered compressed fallback formula has no exact source text",
8667 )
8668 })?;
8669 preparation
8670 .eager_replay
8671 .push(crate::engine::DeferredReplayFormula {
8672 source_order,
8673 row: formula.row,
8674 col: formula.col,
8675 text: text.to_string(),
8676 family: formula.source_family,
8677 partition_owner: formula.partition_owner,
8678 });
8679 }
8680 preparation
8681 .eager_replay
8682 .sort_by_key(|record| record.source_order);
8683 source.source_spool_replays = source
8684 .source_spool_replays
8685 .saturating_add(preparation.preparation_spool_replays);
8686 let stale_reason = preparation
8687 .function_semantics_used
8688 .then(|| {
8689 if preparation.function_provider_revision != initial_provider_revision {
8690 Some("FunctionProviderRevisionChanged")
8691 } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
8692 {
8693 Some("FunctionSemanticEpochChanged")
8694 } else {
8695 None
8696 }
8697 })
8698 .flatten();
8699 let stale_semantics = stale_reason.is_some();
8700 if let Some(reason) = stale_reason {
8701 for (family, _, _) in &preparation.prepared {
8702 preparation.rejected.insert(*family, reason.to_string());
8703 }
8704 }
8705 for reason in preparation.rejected.values() {
8706 *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
8707 }
8708 let clean_direct_cells: u64 = preparation
8709 .prepared
8710 .iter()
8711 .map(|(_, _, prepared)| prepared.member_count)
8712 .sum();
8713 let fragmented_direct_cells: u64 = preparation
8714 .fragmented
8715 .iter()
8716 .map(|fragment| fragment.source.surviving_member_count)
8717 .sum();
8718 let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
8719 source.replay_families = source.families_seen.saturating_sub(
8720 preparation
8721 .prepared
8722 .len()
8723 .saturating_add(preparation.fragmented.len()) as u64,
8724 );
8725 source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
8726 let compressed = crate::engine::FormulaCompressedSourceBatch::new(
8727 fallback.sheet_name.clone(),
8728 source,
8729 );
8730 if stale_semantics {
8731 stale_fallback_batches.push((fallback, compressed));
8732 } else {
8733 fallback_batches.push((fallback, compressed));
8734 }
8735 pending_preparations.push((preparation, stale_semantics));
8736 }
8737 drop(initial_guard);
8738
8739 let configured_mode = self.config.formula_plane_mode;
8741 self.config.formula_plane_mode = FormulaPlaneMode::Off;
8742 let stale_result =
8743 self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
8744 self.config.formula_plane_mode = configured_mode;
8745 let mut report = stale_result?;
8746 report.mode = configured_mode;
8747
8748 self.config.formula_plane_mode = FormulaPlaneMode::Off;
8749 let fallback_result =
8750 self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
8751 self.config.formula_plane_mode = configured_mode;
8752 match fallback_result {
8753 Ok(fallback_report) => report.accumulate(&fallback_report),
8754 Err(error) => {
8755 self.record_formula_ingest_report(report);
8756 return Err(error);
8757 }
8758 }
8759
8760 let mut direct_report =
8761 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
8762 for (preparation, _) in &pending_preparations {
8763 direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
8764 preparation.clean_rejected_anchor_counts[0]
8765 .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
8766 );
8767 direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
8768 preparation.clean_rejected_anchor_counts[1]
8769 .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
8770 );
8771 direct_report.source_anchor_analyses =
8772 direct_report.source_anchor_analyses.saturating_add(
8773 preparation.clean_rejected_anchor_counts[2]
8774 .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
8775 );
8776 let rejected_partitions: Vec<_> = preparation
8777 .rejected
8778 .keys()
8779 .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
8780 .collect();
8781 direct_report.source_partitioned_families_rejected = direct_report
8782 .source_partitioned_families_rejected
8783 .saturating_add(rejected_partitions.len() as u64);
8784 direct_report.source_partition_failures = direct_report
8785 .source_partition_failures
8786 .saturating_add(rejected_partitions.len() as u64);
8787 direct_report.source_partition_fallback_cells = direct_report
8788 .source_partition_fallback_cells
8789 .saturating_add(
8790 rejected_partitions
8791 .iter()
8792 .map(|source| source.surviving_member_count)
8793 .sum::<u64>(),
8794 );
8795 }
8796 loop {
8797 #[cfg(any(test, feature = "test-support"))]
8798 if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8799 hook();
8800 }
8801 let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8802 let commit_provider_revision = self.resolver.planning_semantic_revision();
8803 let mut newly_stale = Vec::new();
8804 let mut current = Vec::new();
8805 for pending in pending_preparations.drain(..) {
8806 let stale_reason = pending
8807 .0
8808 .function_semantics_used
8809 .then(|| {
8810 if pending.0.function_provider_revision != commit_provider_revision {
8811 Some("FunctionProviderRevisionChanged")
8812 } else if self
8813 .prepared_function_semantics_changed(&pending.0, &commit_guard)
8814 {
8815 Some("FunctionSemanticEpochChanged")
8816 } else {
8817 None
8818 }
8819 })
8820 .flatten();
8821 if let Some(reason) = stale_reason {
8822 newly_stale.push((pending, reason));
8823 } else {
8824 current.push(pending);
8825 }
8826 }
8827 if !newly_stale.is_empty() {
8828 drop(commit_guard);
8829 for ((mut preparation, was_initially_stale), reason) in newly_stale {
8830 for (family, _, _) in &preparation.prepared {
8831 preparation.rejected.insert(*family, reason.to_string());
8832 }
8833 let exact = match self.replay_prepared_families_exact_records(&preparation) {
8834 Ok(exact) => exact,
8835 Err(error) => {
8836 self.publish_compressed_partial_report(&report, &direct_report);
8837 return Err(error);
8838 }
8839 };
8840 preparation.eager_replay.extend(exact);
8841 preparation
8842 .eager_replay
8843 .sort_by_key(|record| record.source_order);
8844 direct_report.source_spool_replays =
8845 direct_report.source_spool_replays.saturating_add(1);
8846 if !was_initially_stale {
8847 *direct_report
8848 .fallback_reasons
8849 .entry(reason.to_string())
8850 .or_default() += preparation.prepared.len() as u64;
8851 }
8852 direct_report.source_family_fallback = direct_report
8853 .source_family_fallback
8854 .saturating_add(preparation.prepared.len() as u64);
8855 direct_report.source_family_fallback_cells =
8856 direct_report.source_family_fallback_cells.saturating_add(
8857 preparation
8858 .prepared
8859 .iter()
8860 .map(|(_, _, prepared)| prepared.member_count)
8861 .sum(),
8862 );
8863 preparation.prepared.clear();
8864 preparation.function_semantics_used = false;
8865 if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8866 current.push((preparation, false));
8867 }
8868 }
8869 pending_preparations = current;
8870 continue;
8871 }
8872 drop(commit_guard);
8873
8874 enum SourceProposal {
8877 KnownFallback {
8878 source_order: crate::engine::SourceFormulaOrder,
8879 records: Vec<crate::engine::DeferredReplayFormula>,
8880 },
8881 Clean(
8882 crate::engine::SourceFamilyId,
8883 crate::engine::SourceFormulaOrder,
8884 crate::formula_plane::placement::PreparedAnchorOncePlacement,
8885 ),
8886 Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8887 }
8888
8889 impl SourceProposal {
8890 fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8891 match self {
8892 Self::KnownFallback { source_order, .. } => *source_order,
8893 Self::Clean(_, source_order, _) => *source_order,
8894 Self::Fragment(fragment) => fragment.source.source_order,
8895 }
8896 }
8897 }
8898
8899 for (mut preparation, _) in current {
8900 let mut proposals = Vec::with_capacity(
8901 preparation
8902 .prepared
8903 .len()
8904 .saturating_add(preparation.fragmented.len())
8905 .saturating_add(preparation.eager_replay.len()),
8906 );
8907 proposals.extend(preparation.prepared.drain(..).map(
8908 |(source_id, source_order, prepared)| {
8909 SourceProposal::Clean(source_id, source_order, prepared)
8910 },
8911 ));
8912
8913 let accepted_fragments: BTreeMap<_, _> = preparation
8914 .fragmented
8915 .iter()
8916 .map(|fragment| (fragment.source.source_id, &fragment.source))
8917 .collect();
8918 let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8919 for record in preparation.eager_replay.drain(..).filter(|record| {
8920 !record.partition_owner.is_some_and(|owner| {
8921 accepted_fragments.get(&owner).is_some_and(|source| {
8922 Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8923 })
8924 })
8925 }) {
8926 if let Some(owner) = record.partition_owner.or(record.family) {
8927 family_fallbacks.entry(owner).or_default().push(record);
8928 } else {
8929 proposals.push(SourceProposal::KnownFallback {
8930 source_order: record.source_order,
8931 records: vec![record],
8932 });
8933 }
8934 }
8935 for (_, mut records) in family_fallbacks {
8936 records.sort_by_key(|record| record.source_order);
8937 if records
8938 .windows(2)
8939 .any(|window| window[0].source_order == window[1].source_order)
8940 {
8941 self.publish_compressed_partial_report(&report, &direct_report);
8942 return Err(ExcelError::new(ExcelErrorKind::Value)
8943 .with_message("duplicate exact-replay source-order proof"));
8944 }
8945 let Some(source_order) = records.first().map(|record| record.source_order)
8946 else {
8947 self.publish_compressed_partial_report(&report, &direct_report);
8948 return Err(ExcelError::new(ExcelErrorKind::Value)
8949 .with_message("empty exact-replay fallback family"));
8950 };
8951 proposals.push(SourceProposal::KnownFallback {
8952 source_order,
8953 records,
8954 });
8955 }
8956 proposals.extend(
8957 preparation
8958 .fragmented
8959 .drain(..)
8960 .map(SourceProposal::Fragment),
8961 );
8962 proposals.sort_by_key(SourceProposal::source_order);
8963 if proposals
8964 .windows(2)
8965 .any(|window| window[0].source_order() == window[1].source_order())
8966 {
8967 self.publish_compressed_partial_report(&report, &direct_report);
8968 return Err(ExcelError::new(ExcelErrorKind::Value)
8969 .with_message("ambiguous compressed source-order proof"));
8970 }
8971
8972 for proposal in proposals {
8973 match proposal {
8974 SourceProposal::KnownFallback { records, .. } => {
8975 let batch = match self.formula_batch_from_exact_replay(
8976 preparation.sheet_name.as_ref(),
8977 records,
8978 ) {
8979 Ok(batch) => batch,
8980 Err(error) => {
8981 self.publish_compressed_partial_report(&report, &direct_report);
8982 return Err(error);
8983 }
8984 };
8985 if batch.is_empty() {
8986 continue;
8987 }
8988 let snapshot = match self.fallback_planning_snapshot(&batch) {
8989 Ok(snapshot) => snapshot,
8990 Err(error) => {
8991 self.publish_compressed_partial_report(&report, &direct_report);
8992 return Err(error);
8993 }
8994 };
8995 let commit_guard =
8996 crate::function_registry::semantic_epoch_read_guard();
8997 let provider_revision_initial =
8998 self.resolver.planning_semantic_revision();
8999 if (commit_guard.epoch() != snapshot.epoch()
9000 && snapshot.semantic_changes_affect_requests_since_guarded(
9001 &commit_guard,
9002 snapshot.epoch(),
9003 ))
9004 || snapshot.provider_revision().is_some_and(|revision| {
9005 Some(revision) != provider_revision_initial
9006 })
9007 {
9008 drop(commit_guard);
9009 self.publish_compressed_partial_report(&report, &direct_report);
9010 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9011 "ordered fallback planning snapshot became stale",
9012 ));
9013 }
9014 let (plan, formula_count) = match self
9015 .prepare_legacy_batch_fallback(batch, &snapshot)
9016 {
9017 Ok(plan) => plan,
9018 Err(error) => {
9019 drop(commit_guard);
9020 self.publish_compressed_partial_report(&report, &direct_report);
9021 return Err(error);
9022 }
9023 };
9024 let provider_revision_after =
9025 self.resolver.planning_semantic_revision();
9026 if provider_revision_after != provider_revision_initial {
9027 drop(commit_guard);
9028 self.publish_compressed_partial_report(&report, &direct_report);
9029 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9030 "function provider changed while preparing ordered fallback",
9031 ));
9032 }
9033 let graph_vertices = plan.new_vertex_count();
9034 let Some(graph_edges) = plan.planned_edge_count() else {
9035 drop(commit_guard);
9036 self.publish_compressed_partial_report(&report, &direct_report);
9037 return Err(ExcelError::new(ExcelErrorKind::Value)
9038 .with_message("prepared ordered fallback size overflow"));
9039 };
9040 if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
9041 {
9042 drop(commit_guard);
9043 self.publish_compressed_partial_report(&report, &direct_report);
9044 return Err(error);
9045 }
9046 if let Err(error) =
9047 self.graph.validate_prepared_legacy_graph_plan(&plan)
9048 {
9049 drop(commit_guard);
9050 self.publish_compressed_partial_report(&report, &direct_report);
9051 return Err(ExcelError::new(ExcelErrorKind::Value)
9052 .with_message(error.to_string()));
9053 }
9054 #[cfg(test)]
9055 if let Some(hook) = self
9056 .before_legacy_fallback_final_provider_sample_hook
9057 .take()
9058 {
9059 hook();
9060 }
9061 let provider_revision_final =
9062 self.resolver.planning_semantic_revision();
9063 if provider_revision_final != provider_revision_initial {
9064 drop(commit_guard);
9065 self.publish_compressed_partial_report(&report, &direct_report);
9066 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9067 "function provider changed after ordered fallback validation",
9068 ));
9069 }
9070 let graph_formulas =
9071 self.graph.apply_prevalidated_legacy_graph_plan(plan);
9072 direct_report.formula_cells_seen = direct_report
9073 .formula_cells_seen
9074 .saturating_add(formula_count);
9075 direct_report.graph_formula_cells_materialized = direct_report
9076 .graph_formula_cells_materialized
9077 .saturating_add(graph_formulas as u64);
9078 direct_report.graph_vertices_created = direct_report
9079 .graph_vertices_created
9080 .saturating_add(graph_vertices as u64);
9081 direct_report.graph_edges_created = direct_report
9082 .graph_edges_created
9083 .saturating_add(graph_edges as u64);
9084 }
9085 SourceProposal::Clean(source_id, _, prepared) => {
9086 let commit_guard =
9087 crate::function_registry::semantic_epoch_read_guard();
9088 let commit_provider_revision =
9089 self.resolver.planning_semantic_revision();
9090 let semantic_changed = preparation.function_semantics_used
9091 && self.prepared_placement_function_semantics_changed(
9092 preparation.function_semantic_epoch,
9093 &prepared,
9094 &commit_guard,
9095 );
9096 let cells = prepared.member_count;
9097 let stats = self.graph.baseline_stats();
9098 self.preflight_graph_admission(
9099 crate::engine::resource_ledger::GraphAdmission {
9100 final_vertices: stats.graph_vertex_count,
9101 final_edges: stats.graph_edge_count,
9102 materialization_cells: 0,
9103 added_vertices: 0,
9104 added_edges: 0,
9105 },
9106 )?;
9107 let append = self
9108 .graph
9109 .formula_authority()
9110 .prepare_formula_plane_append(
9111 vec![prepared],
9112 self.graph.data_store(),
9113 self.graph.sheet_reg(),
9114 )
9115 .map_err(|error| error.to_string())
9116 .and_then(|append| {
9117 if preparation.function_semantics_used
9118 && preparation.function_provider_revision
9119 != commit_provider_revision
9120 {
9121 return Err("FunctionProviderRevisionChanged".to_string());
9122 }
9123 if preparation.function_semantics_used
9124 && self.resolver.planning_semantic_revision()
9125 != commit_provider_revision
9126 {
9127 return Err("FunctionProviderRevisionChanged".to_string());
9128 }
9129 if semantic_changed {
9130 return Err("FunctionSemanticEpochChanged".to_string());
9131 }
9132 self.graph
9133 .formula_authority()
9134 .validate_prepared_formula_plane_append(
9135 &append,
9136 self.graph.data_store(),
9137 self.graph.sheet_reg(),
9138 )
9139 .map_err(|error| error.to_string())?;
9140 Ok(append)
9141 });
9142 match append {
9143 Ok(append) => {
9144 let placement_report = self
9145 .graph
9146 .formula_authority_mut()
9147 .apply_prevalidated_formula_plane_append(append);
9148 self.graph.mark_formula_spans_dirty(
9149 placement_report.spans.iter().copied(),
9150 WholeSpanDirtyReason::NewSpan,
9151 );
9152 direct_report.formula_cells_seen =
9153 direct_report.formula_cells_seen.saturating_add(cells);
9154 direct_report.shadow_candidate_cells =
9155 direct_report.shadow_candidate_cells.saturating_add(cells);
9156 direct_report.shadow_accepted_span_cells = direct_report
9157 .shadow_accepted_span_cells
9158 .saturating_add(cells);
9159 direct_report.shadow_templates_interned =
9160 direct_report.shadow_templates_interned.saturating_add(
9161 placement_report.work.templates_to_append as u64,
9162 );
9163 direct_report.shadow_spans_created = direct_report
9164 .shadow_spans_created
9165 .saturating_add(placement_report.spans.len() as u64);
9166 direct_report.graph_formula_vertices_avoided_shadow =
9167 direct_report
9168 .graph_formula_vertices_avoided_shadow
9169 .saturating_add(cells);
9170 direct_report.ast_roots_avoided_shadow = direct_report
9171 .ast_roots_avoided_shadow
9172 .saturating_add(cells.saturating_sub(1));
9173 direct_report.edge_rows_avoided_shadow = direct_report
9174 .edge_rows_avoided_shadow
9175 .saturating_add(cells);
9176 direct_report.source_family_promoted =
9177 direct_report.source_family_promoted.saturating_add(1);
9178 direct_report.source_family_promoted_cells = direct_report
9179 .source_family_promoted_cells
9180 .saturating_add(cells);
9181 direct_report.source_anchor_parses =
9182 direct_report.source_anchor_parses.saturating_add(1);
9183 direct_report.source_anchor_asts =
9184 direct_report.source_anchor_asts.saturating_add(1);
9185 direct_report.source_anchor_analyses =
9186 direct_report.source_anchor_analyses.saturating_add(1);
9187 let descendants = cells.saturating_sub(1);
9188 direct_report.source_descendant_strings_avoided = direct_report
9189 .source_descendant_strings_avoided
9190 .saturating_add(descendants);
9191 direct_report.source_descendant_events_avoided = direct_report
9192 .source_descendant_events_avoided
9193 .saturating_add(descendants);
9194 direct_report.source_descendant_analyses_avoided =
9195 direct_report
9196 .source_descendant_analyses_avoided
9197 .saturating_add(descendants);
9198 direct_report.source_compressed_families_prepared =
9199 direct_report
9200 .source_compressed_families_prepared
9201 .saturating_add(1);
9202 direct_report.source_compressed_cells_prepared = direct_report
9203 .source_compressed_cells_prepared
9204 .saturating_add(cells);
9205 }
9206 Err(error) => {
9207 drop(commit_guard);
9208 let reason = format!("CleanFormulaPlaneAppend:{error}");
9209 let batch = match self.replay_one_prepared_family_exact(
9210 &preparation,
9211 source_id,
9212 cells,
9213 ) {
9214 Ok(batch) => batch,
9215 Err(error) => {
9216 self.publish_compressed_partial_report(
9217 &report,
9218 &direct_report,
9219 );
9220 return Err(error);
9221 }
9222 };
9223 let snapshot = match self.fallback_planning_snapshot(&batch) {
9224 Ok(snapshot) => snapshot,
9225 Err(error) => {
9226 self.publish_compressed_partial_report(
9227 &report,
9228 &direct_report,
9229 );
9230 return Err(error);
9231 }
9232 };
9233 let commit_guard =
9234 crate::function_registry::semantic_epoch_read_guard();
9235 let provider_revision =
9236 self.resolver.planning_semantic_revision();
9237 if (commit_guard.epoch() != snapshot.epoch()
9238 && snapshot.semantic_changes_affect_requests_since_guarded(
9239 &commit_guard,
9240 snapshot.epoch(),
9241 ))
9242 || snapshot.provider_revision().is_some_and(|revision| {
9243 Some(revision) != provider_revision
9244 })
9245 {
9246 drop(commit_guard);
9247 self.publish_compressed_partial_report(
9248 &report,
9249 &direct_report,
9250 );
9251 return Err(ExcelError::new(ExcelErrorKind::Value)
9252 .with_message(
9253 "clean-family fallback planning snapshot became stale",
9254 ));
9255 }
9256 let (plan, formula_count) = match self
9257 .prepare_legacy_batch_fallback(batch, &snapshot)
9258 {
9259 Ok(plan) => plan,
9260 Err(error) => {
9261 drop(commit_guard);
9262 self.publish_compressed_partial_report(
9263 &report,
9264 &direct_report,
9265 );
9266 return Err(error);
9267 }
9268 };
9269 let provider_revision_after =
9270 self.resolver.planning_semantic_revision();
9271 if provider_revision_after != provider_revision {
9272 drop(commit_guard);
9273 self.publish_compressed_partial_report(
9274 &report,
9275 &direct_report,
9276 );
9277 return Err(ExcelError::new(ExcelErrorKind::Value)
9278 .with_message(
9279 "function provider changed while preparing clean-family fallback",
9280 ));
9281 }
9282 let graph_vertices = plan.new_vertex_count();
9283 let Some(graph_edges) = plan.planned_edge_count() else {
9284 drop(commit_guard);
9285 self.publish_compressed_partial_report(
9286 &report,
9287 &direct_report,
9288 );
9289 return Err(ExcelError::new(ExcelErrorKind::Value)
9290 .with_message(
9291 "prepared clean-family fallback size overflow",
9292 ));
9293 };
9294 if let Err(error) =
9295 self.prepared_legacy_admission(&plan, formula_count)
9296 {
9297 drop(commit_guard);
9298 self.publish_compressed_partial_report(
9299 &report,
9300 &direct_report,
9301 );
9302 return Err(error);
9303 }
9304 if let Err(error) =
9305 self.graph.validate_prepared_legacy_graph_plan(&plan)
9306 {
9307 drop(commit_guard);
9308 self.publish_compressed_partial_report(
9309 &report,
9310 &direct_report,
9311 );
9312 return Err(ExcelError::new(ExcelErrorKind::Value)
9313 .with_message(error.to_string()));
9314 }
9315 #[cfg(test)]
9316 if let Some(hook) = self
9317 .before_legacy_fallback_final_provider_sample_hook
9318 .take()
9319 {
9320 hook();
9321 }
9322 let provider_revision_final =
9323 self.resolver.planning_semantic_revision();
9324 if provider_revision_final != provider_revision {
9325 drop(commit_guard);
9326 self.publish_compressed_partial_report(
9327 &report,
9328 &direct_report,
9329 );
9330 return Err(ExcelError::new(ExcelErrorKind::Value)
9331 .with_message(
9332 "function provider changed after clean-family fallback validation",
9333 ));
9334 }
9335 let graph_formulas =
9336 self.graph.apply_prevalidated_legacy_graph_plan(plan);
9337 direct_report.formula_cells_seen = direct_report
9338 .formula_cells_seen
9339 .saturating_add(formula_count);
9340 direct_report.graph_formula_cells_materialized = direct_report
9341 .graph_formula_cells_materialized
9342 .saturating_add(graph_formulas as u64);
9343 direct_report.graph_vertices_created = direct_report
9344 .graph_vertices_created
9345 .saturating_add(graph_vertices as u64);
9346 direct_report.graph_edges_created = direct_report
9347 .graph_edges_created
9348 .saturating_add(graph_edges as u64);
9349 direct_report.source_spool_replays =
9350 direct_report.source_spool_replays.saturating_add(1);
9351 direct_report.source_family_fallback =
9352 direct_report.source_family_fallback.saturating_add(1);
9353 direct_report.source_family_fallback_cells = direct_report
9354 .source_family_fallback_cells
9355 .saturating_add(cells);
9356 direct_report.source_anchor_parses =
9357 direct_report.source_anchor_parses.saturating_add(1);
9358 direct_report.source_anchor_asts =
9359 direct_report.source_anchor_asts.saturating_add(1);
9360 direct_report.source_anchor_analyses =
9361 direct_report.source_anchor_analyses.saturating_add(1);
9362 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
9363 }
9364 }
9365 }
9366 SourceProposal::Fragment(fragment) => {
9367 let commit_guard =
9368 crate::function_registry::semantic_epoch_read_guard();
9369 let commit_epoch = commit_guard.epoch();
9370 let commit_provider_revision =
9371 self.resolver.planning_semantic_revision();
9372 let source = fragment.source;
9373 let source_id = source.source_id;
9374 #[cfg(test)]
9375 let commit_fault = self
9376 .fragmented_commit_fault_for_test
9377 .take()
9378 .unwrap_or(
9379 crate::engine::fragmented_transaction::FragmentedCommitFault::None,
9380 );
9381 #[cfg(not(test))]
9382 let commit_fault =
9383 crate::engine::fragmented_transaction::FragmentedCommitFault::None;
9384 let outcome = self
9385 .prepare_fragmented_source_transaction(
9386 &source,
9387 &preparation.replay_disposition,
9388 fragment.prepared,
9389 fragment.legacy,
9390 )
9391 .map(|transaction| {
9392 let resolver = &self.resolver;
9393 self.graph.commit_fragmented_source_transaction(
9394 transaction,
9395 &self.source_formula_token,
9396 &preparation.replay_disposition,
9397 commit_epoch,
9398 commit_provider_revision,
9399 || resolver.planning_semantic_revision(),
9400 commit_fault,
9401 )
9402 });
9403 let fallback_reason = match outcome {
9404 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
9405 let mut delta = success.report_delta;
9406 delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
9407 delta.source_family_promoted = 1;
9408 delta.source_family_promoted_cells =
9409 source.surviving_member_count;
9410 delta.source_partitioned_families_seen = 0;
9411 delta.source_partition_holes = 0;
9412 delta.source_partition_ordinary_exceptions = 0;
9413 delta.source_partition_surviving_cells = 0;
9414 delta.shadow_templates_interned =
9415 success.plane.work.templates_to_append as u64;
9416 delta.shadow_spans_created = success.plane.spans.len() as u64;
9417 let descendants =
9418 source.surviving_member_count.saturating_sub(1);
9419 delta.source_descendant_strings_avoided = descendants;
9420 delta.source_descendant_events_avoided = descendants;
9421 delta.source_descendant_analyses_avoided = descendants;
9422 direct_report.accumulate(&delta);
9423 None
9424 }
9425 Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
9426 Some(format!("{reason:?}"))
9427 }
9428 Err(error) => Some(format!("{error:?}")),
9429 };
9430 if let Some(reason) = fallback_reason {
9431 drop(commit_guard);
9432 let mut replay_disposition = preparation.replay_disposition.clone();
9433 replay_disposition.force_family_legacy(source_id);
9434 let replayed = (|| {
9435 let records = fragment
9436 .replay
9437 .lock()
9438 .map_err(|_| {
9439 ExcelError::new(ExcelErrorKind::Value).with_message(
9440 "compressed formula exact replay lock poisoned",
9441 )
9442 })?
9443 .replay_partitioned(
9444 &replay_disposition,
9445 std::slice::from_ref(&source),
9446 )
9447 .map_err(|message| {
9448 ExcelError::new(ExcelErrorKind::Value)
9449 .with_message(message)
9450 })?
9451 .into_iter()
9452 .filter(|record| {
9453 record.family == Some(source_id)
9454 || (record.family.is_none()
9455 && record.partition_owner == Some(source_id))
9456 })
9457 .collect::<Vec<_>>();
9458 Self::validate_whole_partition_replay(&source, &records)?;
9459 Ok::<_, ExcelError>(records)
9460 })();
9461 let replayed = match replayed {
9462 Ok(replayed) => replayed,
9463 Err(error) => {
9464 self.publish_compressed_partial_report(
9465 &report,
9466 &direct_report,
9467 );
9468 return Err(error);
9469 }
9470 };
9471 let batch = match self.formula_batch_from_exact_replay(
9472 preparation.sheet_name.as_ref(),
9473 replayed,
9474 ) {
9475 Ok(batch) => batch,
9476 Err(error) => {
9477 self.publish_compressed_partial_report(
9478 &report,
9479 &direct_report,
9480 );
9481 return Err(error);
9482 }
9483 };
9484 let snapshot = match self.fallback_planning_snapshot(&batch) {
9485 Ok(snapshot) => snapshot,
9486 Err(error) => {
9487 self.publish_compressed_partial_report(
9488 &report,
9489 &direct_report,
9490 );
9491 return Err(error);
9492 }
9493 };
9494 let commit_guard =
9495 crate::function_registry::semantic_epoch_read_guard();
9496 let provider_revision = self.resolver.planning_semantic_revision();
9497 if (commit_guard.epoch() != snapshot.epoch()
9498 && snapshot.semantic_changes_affect_requests_since_guarded(
9499 &commit_guard,
9500 snapshot.epoch(),
9501 ))
9502 || snapshot
9503 .provider_revision()
9504 .is_some_and(|revision| Some(revision) != provider_revision)
9505 {
9506 self.publish_compressed_partial_report(&report, &direct_report);
9507 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9508 format!(
9509 "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
9510 snapshot.epoch(),
9511 commit_guard.epoch(),
9512 snapshot.provider_revision(),
9513 provider_revision,
9514 ),
9515 ));
9516 }
9517 let (plan, formula_count) =
9518 match self.prepare_legacy_batch_fallback(batch, &snapshot) {
9519 Ok(plan) => plan,
9520 Err(error) => {
9521 drop(commit_guard);
9522 self.publish_compressed_partial_report(
9523 &report,
9524 &direct_report,
9525 );
9526 return Err(error);
9527 }
9528 };
9529 let provider_revision_after =
9530 self.resolver.planning_semantic_revision();
9531 if provider_revision_after != provider_revision {
9532 drop(commit_guard);
9533 self.publish_compressed_partial_report(&report, &direct_report);
9534 return Err(ExcelError::new(ExcelErrorKind::Value)
9535 .with_message(
9536 "function provider changed while preparing whole-family fallback",
9537 ));
9538 }
9539 let graph_vertices = plan.new_vertex_count();
9540 let Some(graph_edges) = plan.planned_edge_count() else {
9541 drop(commit_guard);
9542 self.publish_compressed_partial_report(&report, &direct_report);
9543 return Err(ExcelError::new(ExcelErrorKind::Value)
9544 .with_message(
9545 "prepared whole-family fallback size overflow",
9546 ));
9547 };
9548 if let Err(error) =
9549 self.prepared_legacy_admission(&plan, formula_count)
9550 {
9551 drop(commit_guard);
9552 self.publish_compressed_partial_report(&report, &direct_report);
9553 return Err(error);
9554 }
9555 if let Err(error) =
9556 self.graph.validate_prepared_legacy_graph_plan(&plan)
9557 {
9558 drop(commit_guard);
9559 self.publish_compressed_partial_report(&report, &direct_report);
9560 return Err(ExcelError::new(ExcelErrorKind::Value)
9561 .with_message(error.to_string()));
9562 }
9563 #[cfg(test)]
9564 if let Some(hook) = self
9565 .before_legacy_fallback_final_provider_sample_hook
9566 .take()
9567 {
9568 hook();
9569 }
9570 let provider_revision_final =
9571 self.resolver.planning_semantic_revision();
9572 if provider_revision_final != provider_revision {
9573 drop(commit_guard);
9574 self.publish_compressed_partial_report(&report, &direct_report);
9575 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9576 "function provider changed after whole-family fallback validation",
9577 ));
9578 }
9579 let graph_formulas =
9580 self.graph.apply_prevalidated_legacy_graph_plan(plan);
9581 direct_report.formula_cells_seen = direct_report
9582 .formula_cells_seen
9583 .saturating_add(formula_count);
9584 direct_report.graph_formula_cells_materialized = direct_report
9585 .graph_formula_cells_materialized
9586 .saturating_add(graph_formulas as u64);
9587 direct_report.graph_vertices_created = direct_report
9588 .graph_vertices_created
9589 .saturating_add(graph_vertices as u64);
9590 direct_report.graph_edges_created = direct_report
9591 .graph_edges_created
9592 .saturating_add(graph_edges as u64);
9593 direct_report.source_spool_replays =
9594 direct_report.source_spool_replays.saturating_add(1);
9595 direct_report.source_family_fallback =
9596 direct_report.source_family_fallback.saturating_add(1);
9597 direct_report.source_family_fallback_cells = direct_report
9598 .source_family_fallback_cells
9599 .saturating_add(source.surviving_member_count);
9600 direct_report.source_partitioned_families_rejected = direct_report
9601 .source_partitioned_families_rejected
9602 .saturating_add(1);
9603 direct_report.source_partition_failures =
9604 direct_report.source_partition_failures.saturating_add(1);
9605 direct_report.source_partition_fallback_cells = direct_report
9606 .source_partition_fallback_cells
9607 .saturating_add(source.surviving_member_count);
9608 direct_report.source_anchor_parses =
9609 direct_report.source_anchor_parses.saturating_add(1);
9610 direct_report.source_anchor_asts =
9611 direct_report.source_anchor_asts.saturating_add(1);
9612 direct_report.source_anchor_analyses =
9613 direct_report.source_anchor_analyses.saturating_add(1);
9614 *direct_report.fallback_reasons.entry(reason).or_default() += 1;
9615 }
9616 }
9617 }
9618 #[cfg(test)]
9619 if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
9620 hook();
9621 }
9622 }
9623 }
9624 break;
9625 }
9626 report.accumulate(&direct_report);
9627 self.record_formula_ingest_report(report.clone());
9628 Ok(report)
9629 }
9630 pub(crate) fn ingest_compressed_formula_source_batches(
9632 &mut self,
9633 batches: Vec<(
9634 FormulaIngestBatch,
9635 crate::engine::FormulaCompressedSourceBatch,
9636 )>,
9637 ) -> Result<FormulaIngestReport, ExcelError> {
9638 self.ingest_compressed_formula_source_batches_inner(batches, true)
9639 }
9640
9641 fn ingest_compressed_formula_source_batches_inner(
9642 &mut self,
9643 batches: Vec<(
9644 FormulaIngestBatch,
9645 crate::engine::FormulaCompressedSourceBatch,
9646 )>,
9647 publish_report: bool,
9648 ) -> Result<FormulaIngestReport, ExcelError> {
9649 let mut source_counts = [0_u64; 11];
9650 let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
9651 let mut formula_batches = Vec::with_capacity(batches.len());
9652 let mut compressed_families = Vec::new();
9653 let mut partitioned_families = Vec::new();
9654 for (batch, compressed_batch) in batches {
9655 let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
9656 if sheet_name.as_ref() != batch.sheet_name {
9657 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9658 "compressed formula source sheet does not match its replay batch",
9659 ));
9660 }
9661 compressed_families.push((batch.sheet_name.clone(), families));
9662 partitioned_families.push((batch.sheet_name.clone(), partitions));
9663 source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
9664 source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
9665 source_counts[2] =
9666 source_counts[2].saturating_add(compressed.source_shared_anchor_events);
9667 source_counts[3] =
9668 source_counts[3].saturating_add(compressed.source_shared_descendant_events);
9669 source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
9670 source_counts[5] =
9671 source_counts[5].saturating_add(compressed.source_formula_records_spooled);
9672 source_counts[6] =
9673 source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
9674 source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
9675 source_counts[8] =
9676 source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
9677 source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
9678 source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
9679 source_report.families_seen = source_report
9680 .families_seen
9681 .saturating_add(compressed.families_seen);
9682 source_report.family_cells_seen = source_report
9683 .family_cells_seen
9684 .saturating_add(compressed.family_cells_seen);
9685 source_report.source_clean_families = source_report
9686 .source_clean_families
9687 .saturating_add(compressed.source_clean_families);
9688 source_report.source_clean_cells = source_report
9689 .source_clean_cells
9690 .saturating_add(compressed.source_clean_cells);
9691 source_report.source_fragmentable_families = source_report
9692 .source_fragmentable_families
9693 .saturating_add(compressed.source_fragmentable_families);
9694 source_report.source_fragmentable_cells = source_report
9695 .source_fragmentable_cells
9696 .saturating_add(compressed.source_fragmentable_cells);
9697 source_report.source_fragment_count = source_report
9698 .source_fragment_count
9699 .saturating_add(compressed.source_fragment_count);
9700 source_report.source_isolated_fallback_cells = source_report
9701 .source_isolated_fallback_cells
9702 .saturating_add(compressed.source_isolated_fallback_cells);
9703 source_report.source_hole_exclusions = source_report
9704 .source_hole_exclusions
9705 .saturating_add(compressed.source_hole_exclusions);
9706 source_report.source_ordinary_exclusions = source_report
9707 .source_ordinary_exclusions
9708 .saturating_add(compressed.source_ordinary_exclusions);
9709 source_report.source_partition_failures = source_report
9710 .source_partition_failures
9711 .saturating_add(compressed.source_partition_failures);
9712 source_report.replay_families = source_report
9713 .replay_families
9714 .saturating_add(compressed.replay_families);
9715 source_report.replay_cells = source_report
9716 .replay_cells
9717 .saturating_add(compressed.replay_cells);
9718 source_report.forward_descendants = source_report
9719 .forward_descendants
9720 .saturating_add(compressed.forward_descendants);
9721 source_report.evidence_limit_fallbacks = source_report
9722 .evidence_limit_fallbacks
9723 .saturating_add(compressed.evidence_limit_fallbacks);
9724 source_report.evidence_peak_bytes = source_report
9725 .evidence_peak_bytes
9726 .max(compressed.evidence_peak_bytes);
9727 for (reason, count) in compressed.fallback_reasons {
9728 *source_report.fallback_reasons.entry(reason).or_default() += count;
9729 }
9730 formula_batches.push(batch);
9731 }
9732 self.ingest_formula_batches_inner(
9733 formula_batches,
9734 source_counts,
9735 Some(source_report),
9736 compressed_families,
9737 partitioned_families,
9738 publish_report,
9739 )
9740 }
9741
9742 pub fn ingest_formula_batches(
9743 &mut self,
9744 batches: Vec<FormulaIngestBatch>,
9745 ) -> Result<FormulaIngestReport, ExcelError> {
9746 let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
9747 let report = self.ingest_formula_batches_inner(
9748 batches,
9749 [0; 11],
9750 None,
9751 Vec::new(),
9752 Vec::new(),
9753 true,
9754 )?;
9755 if has_formulas {
9756 self.mark_topology_edited();
9757 }
9758 Ok(report)
9759 }
9760
9761 fn ingest_formula_batches_unpublished(
9762 &mut self,
9763 batches: Vec<FormulaIngestBatch>,
9764 ) -> Result<FormulaIngestReport, ExcelError> {
9765 self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
9766 }
9767
9768 fn ingest_formula_batches_inner(
9769 &mut self,
9770 batches: Vec<FormulaIngestBatch>,
9771 source_counts: [u64; 11],
9772 source_report: Option<crate::engine::FormulaCompressedSourceReport>,
9773 compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
9774 partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
9775 publish_report: bool,
9776 ) -> Result<FormulaIngestReport, ExcelError> {
9777 self.observe_function_semantic_epoch()?;
9778 let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
9779 #[cfg(feature = "benchmark_internal")]
9780 let benchmark_forced_replay =
9781 std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9782 #[cfg(not(feature = "benchmark_internal"))]
9783 let benchmark_forced_replay = false;
9784 let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9785 && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9786 {
9787 let mut report =
9788 FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9789 report
9790 .fallback_reasons
9791 .insert("ForcedReplay".to_string(), formula_cells_seen);
9792 (report, batches, BTreeMap::new())
9793 } else {
9794 match self.config.formula_plane_mode {
9795 FormulaPlaneMode::Off => (
9796 FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9797 batches,
9798 BTreeMap::new(),
9799 ),
9800 FormulaPlaneMode::Shadow => (
9801 if partitioned_families
9802 .iter()
9803 .all(|(_, families)| families.is_empty())
9804 {
9805 if compressed_families.is_empty() {
9806 self.analyze_formula_plane_shadow_candidates(&batches)
9807 } else {
9808 self.analyze_compressed_formula_plane_shadow(&compressed_families)
9809 }
9810 } else {
9811 let mut partition_report =
9812 self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9813 if compressed_families
9814 .iter()
9815 .any(|(_, families)| !families.is_empty())
9816 {
9817 partition_report.accumulate(
9818 &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9819 );
9820 }
9821 partition_report
9822 },
9823 batches,
9824 BTreeMap::new(),
9825 ),
9826 FormulaPlaneMode::AuthoritativeExperimental => {
9827 self.analyze_formula_plane_authoritative_ingest(&batches)
9828 }
9829 }
9830 };
9831 report.formula_cells_seen = formula_cells_seen;
9832 report.source_formula_events = source_counts[0];
9833 report.source_ordinary_events = source_counts[1];
9834 report.source_shared_anchor_events = source_counts[2];
9835 report.source_shared_descendant_events = source_counts[3];
9836 report.source_unknown_events = source_counts[4];
9837 report.source_formula_records_spooled = source_counts[5];
9838 report.source_spool_encoded_bytes = source_counts[6];
9839 report.source_spool_peak_memory_bytes = source_counts[7];
9840 report.source_spool_spilled_bytes = source_counts[8];
9841 report.source_spool_spill_files = source_counts[9];
9842 report.source_spool_replays = source_counts[10];
9843 if let Some(source) = source_report {
9844 report.source_families_seen = source.families_seen;
9845 report.source_family_cells_seen = source.family_cells_seen;
9846 report.source_family_shadow_eligible = source.source_clean_families;
9847 report.source_family_shadow_eligible_cells = source.source_clean_cells;
9848 report.source_partitioned_families_seen = source.source_fragmentable_families;
9849 report.source_partition_holes = source.source_hole_exclusions;
9850 report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9851 report.source_partition_failures = source.source_partition_failures;
9852 report.source_partition_surviving_cells = source.source_fragmentable_cells;
9853 report.source_family_fallback = report
9854 .source_family_fallback
9855 .saturating_add(source.replay_families);
9856 report.source_family_fallback_cells = report
9857 .source_family_fallback_cells
9858 .saturating_add(source.replay_cells);
9859 report.source_forward_descendants = source.forward_descendants;
9860 report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9861 report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9862 for (reason, count) in source.fallback_reasons {
9863 let total = report.fallback_reasons.entry(reason).or_default();
9864 *total = total.saturating_add(count);
9865 }
9866 }
9867
9868 if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9869 || !planned_materialize.is_empty()
9870 {
9871 let mut prepared_by_sheet = planned_materialize;
9872 for batch in materialize_batches {
9873 if batch.is_empty() {
9874 continue;
9875 }
9876 let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9877 ExcelError::new(ExcelErrorKind::Ref)
9878 .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9879 })?;
9880 let mut pipeline = self.ingest_pipeline();
9881 let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9882 let placement = CellRef::new(
9883 sheet_id,
9884 Coord::from_excel(record.row, record.col, true, true),
9885 );
9886 (
9887 FormulaAstInput::RawArena(record.ast_id),
9888 placement,
9889 record.formula_text,
9890 )
9891 }))?;
9892 prepared_by_sheet
9893 .entry(batch.sheet_name)
9894 .or_default()
9895 .extend(ingested.into_iter().map(|formula| {
9896 (
9897 formula.placement.coord.row() + 1,
9898 formula.placement.coord.col() + 1,
9899 formula.ast_id,
9900 formula.dep_plan,
9901 )
9902 }));
9903 }
9904 let admission_preflighted = self.graph_admission_enabled();
9905 if admission_preflighted {
9906 let mut preview = Vec::new();
9907 for (sheet_name, formulas) in &prepared_by_sheet {
9908 let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9909 ExcelError::new(ExcelErrorKind::Ref)
9910 .with_message(format!("unknown ingest sheet: {sheet_name}"))
9911 })?;
9912 preview.extend(
9913 formulas
9914 .iter()
9915 .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9916 );
9917 }
9918 let admission = self.graph.preview_formula_mutations(&preview)?;
9919 self.preflight_graph_admission(admission)?;
9920 }
9921
9922 let mut builder = self.begin_bulk_ingest();
9923 if admission_preflighted {
9924 builder.mark_admission_preflighted();
9925 }
9926 for (sheet_name, formulas) in prepared_by_sheet {
9927 if formulas.is_empty() {
9928 continue;
9929 }
9930 let sheet_id = builder.add_sheet(&sheet_name);
9931 builder.add_formula_plans(sheet_id, formulas);
9932 }
9933 let summary = builder.finish()?;
9934 report.graph_formula_cells_materialized = summary.formulas as u64;
9935 report.graph_vertices_created = summary.vertices as u64;
9936 report.graph_edges_created = summary.edges as u64;
9937 }
9938
9939 if publish_report {
9940 self.record_formula_ingest_report(report.clone());
9941 }
9942 Ok(report)
9943 }
9944
9945 fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9946 let mut unique = Vec::new();
9947 for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9948 let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9949 prior.sheet == diagnostic.sheet
9950 && prior.row == diagnostic.row
9951 && prior.col == diagnostic.col
9952 && prior.formula == diagnostic.formula
9953 && prior.policy == diagnostic.policy
9954 });
9955 if !duplicate {
9956 unique.push(diagnostic);
9957 }
9958 }
9959 self.formula_parse_diagnostics.extend(unique);
9960 }
9961
9962 pub fn handle_formula_parse_error(
9963 &mut self,
9964 sheet: &str,
9965 row: u32,
9966 col: u32,
9967 formula: &str,
9968 message: String,
9969 ) -> Result<Option<ASTNode>, ExcelError> {
9970 let policy = self.config.formula_parse_policy;
9971
9972 if policy == FormulaParsePolicy::Strict {
9973 let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9974 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9975 "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9976 )));
9977 }
9978
9979 self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9980 sheet: sheet.to_string(),
9981 row,
9982 col,
9983 formula: formula.to_string(),
9984 message: message.clone(),
9985 policy,
9986 });
9987
9988 match policy {
9989 FormulaParsePolicy::Strict => unreachable!(),
9990 FormulaParsePolicy::KeepCachedValue => Ok(None),
9991 FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9992 ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9993 None,
9994 ))),
9995 FormulaParsePolicy::CoerceToError => {
9996 let err = ExcelError::new(ExcelErrorKind::Error)
9997 .with_message(format!("Malformed formula: {message}"));
9998 Ok(Some(ASTNode::new(
9999 ASTNodeType::Literal(LiteralValue::Error(err)),
10000 None,
10001 )))
10002 }
10003 }
10004 }
10005
10006 #[cfg(test)]
10007 fn target_preparation_fault(
10008 &mut self,
10009 seam: crate::engine::target_preparation::TargetPreparationFault,
10010 ) -> Result<(), ExcelError> {
10011 if self.target_preparation_fault_for_test == Some(seam) {
10012 self.target_preparation_fault_for_test = None;
10013 Err(ExcelError::new(ExcelErrorKind::Value)
10014 .with_message(format!("injected target preparation fault: {seam:?}")))
10015 } else {
10016 Ok(())
10017 }
10018 }
10019
10020 fn preparation_stale(
10021 reason: formualizer_common::PreparationStaleReason,
10022 message: impl Into<String>,
10023 ) -> ExcelError {
10024 ExcelError::new(ExcelErrorKind::Value)
10025 .with_message(message)
10026 .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
10027 }
10028
10029 fn preparation_revision_stale_reason(
10030 assumptions: &crate::engine::PreparationRevision,
10031 current: &crate::engine::PreparationRevision,
10032 planning_requests: &BTreeSet<(String, String, usize)>,
10033 staged_leases_match: bool,
10034 ) -> Option<formualizer_common::PreparationStaleReason> {
10035 if assumptions.graph != current.graph {
10036 Some(formualizer_common::PreparationStaleReason::Graph)
10037 } else if assumptions.authority != current.authority
10038 || assumptions.authority_indexes != current.authority_indexes
10039 || assumptions.authority_indexed_plane != current.authority_indexed_plane
10040 {
10041 Some(formualizer_common::PreparationStaleReason::Authority)
10042 } else if assumptions.staged != current.staged || !staged_leases_match {
10043 Some(formualizer_common::PreparationStaleReason::Staged)
10044 } else if assumptions.symbols != current.symbols {
10045 Some(formualizer_common::PreparationStaleReason::Symbols)
10046 } else if assumptions.provider != current.provider {
10047 Some(formualizer_common::PreparationStaleReason::Provider)
10048 } else if assumptions.semantic != current.semantic
10049 && crate::function_registry::semantic_changes_affect_requests_since(
10050 assumptions.semantic,
10051 planning_requests.iter().cloned(),
10052 )
10053 {
10054 Some(formualizer_common::PreparationStaleReason::Semantic)
10055 } else {
10056 None
10057 }
10058 }
10059
10060 fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
10061 let (authority, authority_indexes, authority_indexed_plane) =
10062 self.graph.authority_revisions();
10063 crate::engine::PreparationRevision {
10064 graph: self.graph.topology_revision(),
10065 authority,
10066 authority_indexes,
10067 authority_indexed_plane,
10068 staged: self.staged_formula_index.revision(),
10069 symbols: self.graph.symbol_revision(),
10070 semantic: crate::function_registry::semantic_epoch(),
10071 provider: self.resolver.planning_semantic_revision(),
10072 }
10073 }
10074
10075 fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
10076 let registry_guard = crate::function_registry::semantic_epoch_read_guard();
10077 let provider = self.resolver.planning_semantic_revision();
10078 let semantic = registry_guard.epoch();
10079 let (authority, authority_indexes, authority_indexed_plane) =
10080 self.graph.authority_revisions();
10081 let mut span_refs = self.graph.formula_authority().active_span_refs();
10082 span_refs.sort_unstable_by_key(|span_ref| {
10083 (span_ref.id.0, span_ref.generation, span_ref.version)
10084 });
10085 PlanningRevisionSnapshot {
10086 engine_topology_epoch: self.topology_epoch,
10087 graph_topology_revision: self.graph.topology_revision(),
10088 authority,
10089 authority_indexes,
10090 authority_indexed_plane,
10091 staged: self.staged_formula_index.revision(),
10092 symbols: self.graph.symbol_revision(),
10093 semantic,
10094 provider,
10095 formula_plane_mode: self.config.formula_plane_mode,
10096 deterministic_mode: self.config.deterministic_mode.clone(),
10097 budgets: self.evaluation_resource_budgets.clone(),
10098 span_refs,
10099 }
10100 }
10101
10102 fn recalc_plan_key(&self) -> RecalcPlanKey {
10103 RecalcPlanKey {
10104 engine_token: Arc::clone(&self.recalc_plan_token),
10105 revisions: self.planning_revision_snapshot(),
10106 }
10107 }
10108
10109 fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
10110 ExcelError::new(ExcelErrorKind::Value)
10111 .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
10112 .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
10113 }
10114
10115 fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
10116 use formualizer_common::PlanStaleReason;
10117
10118 if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
10119 return Err(Self::plan_stale(PlanStaleReason::Engine));
10120 }
10121
10122 let current = self.planning_revision_snapshot();
10123 let expected = &key.revisions;
10124 let stale = if expected.provider != current.provider {
10125 Some(PlanStaleReason::Provider)
10126 } else if expected.semantic != current.semantic {
10127 Some(PlanStaleReason::Semantic)
10128 } else if expected.budgets != current.budgets
10129 || expected.deterministic_mode != current.deterministic_mode
10130 {
10131 Some(PlanStaleReason::Budget)
10132 } else if expected.staged != current.staged {
10133 Some(PlanStaleReason::Staged)
10134 } else if expected.symbols != current.symbols {
10135 Some(PlanStaleReason::Symbols)
10136 } else if expected.formula_plane_mode != current.formula_plane_mode
10137 || expected.authority != current.authority
10138 || expected.authority_indexes != current.authority_indexes
10139 || expected.authority_indexed_plane != current.authority_indexed_plane
10140 {
10141 Some(PlanStaleReason::Authority)
10142 } else if expected.span_refs != current.span_refs {
10143 Some(PlanStaleReason::SpanGeneration)
10144 } else if expected.graph_topology_revision != current.graph_topology_revision
10145 || expected.engine_topology_epoch != current.engine_topology_epoch
10146 {
10147 Some(PlanStaleReason::Graph)
10148 } else {
10149 None
10150 };
10151 stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
10152 }
10153
10154 fn target_preparation_checkpoint(
10155 &mut self,
10156 deadline: Option<std::time::Instant>,
10157 work: u64,
10158 ) -> Result<(), ExcelError> {
10159 if self
10160 .active_cancel_flag
10161 .as_ref()
10162 .is_some_and(|cancel| cancel.is_cancelled())
10163 {
10164 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
10165 .with_message("target graph preparation cancelled"));
10166 }
10167 if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
10168 return Err(crate::engine::ResourceLedgerError::Exhausted(
10169 formualizer_common::ResourceExhaustionDetail {
10170 reason: formualizer_common::ResourceExhaustionReason::Deadline,
10171 limit: 0,
10172 observed: 1,
10173 request_id: self
10174 .active_evaluation_resource_request
10175 .as_ref()
10176 .map(|stats| stats.request_id),
10177 },
10178 )
10179 .into_excel_error());
10180 }
10181 self.charge_bounded_work(work)
10182 }
10183
10184 fn opaque_reason_in_ast(
10185 &self,
10186 ast: &ASTNode,
10187 provider: &dyn crate::traits::FunctionProvider,
10188 ) -> Option<crate::engine::OpaqueReason> {
10189 match &ast.node_type {
10190 ASTNodeType::Function { name, args } => {
10191 let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
10192 if canonical == "INDIRECT" {
10193 return Some(crate::engine::OpaqueReason::RuntimeTextReference);
10194 }
10195 let Some(function) = provider.get_function_for_planning("", name) else {
10196 return Some(crate::engine::OpaqueReason::UnknownFunction);
10197 };
10198 let caps = function.caps();
10199 if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
10200 || caps.contains(FnCaps::RETURNS_REFERENCE)
10201 {
10202 return Some(crate::engine::OpaqueReason::DynamicReference);
10203 }
10204 args.iter()
10205 .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
10206 }
10207 ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
10208 ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
10209 ASTNodeType::BinaryOp { left, right, .. } => self
10210 .opaque_reason_in_ast(left, provider)
10211 .or_else(|| self.opaque_reason_in_ast(right, provider)),
10212 ASTNodeType::Array(rows) => rows
10213 .iter()
10214 .flat_map(|row| row.iter())
10215 .find_map(|item| self.opaque_reason_in_ast(item, provider)),
10216 ASTNodeType::Reference {
10217 reference:
10218 ReferenceType::Cell {
10219 sheet: Some(sheet), ..
10220 }
10221 | ReferenceType::Range {
10222 sheet: Some(sheet), ..
10223 },
10224 ..
10225 } if self.graph.sheet_id(sheet).is_none() => {
10226 Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
10227 }
10228 ASTNodeType::Reference {
10229 reference:
10230 ReferenceType::External(_)
10231 | ReferenceType::Cell3D { .. }
10232 | ReferenceType::Range3D { .. },
10233 ..
10234 } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
10235 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
10236 }
10237 }
10238
10239 fn target_planning_snapshot(
10240 &mut self,
10241 ast: &ASTNode,
10242 planning_requests: &mut BTreeSet<(String, String, usize)>,
10243 ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
10244 #[cfg(test)]
10245 if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
10246 return Err(Self::preparation_stale(
10247 formualizer_common::PreparationStaleReason::Semantic,
10248 "injected target semantic preparation movement",
10249 ));
10250 }
10251 #[cfg(test)]
10252 if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
10253 hook();
10254 }
10255 let mut requests = Vec::new();
10256 Self::collect_planning_function_requests(ast, &mut requests);
10257 requests.sort();
10258 requests.dedup();
10259 planning_requests.extend(requests.iter().cloned());
10260 crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
10261 &self.resolver,
10262 requests,
10263 )
10264 .map_err(|error| {
10265 ExcelError::new(ExcelErrorKind::Value)
10266 .with_message(format!("target planning snapshot unavailable: {error:?}"))
10267 })
10268 }
10269
10270 fn target_planning_snapshot_stale_reason(
10271 snapshot: &crate::function_registry::RegistryPlanningSnapshot,
10272 assumptions: &crate::engine::PreparationRevision,
10273 ) -> Option<formualizer_common::PreparationStaleReason> {
10274 if snapshot
10275 .provider_revision()
10276 .is_some_and(|revision| Some(revision) != assumptions.provider)
10277 {
10278 Some(formualizer_common::PreparationStaleReason::Provider)
10279 } else if snapshot.epoch() != assumptions.semantic
10280 && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
10281 {
10282 Some(formualizer_common::PreparationStaleReason::Semantic)
10283 } else {
10284 None
10285 }
10286 }
10287
10288 fn widen_target_preparation(
10289 policy: crate::engine::OpaquePreparePolicy,
10290 scope: &mut crate::engine::PrepareScope,
10291 reasons: &mut Vec<crate::engine::OpaqueReason>,
10292 reason: crate::engine::OpaqueReason,
10293 ) -> Result<bool, ExcelError> {
10294 if policy == crate::engine::OpaquePreparePolicy::Error {
10295 return Err(ExcelError::new(ExcelErrorKind::NImpl)
10296 .with_message(format!("opaque target preparation semantics: {reason:?}")));
10297 }
10298 if !reasons.contains(&reason) {
10299 reasons.push(reason);
10300 }
10301 if !matches!(scope, crate::engine::PrepareScope::Workbook) {
10302 *scope = crate::engine::PrepareScope::Workbook;
10303 Ok(true)
10304 } else {
10305 Ok(false)
10306 }
10307 }
10308
10309 fn widen_target_preparation_to_sheet(
10310 policy: crate::engine::OpaquePreparePolicy,
10311 scope: &mut crate::engine::PrepareScope,
10312 reasons: &mut Vec<crate::engine::OpaqueReason>,
10313 reason: crate::engine::OpaqueReason,
10314 sheet: &str,
10315 ) -> Result<bool, ExcelError> {
10316 if policy == crate::engine::OpaquePreparePolicy::Error {
10317 return Err(ExcelError::new(ExcelErrorKind::NImpl)
10318 .with_message(format!("opaque target preparation semantics: {reason:?}")));
10319 }
10320 if !reasons.contains(&reason) {
10321 reasons.push(reason);
10322 }
10323 match scope {
10324 crate::engine::PrepareScope::Exact => {
10325 *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
10326 Ok(true)
10327 }
10328 crate::engine::PrepareScope::Sheets(sheets) => {
10329 if sheets.iter().any(|candidate| candidate == sheet) {
10330 Ok(false)
10331 } else {
10332 sheets.push(sheet.to_string());
10333 sheets.sort();
10334 Ok(true)
10335 }
10336 }
10337 crate::engine::PrepareScope::Workbook => Ok(false),
10338 }
10339 }
10340
10341 fn ast_has_proven_sheet_local_dynamic(
10342 ast: &ASTNode,
10343 provider: &dyn crate::traits::FunctionProvider,
10344 ) -> bool {
10345 fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
10346 match &ast.node_type {
10347 ASTNodeType::Function { name, args } => {
10348 let Some(function) = provider.get_function_for_planning("", name) else {
10349 return (false, false);
10350 };
10351 let caps = function.caps();
10352 let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
10353 || caps.contains(FnCaps::RETURNS_REFERENCE);
10354 let canonical = name.rsplit('.').next().unwrap_or(name);
10355 if dynamic
10356 && !canonical.eq_ignore_ascii_case("OFFSET")
10357 && !canonical.eq_ignore_ascii_case("INDEX")
10358 {
10359 return (false, true);
10360 }
10361 let mut has_dynamic = dynamic;
10362 for arg in args {
10363 let (safe, child_dynamic) = classify(arg, provider);
10364 if !safe {
10365 return (false, has_dynamic || child_dynamic);
10366 }
10367 has_dynamic |= child_dynamic;
10368 }
10369 (true, has_dynamic)
10370 }
10371 ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
10372 ASTNodeType::BinaryOp { left, right, .. } => {
10373 let (left_safe, left_dynamic) = classify(left, provider);
10374 let (right_safe, right_dynamic) = classify(right, provider);
10375 (left_safe && right_safe, left_dynamic || right_dynamic)
10376 }
10377 ASTNodeType::Array(rows) => {
10378 let mut has_dynamic = false;
10379 for item in rows.iter().flatten() {
10380 let (safe, child_dynamic) = classify(item, provider);
10381 if !safe {
10382 return (false, has_dynamic || child_dynamic);
10383 }
10384 has_dynamic |= child_dynamic;
10385 }
10386 (true, has_dynamic)
10387 }
10388 ASTNodeType::Reference {
10389 reference:
10390 ReferenceType::Cell { sheet: None, .. }
10391 | ReferenceType::Range { sheet: None, .. },
10392 ..
10393 }
10394 | ASTNodeType::Literal(_)
10395 | ASTNodeType::Omitted => (true, false),
10396 ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
10397 }
10398 }
10399
10400 let (safe, dynamic) = classify(ast, provider);
10401 safe && dynamic
10402 }
10403
10404 fn table_selection_region(
10405 &self,
10406 entry: &crate::engine::graph::TableEntry,
10407 selection: &crate::engine::TableSelection,
10408 ) -> Result<PreparationRegion, ExcelError> {
10409 let mut start_row = entry.range.start.coord.row() + 1;
10410 let mut end_row = entry.range.end.coord.row() + 1;
10411 let mut start_col = entry.range.start.coord.col() + 1;
10412 let mut end_col = entry.range.end.coord.col() + 1;
10413 match selection {
10414 crate::engine::TableSelection::Whole => {}
10415 crate::engine::TableSelection::Headers => {
10416 if !entry.header_row {
10417 return Err(ExcelError::new(ExcelErrorKind::Value)
10418 .with_message(format!("table {} has no header row", entry.name)));
10419 }
10420 end_row = start_row;
10421 }
10422 crate::engine::TableSelection::Data => {
10423 if entry.header_row {
10424 start_row = start_row.saturating_add(1);
10425 }
10426 if entry.totals_row {
10427 end_row = end_row.saturating_sub(1);
10428 }
10429 }
10430 crate::engine::TableSelection::Totals => {
10431 if !entry.totals_row {
10432 return Err(ExcelError::new(ExcelErrorKind::Value)
10433 .with_message(format!("table {} has no totals row", entry.name)));
10434 }
10435 start_row = end_row;
10436 }
10437 crate::engine::TableSelection::Column(column) => {
10438 let index = entry.col_index(column).ok_or_else(|| {
10439 ExcelError::new(ExcelErrorKind::Name)
10440 .with_message(format!("unknown table column: {column}"))
10441 })?;
10442 start_col = start_col.saturating_add(index as u32);
10443 end_col = start_col;
10444 }
10445 crate::engine::TableSelection::Columns { start, end } => {
10446 let first = entry.col_index(start).ok_or_else(|| {
10447 ExcelError::new(ExcelErrorKind::Name)
10448 .with_message(format!("unknown table column: {start}"))
10449 })?;
10450 let last = entry.col_index(end).ok_or_else(|| {
10451 ExcelError::new(ExcelErrorKind::Name)
10452 .with_message(format!("unknown table column: {end}"))
10453 })?;
10454 if first > last {
10455 return Err(ExcelError::new(ExcelErrorKind::Value)
10456 .with_message("table column selection is reversed"));
10457 }
10458 start_col = start_col.saturating_add(first as u32);
10459 end_col = entry
10460 .range
10461 .start
10462 .coord
10463 .col()
10464 .saturating_add(last as u32)
10465 .saturating_add(1);
10466 }
10467 }
10468 if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
10469 start_row = entry.range.start.coord.row() + 1;
10470 end_row = start_row;
10471 }
10472 if start_row > end_row || start_col > end_col {
10473 return Err(
10474 ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
10475 );
10476 }
10477 Ok(PreparationRegion {
10478 sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
10479 sheet_id: entry.sheet_id(),
10480 start_row,
10481 start_col,
10482 end_row,
10483 end_col,
10484 })
10485 }
10486
10487 #[cfg(test)]
10488 pub(crate) fn reset_target_root_dedup_probes_for_test() {
10489 TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
10490 }
10491
10492 #[cfg(test)]
10493 pub(crate) fn target_root_dedup_probes_for_test() -> usize {
10494 TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
10495 }
10496
10497 pub(crate) fn resolve_target_producers(
10498 &mut self,
10499 targets: &[crate::engine::EvaluationTarget],
10500 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
10501 use crate::engine::target_preparation::TargetProducer;
10502
10503 let request_id = self
10504 .active_evaluation_resource_request
10505 .as_ref()
10506 .map(|request| request.request_id);
10507 let mut roots = OrderedTargetProducers::with_capacity(targets.len())
10508 .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
10509 let active_span_refs = self
10510 .graph
10511 .formula_authority()
10512 .active_span_refs()
10513 .into_iter()
10514 .map(|span_ref| (span_ref.id, span_ref))
10515 .collect::<BTreeMap<_, _>>();
10516 let resolve_region = |engine: &mut Self,
10517 region: Region,
10518 value_only: Option<CellRef>,
10519 roots: &mut OrderedTargetProducers|
10520 -> Result<(), ExcelError> {
10521 let before = roots.len();
10522 let authority = engine.graph.formula_authority();
10523 for matched in authority.producer_results.query(region).matches {
10524 if let FormulaProducerId::Span(span_id) = matched.value.producer
10525 && let Some(span_ref) = active_span_refs.get(&span_id).copied()
10526 && let Some(demanded) = matched.value.result_region.intersection(region)
10527 {
10528 roots
10529 .push(TargetProducer::Span { span_ref, demanded })
10530 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10531 }
10532 }
10533 for anchor in engine.graph.spill_anchors_in_region(
10534 region.sheet_id(),
10535 region.axis_ranges().0.query_bounds().0,
10536 region.axis_ranges().1.query_bounds().0,
10537 region.axis_ranges().0.query_bounds().1,
10538 region.axis_ranges().1.query_bounds().1,
10539 ) {
10540 roots
10541 .push(TargetProducer::Legacy(anchor))
10542 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10543 }
10544 for vertex in engine.graph.vertices_in_region(
10545 region.sheet_id(),
10546 region.axis_ranges().0.query_bounds().0,
10547 region.axis_ranges().0.query_bounds().1,
10548 region.axis_ranges().1.query_bounds().0,
10549 region.axis_ranges().1.query_bounds().1,
10550 ) {
10551 let vertex = engine
10552 .graph
10553 .get_cell_ref(vertex)
10554 .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
10555 .unwrap_or(vertex);
10556 if matches!(
10561 engine.graph.get_vertex_kind(vertex),
10562 VertexKind::FormulaScalar | VertexKind::FormulaArray
10563 ) {
10564 roots
10565 .push(TargetProducer::Legacy(vertex))
10566 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10567 }
10568 }
10569 if roots.len() == before
10570 && let Some(cell) = value_only
10571 {
10572 roots
10573 .push(TargetProducer::ValueOnly(cell))
10574 .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10575 }
10576 Ok(())
10577 };
10578
10579 for target in targets {
10580 match target {
10581 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10582 if *row == 0 || *col == 0 {
10583 return Err(ExcelError::new(ExcelErrorKind::Ref)
10584 .with_message("target cell coordinates are one-based"));
10585 }
10586 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10587 ExcelError::new(ExcelErrorKind::Ref)
10588 .with_message(format!("target sheet not found: {sheet}"))
10589 })?;
10590 let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
10591 resolve_region(
10592 self,
10593 Region::point(sheet_id, *row - 1, *col - 1),
10594 Some(cell),
10595 &mut roots,
10596 )?;
10597 }
10598 crate::engine::EvaluationTarget::Range(range) => {
10599 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10600 ExcelError::new(ExcelErrorKind::Ref)
10601 .with_message(format!("target sheet not found: {}", range.sheet))
10602 })?;
10603 resolve_region(
10604 self,
10605 Region::rect(
10606 sheet_id,
10607 range.start_row - 1,
10608 range.end_row - 1,
10609 range.start_col - 1,
10610 range.end_col - 1,
10611 ),
10612 None,
10613 &mut roots,
10614 )?;
10615 }
10616 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10617 let scope = self.name_query_scope(scope_sheet.as_deref())?;
10618 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
10619 roots
10620 .push(TargetProducer::Symbol(entry.vertex))
10621 .map_err(|_| {
10622 target_root_allocation_error(roots.len() + 1, request_id)
10623 })?;
10624 }
10625 }
10626 crate::engine::EvaluationTarget::Table { name, .. } => {
10627 if let Some(entry) = self.graph.resolve_table_entry(name) {
10628 roots
10629 .push(TargetProducer::Symbol(entry.vertex))
10630 .map_err(|_| {
10631 target_root_allocation_error(roots.len() + 1, request_id)
10632 })?;
10633 }
10634 }
10635 }
10636 }
10637 Ok(roots.into_vec())
10638 }
10639
10640 fn resolve_target_producers_from_existing_roots(
10641 &mut self,
10642 roots: &[crate::engine::target_preparation::TargetProducer],
10643 ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
10644 use crate::engine::target_preparation::TargetProducer;
10645 let request_id = self
10646 .active_evaluation_resource_request
10647 .as_ref()
10648 .map(|request| request.request_id);
10649 let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
10650 .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
10651 for root in roots {
10652 match *root {
10653 TargetProducer::Span { demanded, .. } => {
10654 let (rows, cols) = demanded.axis_ranges();
10655 for vertex in self.graph.vertices_in_region(
10656 demanded.sheet_id(),
10657 rows.query_bounds().0,
10658 rows.query_bounds().1,
10659 cols.query_bounds().0,
10660 cols.query_bounds().1,
10661 ) {
10662 if matches!(
10663 self.graph.get_vertex_kind(vertex),
10664 VertexKind::FormulaScalar | VertexKind::FormulaArray
10665 ) {
10666 refreshed
10667 .push(TargetProducer::Legacy(vertex))
10668 .map_err(|_| {
10669 target_root_allocation_error(refreshed.len() + 1, request_id)
10670 })?;
10671 }
10672 }
10673 }
10674 other => {
10675 refreshed.push(other).map_err(|_| {
10676 target_root_allocation_error(refreshed.len() + 1, request_id)
10677 })?;
10678 }
10679 }
10680 }
10681 Ok(refreshed.into_vec())
10682 }
10683
10684 pub fn prepare_graph_for_targets(
10688 &mut self,
10689 targets: &[crate::engine::EvaluationTarget],
10690 options: crate::engine::TargetEvalOptions<'_>,
10691 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10692 let previous_budgets = self.evaluation_resource_budgets.clone();
10693 let diagnostics_len = self.formula_parse_diagnostics.len();
10694 let previous_report = self.last_formula_ingest_report.clone();
10695 if let Some(budgets) = options.budgets {
10696 self.evaluation_resource_budgets = budgets.clone();
10697 }
10698 let previous_graph_budget_override = self
10699 .graph
10700 .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
10701 let previous_cancel = self.active_cancel_flag.take();
10705 self.active_cancel_flag = options.cancel.clone();
10706 let result = self.observe_evaluation_resource_request(
10707 EvaluationRequestKind::TargetPreparation,
10708 |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
10709 );
10710 self.active_cancel_flag = previous_cancel;
10711 self.graph
10712 .set_admission_budget_override(previous_graph_budget_override);
10713 self.evaluation_resource_budgets = previous_budgets;
10714 if result.is_err() {
10715 self.formula_parse_diagnostics.truncate(diagnostics_len);
10716 self.last_formula_ingest_report = previous_report;
10717 }
10718 result
10719 }
10720
10721 fn prepare_graph_for_targets_unobserved(
10722 &mut self,
10723 targets: &[crate::engine::EvaluationTarget],
10724 options: &crate::engine::TargetEvalOptions<'_>,
10725 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10726 let scratch_checkpoint = self
10727 .active_resource_ledger
10728 .as_ref()
10729 .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
10730 let result = self.prepare_graph_for_targets_transaction(targets, options);
10731 let release = self
10732 .active_resource_ledger
10733 .as_mut()
10734 .map_or(Ok(()), |ledger| {
10735 ledger.release_scratch_to(scratch_checkpoint)
10736 });
10737 match (result, release) {
10738 (result, Ok(())) => result,
10739 (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
10740 }
10741 }
10742
10743 fn prepare_graph_for_targets_transaction(
10744 &mut self,
10745 targets: &[crate::engine::EvaluationTarget],
10746 options: &crate::engine::TargetEvalOptions<'_>,
10747 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10748 use crate::engine::{
10749 OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
10750 TableSelection,
10751 };
10752
10753 self.target_preparation_checkpoint(options.deadline, 0)?;
10754 self.observe_function_semantic_epoch()?;
10755 let assumptions = self.preparation_revisions();
10756 let ledger_at_start = self
10757 .active_resource_ledger
10758 .as_ref()
10759 .map(|ledger| ledger.snapshot());
10760 let diagnostics_len = self.formula_parse_diagnostics.len();
10761 let report_len = self.last_formula_ingest_report.clone();
10762 let request_id = options.request_id.or_else(|| {
10763 self.active_evaluation_resource_request
10764 .as_ref()
10765 .map(|stats| stats.request_id)
10766 });
10767
10768 let mut scope = PrepareScope::Exact;
10769 let mut reasons = Vec::new();
10770 let mut regions = VecDeque::new();
10771 let mut normalized = Vec::with_capacity(targets.len());
10772 let mut symbol_vertices = VecDeque::new();
10773 for target in targets {
10774 self.target_preparation_checkpoint(options.deadline, 1)?;
10775 match target {
10776 crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10777 if *row == 0 || *col == 0 {
10778 return Err(ExcelError::new(ExcelErrorKind::Ref)
10779 .with_message("target cell coordinates are one-based"));
10780 }
10781 let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10782 ExcelError::new(ExcelErrorKind::Ref)
10783 .with_message(format!("target sheet not found: {sheet}"))
10784 })?;
10785 regions.push_back(PreparationRegion {
10786 sheet: sheet.clone(),
10787 sheet_id,
10788 start_row: *row,
10789 start_col: *col,
10790 end_row: *row,
10791 end_col: *col,
10792 });
10793 normalized.push(target.clone());
10794 }
10795 crate::engine::EvaluationTarget::Range(range) => {
10796 if range.start_row == 0
10797 || range.start_col == 0
10798 || range.end_row < range.start_row
10799 || range.end_col < range.start_col
10800 {
10801 return Err(ExcelError::new(ExcelErrorKind::Ref)
10802 .with_message("invalid target range"));
10803 }
10804 let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10805 ExcelError::new(ExcelErrorKind::Ref)
10806 .with_message(format!("target sheet not found: {}", range.sheet))
10807 })?;
10808 regions.push_back(PreparationRegion {
10809 sheet: range.sheet.clone(),
10810 sheet_id,
10811 start_row: range.start_row,
10812 start_col: range.start_col,
10813 end_row: range.end_row,
10814 end_col: range.end_col,
10815 });
10816 normalized.push(target.clone());
10817 }
10818 crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10819 let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
10820 if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
10821 symbol_vertices.push_back(entry.vertex);
10822 } else {
10823 Self::widen_target_preparation(
10824 options.opaque_policy,
10825 &mut scope,
10826 &mut reasons,
10827 OpaqueReason::UnresolvedName,
10828 )?;
10829 }
10830 normalized.push(target.clone());
10831 }
10832 crate::engine::EvaluationTarget::Table { name, selection } => {
10833 if let Some(entry) = self.graph.resolve_table_entry(name) {
10834 let region = self.table_selection_region(entry, selection)?;
10835 symbol_vertices.push_back(entry.vertex);
10836 regions.push_back(region);
10837 } else {
10838 Self::widen_target_preparation(
10839 options.opaque_policy,
10840 &mut scope,
10841 &mut reasons,
10842 OpaqueReason::UnresolvedTable,
10843 )?;
10844 }
10845 normalized.push(target.clone());
10846 }
10847 }
10848 }
10849
10850 let mut visited_regions = FxHashSet::default();
10851 let mut visited_vertices = FxHashSet::default();
10852 let mut selected = FxHashSet::default();
10853 let mut prepared = Vec::new();
10854 let mut pending_diagnostics = Vec::new();
10855 let mut planning_requests = BTreeSet::new();
10856 let mut selected_cells = Vec::new();
10857 let mut workbook_seeded = false;
10858 let mut sheet_scope_seeded = BTreeSet::new();
10859 let mut indexed_query_sheets = FxHashSet::default();
10860 let mut discovery_scratch_reserved = 0u64;
10861 let mut package_encountered = false;
10862 let mut selected_package_sheets = FxHashSet::default();
10863 let mut prepared_packages = Vec::new();
10864 let authoritative_with_ordinary = self.config.formula_plane_mode
10865 == FormulaPlaneMode::AuthoritativeExperimental
10866 && self.staged_formula_index.ordinary_count() != 0;
10867 let has_unknown_package_sheet = self
10868 .staged_formula_index
10869 .package_sheets()
10870 .any(|sheet| self.graph.sheet_id(sheet).is_none());
10871
10872 loop {
10873 if let PrepareScope::Sheets(sheets) = &scope {
10874 for sheet in sheets.clone() {
10875 if !sheet_scope_seeded.insert(sheet.clone()) {
10876 continue;
10877 }
10878 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10879 package_encountered = true;
10880 Self::widen_target_preparation(
10881 options.opaque_policy,
10882 &mut scope,
10883 &mut reasons,
10884 OpaqueReason::UnsupportedSourceSemantics,
10885 )?;
10886 continue;
10887 };
10888 for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10889 self.target_preparation_checkpoint(options.deadline, 1)?;
10890 regions.push_back(PreparationRegion {
10891 sheet: sheet.clone(),
10892 sheet_id,
10893 start_row: lease.row,
10894 start_col: lease.col,
10895 end_row: lease.row,
10896 end_col: lease.col,
10897 });
10898 }
10899 if self
10900 .staged_formula_index
10901 .package_lease_for_sheet(&sheet)
10902 .is_some()
10903 {
10904 regions.push_back(PreparationRegion {
10905 sheet: sheet.clone(),
10906 sheet_id,
10907 start_row: 1,
10908 start_col: 1,
10909 end_row: self.workbook_load_limits.max_sheet_rows,
10910 end_col: self.workbook_load_limits.max_sheet_cols,
10911 });
10912 }
10913 }
10914 }
10915 if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10916 workbook_seeded = true;
10917 for (sheet, lease) in self.staged_formula_index.all_leases() {
10918 self.target_preparation_checkpoint(options.deadline, 1)?;
10919 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10920 package_encountered = true;
10921 Self::widen_target_preparation(
10922 options.opaque_policy,
10923 &mut scope,
10924 &mut reasons,
10925 OpaqueReason::UnsupportedSourceSemantics,
10926 )?;
10927 continue;
10928 };
10929 regions.push_back(PreparationRegion {
10930 sheet,
10931 sheet_id,
10932 start_row: lease.row,
10933 start_col: lease.col,
10934 end_row: lease.row,
10935 end_col: lease.col,
10936 });
10937 }
10938 let package_sheets = self
10939 .staged_formula_index
10940 .package_sheets()
10941 .map(str::to_string)
10942 .collect::<Vec<_>>();
10943 for sheet in package_sheets {
10944 let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10945 package_encountered = true;
10946 Self::widen_target_preparation(
10947 options.opaque_policy,
10948 &mut scope,
10949 &mut reasons,
10950 OpaqueReason::UnsupportedSourceSemantics,
10951 )?;
10952 continue;
10953 };
10954 regions.push_back(PreparationRegion {
10955 sheet,
10956 sheet_id,
10957 start_row: 1,
10958 start_col: 1,
10959 end_row: self.workbook_load_limits.max_sheet_rows,
10960 end_col: self.workbook_load_limits.max_sheet_cols,
10961 });
10962 }
10963 }
10964
10965 let Some(region) = regions.pop_front() else {
10966 if let Some(vertex) = symbol_vertices.pop_front() {
10967 self.target_preparation_checkpoint(options.deadline, 1)?;
10968 if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10969 continue;
10970 }
10971 let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10972 if let Some(ast) = self.graph.get_formula(vertex) {
10973 let snapshot =
10974 self.target_planning_snapshot(&ast, &mut planning_requests)?;
10975 if let Some(reason) =
10976 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10977 {
10978 return Err(Self::preparation_stale(
10979 reason,
10980 "target planning snapshot became stale during discovery",
10981 ));
10982 }
10983 let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10984 if let Some(reason) =
10985 opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10986 {
10987 if reason == OpaqueReason::DynamicReference
10988 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10989 {
10990 let sheet = self.graph.get_vertex_sheet_id(vertex);
10991 let sheet = self.graph.sheet_name(sheet).to_string();
10992 Self::widen_target_preparation_to_sheet(
10993 options.opaque_policy,
10994 &mut scope,
10995 &mut reasons,
10996 reason,
10997 &sheet,
10998 )?;
10999 } else {
11000 Self::widen_target_preparation(
11001 options.opaque_policy,
11002 &mut scope,
11003 &mut reasons,
11004 reason,
11005 )?;
11006 }
11007 }
11008 } else if vertex_is_dynamic {
11009 Self::widen_target_preparation(
11010 options.opaque_policy,
11011 &mut scope,
11012 &mut reasons,
11013 OpaqueReason::DynamicReference,
11014 )?;
11015 }
11016 if let Some(anchor) = self
11017 .graph
11018 .get_cell_ref(vertex)
11019 .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
11020 {
11021 symbol_vertices.push_back(anchor);
11022 }
11023 if let Some(cell) = self.graph.get_cell_ref(vertex) {
11024 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
11025 regions.push_back(PreparationRegion {
11026 sheet,
11027 sheet_id: cell.sheet_id,
11028 start_row: cell.coord.row() + 1,
11029 start_col: cell.coord.col() + 1,
11030 end_row: cell.coord.row() + 1,
11031 end_col: cell.coord.col() + 1,
11032 });
11033 }
11034 for dependency in self.graph.get_dependencies(vertex) {
11035 self.target_preparation_checkpoint(options.deadline, 1)?;
11036 symbol_vertices.push_back(dependency);
11037 }
11038 if let Some(range_dependencies) = self
11039 .graph
11040 .formula_range_dependencies(vertex)
11041 .map(<[_]>::to_vec)
11042 {
11043 for range in range_dependencies {
11044 self.target_preparation_checkpoint(options.deadline, 1)?;
11045 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
11047 let Ok(sheet_id) =
11048 self.resolve_sheet_locator(&range.sheet, context_sheet)
11049 else {
11050 if Self::widen_target_preparation(
11051 options.opaque_policy,
11052 &mut scope,
11053 &mut reasons,
11054 OpaqueReason::UnresolvedCrossSheetBinding,
11055 )? {
11056 break;
11057 }
11058 continue;
11059 };
11060 let sheet = self.graph.sheet_name(sheet_id).to_string();
11061 regions.push_back(PreparationRegion {
11062 sheet,
11063 sheet_id,
11064 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11065 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11066 end_row: range
11067 .end_row
11068 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11069 bound.index + 1
11070 }),
11071 end_col: range
11072 .end_col
11073 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11074 bound.index + 1
11075 }),
11076 });
11077 }
11078 }
11079 if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
11080 match &name.definition {
11081 NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
11082 sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
11083 sheet_id: cell.sheet_id,
11084 start_row: cell.coord.row() + 1,
11085 start_col: cell.coord.col() + 1,
11086 end_row: cell.coord.row() + 1,
11087 end_col: cell.coord.col() + 1,
11088 }),
11089 NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
11090 sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
11091 sheet_id: range.start.sheet_id,
11092 start_row: range.start.coord.row() + 1,
11093 start_col: range.start.coord.col() + 1,
11094 end_row: range.end.coord.row() + 1,
11095 end_col: range.end.coord.col() + 1,
11096 }),
11097 NamedDefinition::Formula {
11098 ast,
11099 dependencies,
11100 range_deps,
11101 } => {
11102 let snapshot =
11103 self.target_planning_snapshot(ast, &mut planning_requests)?;
11104 if let Some(reason) = Self::target_planning_snapshot_stale_reason(
11105 &snapshot,
11106 &assumptions,
11107 ) {
11108 return Err(Self::preparation_stale(
11109 reason,
11110 "target planning snapshot became stale during discovery",
11111 ));
11112 }
11113 if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
11114 Self::widen_target_preparation(
11115 options.opaque_policy,
11116 &mut scope,
11117 &mut reasons,
11118 reason,
11119 )?;
11120 }
11121 for dependency in dependencies {
11122 self.target_preparation_checkpoint(options.deadline, 1)?;
11123 symbol_vertices.push_back(*dependency);
11124 }
11125 for range in range_deps {
11126 self.target_preparation_checkpoint(options.deadline, 1)?;
11127 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
11135 let Ok(sheet_id) =
11136 self.resolve_sheet_locator(&range.sheet, context_sheet)
11137 else {
11138 if Self::widen_target_preparation(
11139 options.opaque_policy,
11140 &mut scope,
11141 &mut reasons,
11142 OpaqueReason::UnresolvedCrossSheetBinding,
11143 )? {
11144 break;
11145 }
11146 continue;
11147 };
11148 regions.push_back(PreparationRegion {
11149 sheet: self.graph.sheet_name(sheet_id).to_string(),
11150 sheet_id,
11151 start_row: range
11152 .start_row
11153 .map_or(1, |bound| bound.index + 1),
11154 start_col: range
11155 .start_col
11156 .map_or(1, |bound| bound.index + 1),
11157 end_row: range.end_row.map_or(
11158 self.workbook_load_limits.max_sheet_rows,
11159 |bound| bound.index + 1,
11160 ),
11161 end_col: range.end_col.map_or(
11162 self.workbook_load_limits.max_sheet_cols,
11163 |bound| bound.index + 1,
11164 ),
11165 });
11166 }
11167 }
11168 NamedDefinition::Literal(_) => {}
11169 }
11170 }
11171 if let Some(table) = self.graph.table_by_vertex(vertex) {
11172 regions
11173 .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
11174 }
11175 continue;
11176 }
11177 break;
11178 };
11179
11180 self.target_preparation_checkpoint(options.deadline, 1)?;
11181 if !visited_regions.insert(region.clone()) {
11182 continue;
11183 }
11184 if let PrepareScope::Sheets(sheets) = &mut scope
11185 && !sheets.iter().any(|sheet| sheet == ®ion.sheet)
11186 {
11187 sheets.push(region.sheet.clone());
11188 sheets.sort();
11189 }
11190 let package_match = self.staged_formula_index.package_for_region(
11191 ®ion.sheet,
11192 region.start_row,
11193 region.start_col,
11194 region.end_row,
11195 region.end_col,
11196 );
11197 let package_lease = match package_match {
11198 Some(Ok(lease)) => Some(lease),
11199 Some(Err(()))
11200 if region.start_row == 1
11201 && region.start_col == 1
11202 && region.end_row == self.workbook_load_limits.max_sheet_rows
11203 && region.end_col == self.workbook_load_limits.max_sheet_cols =>
11204 {
11205 self.staged_formula_index
11206 .package_lease_for_sheet(®ion.sheet)
11207 }
11208 Some(Err(())) => {
11209 Self::widen_target_preparation(
11210 options.opaque_policy,
11211 &mut scope,
11212 &mut reasons,
11213 OpaqueReason::DeferredSourcePackage,
11214 )?;
11215 None
11216 }
11217 None => None,
11218 };
11219 let compatibility_before_package_replay = package_lease.is_some()
11220 && (authoritative_with_ordinary || has_unknown_package_sheet);
11221 let package_lease = if compatibility_before_package_replay {
11222 package_encountered = true;
11223 Self::widen_target_preparation(
11224 options.opaque_policy,
11225 &mut scope,
11226 &mut reasons,
11227 OpaqueReason::UnsupportedSourceSemantics,
11228 )?;
11229 None
11230 } else {
11231 package_lease
11232 };
11233 if let Some(package_lease) = package_lease
11234 && selected_package_sheets.insert(region.sheet.clone())
11235 {
11236 self.target_preparation_checkpoint(options.deadline, 1)?;
11237 let mut package = self.prepare_target_source_package(
11238 ®ion.sheet,
11239 package_lease,
11240 options.deadline,
11241 )?;
11242 for placement in &package.placements {
11243 self.target_preparation_checkpoint(options.deadline, 1)?;
11244 for dependency in &placement.fragment_dependency_proof().1.dependencies {
11245 self.target_preparation_checkpoint(options.deadline, 1)?;
11246 let (rows, cols) = dependency.read_region.axis_ranges();
11247 let (start_row, end_row) = rows.query_bounds();
11248 let (start_col, end_col) = cols.query_bounds();
11249 let dependency_sheet = dependency.read_region.sheet_id();
11250 regions.push_back(PreparationRegion {
11251 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11252 sheet_id: dependency_sheet,
11253 start_row: start_row.saturating_add(1),
11254 start_col: start_col.saturating_add(1),
11255 end_row: end_row
11256 .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
11257 .saturating_add(1),
11258 end_col: end_col
11259 .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
11260 .saturating_add(1),
11261 });
11262 }
11263 }
11264
11265 let mut final_fallback = BTreeMap::new();
11266 for record in package.fallback_records() {
11267 final_fallback.insert((record.row, record.col), record.clone());
11268 }
11269 let batch = self
11270 .formula_batch_from_exact_replay(®ion.sheet, final_fallback.into_values())?;
11271 for record in batch.formulas {
11272 self.target_preparation_checkpoint(options.deadline, 1)?;
11273 let ast = self
11274 .graph
11275 .data_store()
11276 .retrieve_ast(record.ast_id, self.graph.sheet_reg())
11277 .ok_or_else(|| {
11278 ExcelError::new(ExcelErrorKind::Value)
11279 .with_message("target fallback AST is unavailable")
11280 })?;
11281 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
11282 if let Some(reason) =
11283 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
11284 {
11285 return Err(Self::preparation_stale(
11286 reason,
11287 "target fallback planning snapshot became stale during discovery",
11288 ));
11289 }
11290 let proven_sheet_local_dynamic =
11291 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
11292 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
11293 if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
11294 Self::widen_target_preparation_to_sheet(
11295 options.opaque_policy,
11296 &mut scope,
11297 &mut reasons,
11298 reason,
11299 ®ion.sheet,
11300 )?;
11301 } else {
11302 Self::widen_target_preparation(
11303 options.opaque_policy,
11304 &mut scope,
11305 &mut reasons,
11306 reason,
11307 )?;
11308 }
11309 }
11310 let placement = CellRef::new(
11311 package.sheet_id,
11312 Coord::from_excel(record.row, record.col, true, true),
11313 );
11314 let ingested = self
11315 .graph
11316 .ingest_pipeline(&snapshot)
11317 .enable_function_semantics()
11318 .ingest_formula(
11319 FormulaAstInput::RawArena(record.ast_id),
11320 placement,
11321 record.formula_text,
11322 )?;
11323 if ingested.dep_plan.dynamic {
11324 if proven_sheet_local_dynamic {
11325 Self::widen_target_preparation_to_sheet(
11326 options.opaque_policy,
11327 &mut scope,
11328 &mut reasons,
11329 OpaqueReason::DynamicReference,
11330 ®ion.sheet,
11331 )?;
11332 } else {
11333 Self::widen_target_preparation(
11334 options.opaque_policy,
11335 &mut scope,
11336 &mut reasons,
11337 OpaqueReason::DynamicReference,
11338 )?;
11339 }
11340 }
11341 for dep in &ingested.dep_plan.direct_cell_deps {
11342 self.target_preparation_checkpoint(options.deadline, 1)?;
11343 regions.push_back(PreparationRegion {
11344 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
11345 sheet_id: dep.sheet_id,
11346 start_row: dep.coord.row().saturating_add(1),
11347 start_col: dep.coord.col().saturating_add(1),
11348 end_row: dep.coord.row().saturating_add(1),
11349 end_col: dep.coord.col().saturating_add(1),
11350 });
11351 }
11352 for range in &ingested.dep_plan.range_deps {
11353 self.target_preparation_checkpoint(options.deadline, 1)?;
11354 let Ok(dependency_sheet) =
11357 self.resolve_sheet_locator(&range.sheet, package.sheet_id)
11358 else {
11359 Self::widen_target_preparation(
11360 options.opaque_policy,
11361 &mut scope,
11362 &mut reasons,
11363 OpaqueReason::UnresolvedCrossSheetBinding,
11364 )?;
11365 continue;
11366 };
11367 regions.push_back(PreparationRegion {
11368 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11369 sheet_id: dependency_sheet,
11370 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11371 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11372 end_row: range
11373 .end_row
11374 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11375 bound.index + 1
11376 }),
11377 end_col: range
11378 .end_col
11379 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11380 bound.index + 1
11381 }),
11382 });
11383 }
11384 for name in ingested
11385 .dep_plan
11386 .resolved_named_refs
11387 .iter()
11388 .chain(&ingested.dep_plan.named_refs)
11389 {
11390 self.target_preparation_checkpoint(options.deadline, 1)?;
11391 if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
11392 symbol_vertices.push_back(entry.vertex);
11393 } else if self.graph.resolve_source_scalar_entry(name).is_none()
11394 && self.graph.resolve_source_table_entry(name).is_none()
11395 {
11396 Self::widen_target_preparation(
11397 options.opaque_policy,
11398 &mut scope,
11399 &mut reasons,
11400 OpaqueReason::UnresolvedName,
11401 )?;
11402 }
11403 }
11404 for table in &ingested.dep_plan.table_refs {
11405 self.target_preparation_checkpoint(options.deadline, 1)?;
11406 if let Some(entry) = self.graph.resolve_table_entry(table) {
11407 symbol_vertices.push_back(entry.vertex);
11408 } else if self.graph.resolve_source_table_entry(table).is_none() {
11409 Self::widen_target_preparation(
11410 options.opaque_policy,
11411 &mut scope,
11412 &mut reasons,
11413 OpaqueReason::UnresolvedTable,
11414 )?;
11415 }
11416 }
11417 package.legacy.push((
11418 record.row,
11419 record.col,
11420 ingested.ast_id,
11421 ingested.dep_plan,
11422 ));
11423 }
11424 prepared_packages.push(package);
11425 }
11426 let leases = self.staged_formula_index.leases_in_region(
11427 ®ion.sheet,
11428 region.start_row,
11429 region.start_col,
11430 region.end_row,
11431 region.end_col,
11432 );
11433 for lease in leases {
11434 self.target_preparation_checkpoint(options.deadline, 1)?;
11435 let sheet_id = self.graph.sheet_id(®ion.sheet).ok_or_else(|| {
11436 ExcelError::new(ExcelErrorKind::Ref)
11437 .with_message(format!("staged formula sheet not found: {}", region.sheet))
11438 })?;
11439 let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
11440 if !selected.insert(key) {
11441 continue;
11442 }
11443 let text = self
11444 .staged_formulas
11445 .get(®ion.sheet)
11446 .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
11447 .ok_or_else(|| {
11448 ExcelError::new(ExcelErrorKind::Value)
11449 .with_message("staged formula index is stale")
11450 })?
11451 .to_string();
11452 let formula = if text.starts_with('=') {
11453 text.clone()
11454 } else {
11455 format!("={text}")
11456 };
11457 self.target_preparation_checkpoint(options.deadline, 1)?;
11458 let ast = match formualizer_parse::parser::parse(&formula) {
11459 Ok(ast) => ast,
11460 Err(error) => {
11461 if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
11462 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
11463 format!(
11464 "Formula parse error at {}!{}{}: {error}",
11465 region.sheet,
11466 col_letters_from_1based(lease.col)
11467 .unwrap_or_else(|_| "?".to_string()),
11468 lease.row
11469 ),
11470 ));
11471 }
11472 pending_diagnostics.push(FormulaParseDiagnostic {
11473 sheet: region.sheet.clone(),
11474 row: lease.row,
11475 col: lease.col,
11476 formula: formula.clone(),
11477 message: error.to_string(),
11478 policy: self.config.formula_parse_policy,
11479 });
11480 match self.config.formula_parse_policy {
11481 FormulaParsePolicy::KeepCachedValue => {
11482 selected_cells.push(
11483 formualizer_common::RangeAddress::new(
11484 region.sheet.clone(),
11485 lease.row,
11486 lease.col,
11487 lease.row,
11488 lease.col,
11489 )
11490 .expect("selected staged coordinates are valid"),
11491 );
11492 prepared.push(PreparedOrdinaryStagedFormula {
11493 sheet: region.sheet.clone(),
11494 sheet_id,
11495 lease,
11496 ast_id: None,
11497 plan: None,
11498 });
11499 continue;
11500 }
11501 FormulaParsePolicy::AsText => ASTNode::new(
11502 ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
11503 None,
11504 ),
11505 FormulaParsePolicy::CoerceToError => ASTNode::new(
11506 ASTNodeType::Literal(LiteralValue::Error(
11507 ExcelError::new(ExcelErrorKind::Error)
11508 .with_message(format!("Malformed formula: {error}")),
11509 )),
11510 None,
11511 ),
11512 FormulaParsePolicy::Strict => unreachable!(),
11513 }
11514 }
11515 };
11516 self.target_preparation_checkpoint(options.deadline, 1)?;
11517 let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
11518 self.target_preparation_checkpoint(options.deadline, 1)?;
11519 if let Some(reason) =
11520 Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
11521 {
11522 return Err(Self::preparation_stale(
11523 reason,
11524 "target planning snapshot became stale during discovery",
11525 ));
11526 }
11527 if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
11528 if reason == OpaqueReason::DynamicReference
11529 && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
11530 {
11531 Self::widen_target_preparation_to_sheet(
11532 options.opaque_policy,
11533 &mut scope,
11534 &mut reasons,
11535 reason,
11536 ®ion.sheet,
11537 )?;
11538 } else {
11539 Self::widen_target_preparation(
11540 options.opaque_policy,
11541 &mut scope,
11542 &mut reasons,
11543 reason,
11544 )?;
11545 }
11546 }
11547 let proven_sheet_local_dynamic =
11548 Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
11549 let placement = CellRef::new(
11550 sheet_id,
11551 Coord::from_excel(lease.row, lease.col, true, true),
11552 );
11553 let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
11554 FormulaAstInput::Tree(ast),
11555 placement,
11556 Some(Arc::from(formula)),
11557 )?;
11558 self.target_preparation_checkpoint(options.deadline, 1)?;
11559 if ingested.dep_plan.dynamic {
11560 if proven_sheet_local_dynamic {
11561 Self::widen_target_preparation_to_sheet(
11562 options.opaque_policy,
11563 &mut scope,
11564 &mut reasons,
11565 OpaqueReason::DynamicReference,
11566 ®ion.sheet,
11567 )?;
11568 } else {
11569 Self::widen_target_preparation(
11570 options.opaque_policy,
11571 &mut scope,
11572 &mut reasons,
11573 OpaqueReason::DynamicReference,
11574 )?;
11575 }
11576 }
11577 for dep in &ingested.dep_plan.direct_cell_deps {
11578 self.target_preparation_checkpoint(options.deadline, 1)?;
11579 regions.push_back(PreparationRegion {
11580 sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
11581 sheet_id: dep.sheet_id,
11582 start_row: dep.coord.row() + 1,
11583 start_col: dep.coord.col() + 1,
11584 end_row: dep.coord.row() + 1,
11585 end_col: dep.coord.col() + 1,
11586 });
11587 }
11588 for range in &ingested.dep_plan.range_deps {
11589 self.target_preparation_checkpoint(options.deadline, 1)?;
11590 let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
11592 else {
11593 Self::widen_target_preparation(
11594 options.opaque_policy,
11595 &mut scope,
11596 &mut reasons,
11597 OpaqueReason::UnresolvedCrossSheetBinding,
11598 )?;
11599 continue;
11600 };
11601 regions.push_back(PreparationRegion {
11602 sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11603 sheet_id: dependency_sheet,
11604 start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11605 start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11606 end_row: range
11607 .end_row
11608 .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11609 bound.index + 1
11610 }),
11611 end_col: range
11612 .end_col
11613 .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11614 bound.index + 1
11615 }),
11616 });
11617 }
11618 for name in ingested
11619 .dep_plan
11620 .resolved_named_refs
11621 .iter()
11622 .chain(&ingested.dep_plan.named_refs)
11623 {
11624 self.target_preparation_checkpoint(options.deadline, 1)?;
11625 if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
11626 symbol_vertices.push_back(entry.vertex);
11627 } else if self.graph.resolve_source_scalar_entry(name).is_none()
11628 && self.graph.resolve_source_table_entry(name).is_none()
11629 {
11630 Self::widen_target_preparation(
11631 options.opaque_policy,
11632 &mut scope,
11633 &mut reasons,
11634 OpaqueReason::UnresolvedName,
11635 )?;
11636 }
11637 }
11638 for table in &ingested.dep_plan.table_refs {
11639 self.target_preparation_checkpoint(options.deadline, 1)?;
11640 if let Some(entry) = self.graph.resolve_table_entry(table) {
11641 symbol_vertices.push_back(entry.vertex);
11642 } else if self.graph.resolve_source_table_entry(table).is_none() {
11643 Self::widen_target_preparation(
11644 options.opaque_policy,
11645 &mut scope,
11646 &mut reasons,
11647 OpaqueReason::UnresolvedTable,
11648 )?;
11649 }
11650 }
11651 selected_cells.push(
11652 formualizer_common::RangeAddress::new(
11653 region.sheet.clone(),
11654 lease.row,
11655 lease.col,
11656 lease.row,
11657 lease.col,
11658 )
11659 .expect("selected staged coordinates are valid"),
11660 );
11661 prepared.push(PreparedOrdinaryStagedFormula {
11662 sheet: region.sheet.clone(),
11663 sheet_id,
11664 lease,
11665 ast_id: Some(ingested.ast_id),
11666 plan: Some(ingested.dep_plan),
11667 });
11668 }
11669
11670 if indexed_query_sheets.insert(region.sheet_id) {
11671 self.graph.prepare_sheet_index_for_query(region.sheet_id);
11672 let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
11673 .saturating_mul(32);
11674 self.reserve_graph_source_scratch(bytes)?;
11675 discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
11676 }
11677 let spill_anchors = self.graph.spill_anchors_in_region(
11678 region.sheet_id,
11679 region.start_row - 1,
11680 region.start_col - 1,
11681 region.end_row - 1,
11682 region.end_col - 1,
11683 );
11684 for anchor in spill_anchors {
11685 self.target_preparation_checkpoint(options.deadline, 1)?;
11686 symbol_vertices.push_back(anchor);
11687 }
11688 let vertices = self.graph.vertices_in_region(
11689 region.sheet_id,
11690 region.start_row - 1,
11691 region.end_row - 1,
11692 region.start_col - 1,
11693 region.end_col - 1,
11694 );
11695 for vertex in vertices {
11696 self.target_preparation_checkpoint(options.deadline, 1)?;
11697 symbol_vertices.push_back(vertex);
11698 }
11699 }
11700
11701 #[cfg(test)]
11702 self.target_preparation_fault(
11703 crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
11704 )?;
11705
11706 if package_encountered
11707 || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11708 && !prepared.is_empty())
11709 {
11710 Self::widen_target_preparation(
11711 options.opaque_policy,
11712 &mut scope,
11713 &mut reasons,
11714 OpaqueReason::UnsupportedSourceSemantics,
11715 )?;
11716 self.target_preparation_checkpoint(options.deadline, 0)?;
11717 let selected_count = self.staged_formula_count();
11718 let selected_packages = self
11719 .staged_formulas
11720 .values()
11721 .filter_map(|staged| staged.deferred_package.as_ref())
11722 .map(|package| package.families.len() + package.partitioned_families.len())
11723 .sum();
11724 self.formula_parse_diagnostics.truncate(diagnostics_len);
11725 self.last_formula_ingest_report = report_len;
11726 #[cfg(test)]
11727 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11728 hook();
11729 }
11730 self.target_preparation_checkpoint(options.deadline, 0)?;
11731 #[cfg(test)]
11732 self.target_preparation_fault(
11733 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11734 )?;
11735 let current_revisions = self.preparation_revisions();
11736 if let Some(reason) = Self::preparation_revision_stale_reason(
11737 &assumptions,
11738 ¤t_revisions,
11739 &planning_requests,
11740 true,
11741 ) {
11742 return Err(Self::preparation_stale(
11743 reason,
11744 "target compatibility preparation plan is stale",
11745 ));
11746 }
11747 #[cfg(test)]
11748 self.target_preparation_fault(
11749 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11750 )?;
11751 let commit_work_before = self
11752 .active_resource_ledger
11753 .as_ref()
11754 .map_or(0, |ledger| ledger.snapshot().work_charged);
11755 let commit_started = crate::instant::FzInstant::now();
11756 self.build_graph_all_unobserved()?;
11757 let commit_window = commit_started.elapsed();
11758 let ledger_after = self
11759 .active_resource_ledger
11760 .as_ref()
11761 .map(|ledger| ledger.snapshot());
11762 let actual_commit_work = ledger_after
11763 .map_or(0, |snapshot| snapshot.work_charged)
11764 .saturating_sub(commit_work_before);
11765 let observed_scratch_bytes = ledger_after
11766 .map_or(0, |snapshot| snapshot.scratch_peak)
11767 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11768 let revisions = assumptions.clone();
11769 let report = PreparedTargetGraphReport {
11770 request_id: request_id.unwrap_or_default(),
11771 requested_targets: targets.len(),
11772 normalized_regions: visited_regions.len(),
11773 normalized_target_list: normalized,
11774 selected_staged_cells: selected_count,
11775 selected_source_families: selected_packages,
11776 retained_staged_cells: self.staged_formula_count(),
11777 selected_cells,
11778 retained_cells: Vec::new(),
11779 widened_scope: PrepareScope::Workbook,
11780 widening_reasons: reasons,
11781 revisions,
11782 commit_window,
11783 estimated_scratch_bytes: discovery_scratch_reserved
11784 .saturating_add((selected_count as u64).saturating_mul(256)),
11785 observed_scratch_bytes,
11786 estimated_commit_work: selected_count as u64,
11787 actual_commit_work,
11788 outcome: PreparationOutcome::CompatibilityPrepared,
11789 };
11790 self.observe_target_preparation_report(&report);
11791 return Ok(report);
11792 }
11793
11794 prepared.sort_by_key(|formula| formula.lease.insertion_order);
11795 for package in &prepared_packages {
11796 selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11797 formualizer_common::RangeAddress::new(
11798 package.sheet.clone(),
11799 record.row,
11800 record.col,
11801 record.row,
11802 record.col,
11803 )
11804 .ok()
11805 }));
11806 }
11807 let (mut legacy_graph, mut planned_formula_count) =
11808 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11809 let placements = prepared_packages
11810 .iter_mut()
11811 .flat_map(|package| std::mem::take(&mut package.placements))
11812 .collect::<Vec<_>>();
11813 let formula_plane = if placements.is_empty() {
11814 None
11815 } else {
11816 match self.graph.formula_authority().prepare_formula_plane_append(
11817 placements,
11818 self.graph.data_store(),
11819 self.graph.sheet_reg(),
11820 ) {
11821 Ok(append) => Some(append),
11822 Err(error) => {
11823 let reason = format!("TargetFormulaPlaneAppend:{error}");
11824 for package in &mut prepared_packages {
11825 *package
11826 .source_report
11827 .fallback_reasons
11828 .entry(reason.clone())
11829 .or_default() += package.direct_families as u64;
11830 if self.config.formula_plane_mode
11831 == FormulaPlaneMode::AuthoritativeExperimental
11832 {
11833 self.materialize_target_package_direct_records(
11834 package,
11835 &assumptions,
11836 &mut planning_requests,
11837 options.deadline,
11838 )?;
11839 } else {
11840 package.direct_families = 0;
11841 package.direct_cells = 0;
11842 package.direct_fragments = 0;
11843 package.direct_complete_families = 0;
11844 package.direct_complete_cells = 0;
11845 package.direct_partition_families = 0;
11846 package.direct_partition_cells = 0;
11847 }
11848 }
11849 (legacy_graph, planned_formula_count) =
11850 self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11851 None
11852 }
11853 }
11854 };
11855 let new_vertices = legacy_graph.new_vertex_count();
11856 let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11857 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11858 })?;
11859 let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11860 ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11861 })?;
11862 let current = self.graph.baseline_stats();
11863 let final_vertices = current
11864 .graph_vertex_count
11865 .checked_add(new_vertices)
11866 .ok_or_else(|| {
11867 crate::engine::ResourceLedgerError::Exhausted(
11868 formualizer_common::ResourceExhaustionDetail {
11869 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11870 limit: u64::MAX,
11871 observed: u64::MAX,
11872 request_id,
11873 },
11874 )
11875 .into_excel_error()
11876 })?;
11877 let final_edges = current
11878 .graph_edge_count
11879 .checked_sub(removed_edges)
11880 .and_then(|count| count.checked_add(new_edges))
11881 .ok_or_else(|| {
11882 crate::engine::ResourceLedgerError::Exhausted(
11883 formualizer_common::ResourceExhaustionDetail {
11884 reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11885 limit: u64::MAX,
11886 observed: u64::MAX,
11887 request_id,
11888 },
11889 )
11890 .into_excel_error()
11891 })?;
11892 #[cfg(test)]
11893 self.target_preparation_fault(
11894 crate::engine::target_preparation::TargetPreparationFault::Admission,
11895 )?;
11896 let resource = |reason, limit: u64, observed: u64| {
11897 crate::engine::ResourceLedgerError::Exhausted(
11898 formualizer_common::ResourceExhaustionDetail {
11899 reason,
11900 limit,
11901 observed,
11902 request_id,
11903 },
11904 )
11905 .into_excel_error()
11906 };
11907 let admission = crate::engine::resource_ledger::GraphAdmission {
11908 final_vertices,
11909 final_edges,
11910 materialization_cells: planned_formula_count as u64,
11911 added_vertices: new_vertices,
11912 added_edges: new_edges,
11913 };
11914 let materialized_bytes = admission
11915 .materialized_graph_bytes()
11916 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11917 if let Err(error) = self.preflight_graph_admission(admission) {
11918 if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11919 self.observe_target_admission_failure(detail.reason);
11920 }
11921 return Err(error);
11922 }
11923 let selected_package_records = prepared_packages
11924 .iter()
11925 .map(|package| package.replay_records.len() as u64)
11926 .sum::<u64>();
11927 let planned_working_bytes = (prepared.len() as u64)
11928 .saturating_add(selected_package_records)
11929 .saturating_mul(256)
11930 .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11931 .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11932 .saturating_add((selected.len() as u64).saturating_mul(96))
11933 .saturating_add(materialized_bytes);
11934 let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11935 let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11936 if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11937 self.observe_target_admission_failure(
11938 formualizer_common::ResourceExhaustionReason::ScratchMemory,
11939 );
11940 return Err(error);
11941 }
11942
11943 let estimated_commit_duration = std::time::Duration::from_nanos(
11944 (new_vertices as u64)
11945 .saturating_add(new_edges as u64)
11946 .saturating_add(prepared.len() as u64)
11947 .saturating_add(selected_package_records)
11948 .max(1)
11949 .saturating_mul(100),
11950 );
11951 if options.deadline.is_some_and(|deadline| {
11952 std::time::Instant::now()
11953 .checked_add(estimated_commit_duration)
11954 .is_none_or(|finish| finish > deadline)
11955 }) {
11956 self.observe_target_admission_failure(
11957 formualizer_common::ResourceExhaustionReason::Deadline,
11958 );
11959 return Err(resource(
11960 formualizer_common::ResourceExhaustionReason::Deadline,
11961 0,
11962 1,
11963 ));
11964 }
11965 #[cfg(test)]
11966 if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11967 hook();
11968 }
11969 self.target_preparation_checkpoint(options.deadline, 0)?;
11970 #[cfg(test)]
11971 self.target_preparation_fault(
11972 crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11973 )?;
11974 let current_revisions = self.preparation_revisions();
11975 let staged_leases_match = prepared.iter().all(|formula| {
11976 self.staged_formula_index
11977 .lease_matches(&formula.sheet, formula.lease)
11978 }) && prepared_packages.iter().all(|package| {
11979 self.staged_formula_index
11980 .package_lease_matches(&package.sheet, package.lease)
11981 });
11982 let stale_reason = Self::preparation_revision_stale_reason(
11983 &assumptions,
11984 ¤t_revisions,
11985 &planning_requests,
11986 staged_leases_match,
11987 );
11988 if let Some(reason) = stale_reason {
11989 return Err(Self::preparation_stale(
11990 reason,
11991 "target graph preparation plan is stale",
11992 ));
11993 }
11994 #[cfg(test)]
11995 self.target_preparation_fault(
11996 crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11997 )?;
11998 self.graph
11999 .validate_prepared_legacy_graph_plan(&legacy_graph)
12000 .map_err(|error| {
12001 Self::preparation_stale(
12002 formualizer_common::PreparationStaleReason::Graph,
12003 format!("target graph preparation plan is stale: {error}"),
12004 )
12005 })?;
12006 if let Some(append) = formula_plane.as_ref() {
12007 self.graph
12008 .formula_authority()
12009 .validate_prepared_formula_plane_append(
12010 append,
12011 self.graph.data_store(),
12012 self.graph.sheet_reg(),
12013 )
12014 .map_err(|error| {
12015 Self::preparation_stale(
12016 formualizer_common::PreparationStaleReason::Authority,
12017 format!("target FormulaPlane preparation is stale: {error}"),
12018 )
12019 })?;
12020 }
12021 #[cfg(test)]
12022 self.target_preparation_fault(
12023 crate::engine::target_preparation::TargetPreparationFault::Reservation,
12024 )?;
12025 self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
12026 self.formula_parse_diagnostics
12027 .try_reserve(pending_diagnostics.len())
12028 .map_err(|_| {
12029 resource(
12030 formualizer_common::ResourceExhaustionReason::Admission,
12031 pending_diagnostics.len() as u64,
12032 pending_diagnostics.len() as u64,
12033 )
12034 })?;
12035 self.target_preparation_checkpoint(options.deadline, 0)?;
12036 #[cfg(test)]
12037 self.target_preparation_fault(
12038 crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
12039 )?;
12040
12041 let commit_started = crate::instant::FzInstant::now();
12042 let committed = self
12043 .graph
12044 .apply_prevalidated_legacy_graph_plan(legacy_graph);
12045 let plane_report =
12046 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
12047 formula_plane.map(|append| {
12048 self.graph
12049 .formula_authority_mut()
12050 .apply_prevalidated_formula_plane_append(append)
12051 })
12052 } else {
12053 None
12054 };
12055 if let Some(report) = plane_report.as_ref() {
12056 self.graph.mark_formula_spans_dirty(
12057 report.spans.iter().copied(),
12058 WholeSpanDirtyReason::NewSpan,
12059 );
12060 }
12061 for formula in &prepared {
12062 let removed = self
12063 .staged_formulas
12064 .get_mut(&formula.sheet)
12065 .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
12066 debug_assert!(removed.is_some());
12067 let index_removed = self.staged_formula_index.remove(
12068 &formula.sheet,
12069 formula.lease.row,
12070 formula.lease.col,
12071 );
12072 debug_assert!(index_removed);
12073 }
12074 for package in &prepared_packages {
12075 let removed = self
12076 .staged_formulas
12077 .get_mut(&package.sheet)
12078 .and_then(|staged| staged.deferred_package.take());
12079 debug_assert!(removed.is_some());
12080 self.staged_formula_index.set_package(&package.sheet, None);
12081 }
12082 let empty_sheets = self
12083 .staged_formulas
12084 .iter()
12085 .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
12086 .collect::<Vec<_>>();
12087 for sheet in empty_sheets {
12088 self.staged_formulas.remove(&sheet);
12089 }
12090 if committed > 0 || plane_report.is_some() {
12091 self.mark_topology_edited();
12092 }
12093 self.formula_parse_diagnostics.extend(pending_diagnostics);
12094 if !prepared.is_empty() || !prepared_packages.is_empty() {
12095 let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
12096 ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
12097 prepared_packages
12098 .iter()
12099 .map(|package| package.replay_records.len() as u64)
12100 .sum::<u64>(),
12101 );
12102 ingest_delta.graph_formula_cells_materialized = committed as u64;
12103 ingest_delta.graph_vertices_created = new_vertices as u64;
12104 ingest_delta.graph_edges_created = new_edges as u64;
12105 for package in &prepared_packages {
12106 let source = &package.source_report;
12107 ingest_delta.source_formula_events = ingest_delta
12108 .source_formula_events
12109 .saturating_add(source.source_formula_events);
12110 ingest_delta.source_formula_records_spooled = ingest_delta
12111 .source_formula_records_spooled
12112 .saturating_add(source.source_formula_records_spooled);
12113 ingest_delta.source_spool_encoded_bytes = ingest_delta
12114 .source_spool_encoded_bytes
12115 .saturating_add(source.source_spool_encoded_bytes);
12116 ingest_delta.source_spool_peak_memory_bytes = ingest_delta
12117 .source_spool_peak_memory_bytes
12118 .max(source.source_spool_peak_memory_bytes);
12119 ingest_delta.source_spool_spilled_bytes = ingest_delta
12120 .source_spool_spilled_bytes
12121 .saturating_add(source.source_spool_spilled_bytes);
12122 ingest_delta.source_spool_spill_files = ingest_delta
12123 .source_spool_spill_files
12124 .saturating_add(source.source_spool_spill_files);
12125 ingest_delta.source_spool_replays = ingest_delta
12126 .source_spool_replays
12127 .saturating_add(source.source_spool_replays)
12128 .saturating_add(1);
12129 ingest_delta.source_families_seen = ingest_delta
12130 .source_families_seen
12131 .saturating_add(source.families_seen);
12132 ingest_delta.source_family_cells_seen = ingest_delta
12133 .source_family_cells_seen
12134 .saturating_add(source.family_cells_seen);
12135 ingest_delta.source_family_shadow_eligible = ingest_delta
12136 .source_family_shadow_eligible
12137 .saturating_add(source.source_clean_families);
12138 ingest_delta.source_family_shadow_eligible_cells = ingest_delta
12139 .source_family_shadow_eligible_cells
12140 .saturating_add(source.source_clean_cells);
12141 ingest_delta.source_partitioned_families_seen = ingest_delta
12142 .source_partitioned_families_seen
12143 .saturating_add(source.source_fragmentable_families);
12144 ingest_delta.source_partition_holes = ingest_delta
12145 .source_partition_holes
12146 .saturating_add(source.source_hole_exclusions);
12147 ingest_delta.source_partition_ordinary_exceptions = ingest_delta
12148 .source_partition_ordinary_exceptions
12149 .saturating_add(source.source_ordinary_exclusions);
12150 ingest_delta.source_partition_surviving_cells = ingest_delta
12151 .source_partition_surviving_cells
12152 .saturating_add(source.source_fragmentable_cells);
12153 for (reason, count) in &source.fallback_reasons {
12154 let total = ingest_delta
12155 .fallback_reasons
12156 .entry(reason.clone())
12157 .or_default();
12158 *total = total.saturating_add(*count);
12159 }
12160 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12161 ingest_delta.source_family_fallback = ingest_delta
12162 .source_family_fallback
12163 .saturating_add(source.families_seen);
12164 ingest_delta.source_family_fallback_cells = ingest_delta
12165 .source_family_fallback_cells
12166 .saturating_add(source.family_cells_seen);
12167 } else {
12168 ingest_delta.shadow_candidate_cells = ingest_delta
12169 .shadow_candidate_cells
12170 .saturating_add(source.family_cells_seen);
12171 ingest_delta.shadow_accepted_span_cells = ingest_delta
12172 .shadow_accepted_span_cells
12173 .saturating_add(package.direct_cells);
12174 ingest_delta.shadow_fallback_cells =
12175 ingest_delta.shadow_fallback_cells.saturating_add(
12176 source
12177 .family_cells_seen
12178 .saturating_sub(package.direct_cells),
12179 );
12180 ingest_delta.source_anchor_parses = ingest_delta
12181 .source_anchor_parses
12182 .saturating_add(package.anchor_parses);
12183 ingest_delta.source_anchor_asts = ingest_delta
12184 .source_anchor_asts
12185 .saturating_add(package.anchor_asts);
12186 ingest_delta.source_anchor_analyses = ingest_delta
12187 .source_anchor_analyses
12188 .saturating_add(package.anchor_analyses);
12189 ingest_delta.source_compressed_families_prepared = ingest_delta
12190 .source_compressed_families_prepared
12191 .saturating_add(package.direct_complete_families);
12192 ingest_delta.source_compressed_cells_prepared = ingest_delta
12193 .source_compressed_cells_prepared
12194 .saturating_add(package.direct_complete_cells);
12195 ingest_delta.source_partitioned_families_prepared = ingest_delta
12196 .source_partitioned_families_prepared
12197 .saturating_add(package.direct_partition_families);
12198 ingest_delta.source_partition_fragments_prepared = ingest_delta
12199 .source_partition_fragments_prepared
12200 .saturating_add(package.direct_fragments);
12201 ingest_delta.source_partition_span_cells_prepared = ingest_delta
12202 .source_partition_span_cells_prepared
12203 .saturating_add(package.direct_partition_cells);
12204 ingest_delta.source_partition_analyses_reused = ingest_delta
12205 .source_partition_analyses_reused
12206 .saturating_add(
12207 package
12208 .direct_fragments
12209 .saturating_sub(package.direct_partition_families),
12210 );
12211 ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
12212 .graph_formula_vertices_avoided_shadow
12213 .saturating_add(package.direct_cells);
12214 ingest_delta.ast_roots_avoided_shadow =
12215 ingest_delta.ast_roots_avoided_shadow.saturating_add(
12216 package
12217 .direct_complete_cells
12218 .saturating_sub(package.direct_complete_families)
12219 .saturating_add(
12220 package
12221 .direct_partition_cells
12222 .saturating_sub(package.direct_fragments),
12223 ),
12224 );
12225 ingest_delta.edge_rows_avoided_shadow = ingest_delta
12226 .edge_rows_avoided_shadow
12227 .saturating_add(package.direct_cells);
12228 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
12229 {
12230 ingest_delta.source_family_promoted = ingest_delta
12231 .source_family_promoted
12232 .saturating_add(package.direct_families as u64);
12233 ingest_delta.source_family_promoted_cells = ingest_delta
12234 .source_family_promoted_cells
12235 .saturating_add(package.direct_cells);
12236 let descendants = package
12237 .direct_complete_cells
12238 .saturating_sub(package.direct_complete_families)
12239 .saturating_add(
12240 package
12241 .direct_partition_cells
12242 .saturating_sub(package.direct_partition_families),
12243 );
12244 ingest_delta.source_descendant_strings_avoided = ingest_delta
12245 .source_descendant_strings_avoided
12246 .saturating_add(descendants);
12247 ingest_delta.source_descendant_events_avoided = ingest_delta
12248 .source_descendant_events_avoided
12249 .saturating_add(descendants);
12250 ingest_delta.source_descendant_analyses_avoided = ingest_delta
12251 .source_descendant_analyses_avoided
12252 .saturating_add(descendants);
12253 }
12254 ingest_delta.source_family_fallback =
12255 ingest_delta.source_family_fallback.saturating_add(
12256 source
12257 .families_seen
12258 .saturating_sub(package.direct_families as u64),
12259 );
12260 ingest_delta.source_family_fallback_cells =
12261 ingest_delta.source_family_fallback_cells.saturating_add(
12262 source
12263 .family_cells_seen
12264 .saturating_sub(package.direct_cells),
12265 );
12266 }
12267 }
12268 if let Some(report) = plane_report.as_ref() {
12269 ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
12270 ingest_delta.shadow_spans_created = report.spans.len() as u64;
12271 }
12272 self.record_formula_ingest_report(ingest_delta);
12273 }
12274 let commit_window = commit_started.elapsed();
12275 let retained_cells = self
12276 .staged_formula_index
12277 .all_leases()
12278 .into_iter()
12279 .filter_map(|(sheet, lease)| {
12280 formualizer_common::RangeAddress::new(
12281 sheet, lease.row, lease.col, lease.row, lease.col,
12282 )
12283 .ok()
12284 })
12285 .collect::<Vec<_>>();
12286 let observed_scratch_bytes = self
12287 .active_resource_ledger
12288 .as_ref()
12289 .map_or(0, |ledger| ledger.snapshot().scratch_peak)
12290 .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
12291 let committed_spans = plane_report
12292 .as_ref()
12293 .map_or(0, |report| report.spans.len() as u64);
12294 let selected_source_families = prepared_packages
12295 .iter()
12296 .map(|package| package.lease.family_count)
12297 .sum::<usize>();
12298 let selected_staged_cells = prepared
12299 .len()
12300 .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
12301 let actual_commit_work = (new_vertices as u64)
12302 .saturating_add(new_edges as u64)
12303 .saturating_add(committed as u64)
12304 .saturating_add(committed_spans)
12305 .saturating_add(prepared.len() as u64)
12306 .saturating_add(prepared_packages.len() as u64);
12307 let report = PreparedTargetGraphReport {
12308 request_id: request_id.unwrap_or_default(),
12309 requested_targets: targets.len(),
12310 normalized_regions: visited_regions.len(),
12311 normalized_target_list: normalized,
12312 selected_staged_cells,
12313 selected_source_families,
12314 retained_staged_cells: self.staged_formula_count(),
12315 selected_cells,
12316 retained_cells,
12317 widened_scope: scope,
12318 widening_reasons: reasons,
12319 revisions: assumptions,
12320 commit_window,
12321 estimated_scratch_bytes: scratch_bytes,
12322 observed_scratch_bytes,
12323 estimated_commit_work: (new_vertices as u64)
12324 .saturating_add(new_edges as u64)
12325 .saturating_add(prepared.len() as u64)
12326 .saturating_add(selected_package_records),
12327 actual_commit_work,
12328 outcome: PreparationOutcome::Prepared,
12329 };
12330 self.observe_target_preparation_report(&report);
12331 Ok(report)
12332 }
12333
12334 pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
12336 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
12337 engine.build_graph_all_unobserved()
12338 })
12339 }
12340
12341 fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
12342 let selected = self.staged_formula_count();
12343 let started = crate::instant::FzInstant::now();
12344 self.resource_checkpoint(selected as u64)?;
12345 let scratch_bytes = (selected as u64).saturating_mul(256);
12346 let result = self.with_request_scratch(scratch_bytes, |engine| {
12347 let index_snapshot = engine.staged_formula_index.clone();
12348 let collected = std::mem::take(&mut engine.staged_formulas)
12349 .into_iter()
12350 .collect();
12351 engine.staged_formula_index.clear_all();
12352 engine.build_graph_from_staged_batches(collected, false, index_snapshot)
12353 });
12354 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
12355 result
12356 }
12357
12358 pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
12360 &mut self,
12361 sheets: I,
12362 ) -> Result<(), formualizer_parse::ExcelError> {
12363 let mut sheets = sheets.into_iter();
12364 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
12365 let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
12366 engine.with_request_scratch(name_scratch, |engine| {
12367 let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
12370 engine.charge_bounded_work(names.len() as u64)?;
12371 engine.build_graph_for_sheet_names_unobserved(names)
12372 })
12373 })
12374 }
12375
12376 fn build_graph_for_sheet_names_unobserved(
12377 &mut self,
12378 sheets: Vec<String>,
12379 ) -> Result<(), formualizer_parse::ExcelError> {
12380 let started = crate::instant::FzInstant::now();
12381 let selected = sheets
12382 .iter()
12383 .filter_map(|sheet| self.staged_formulas.get(sheet))
12384 .map(StagedSheet::len)
12385 .sum::<usize>();
12386 self.resource_checkpoint(selected as u64)?;
12387 let scratch_bytes = (selected as u64).saturating_mul(256);
12388 self.reserve_request_scratch(scratch_bytes)?;
12389 let index_snapshot = self.staged_formula_index.clone();
12390 let mut collected = Vec::new();
12391 for sheet in sheets {
12392 if let Some(staged) = self.staged_formulas.remove(&sheet) {
12393 self.index_removed_staged_sheet(&sheet, &staged);
12394 collected.push((sheet, staged));
12395 }
12396 }
12397 let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
12398 self.release_request_scratch(scratch_bytes);
12399 self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
12400 result
12401 }
12402
12403 fn build_graph_from_staged_batches(
12404 &mut self,
12405 collected: StagedFormulaBatches,
12406 share_parse_cache_across_sheets: bool,
12407 staged_index_snapshot: StagedFormulaIndex,
12408 ) -> Result<(), formualizer_parse::ExcelError> {
12409 if collected.is_empty() {
12410 return Ok(());
12411 }
12412 for (sheet, _) in &collected {
12413 let _ = self.add_sheet(sheet);
12414 }
12415
12416 let diagnostics_len = self.formula_parse_diagnostics.len();
12417 let mut collected = collected;
12418 let prepared = match self
12419 .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
12420 {
12421 Ok(prepared) => prepared,
12422 Err(error) => {
12423 self.formula_parse_diagnostics.truncate(diagnostics_len);
12424 for (sheet, staged) in collected {
12425 self.restore_staged_sheet(sheet, staged);
12426 }
12427 self.staged_formula_index = staged_index_snapshot;
12428 return Err(error);
12429 }
12430 };
12431 let (ordinary, compressed, direct) = prepared;
12432
12433 if !ordinary.is_empty()
12436 && let Err(error) = self.ingest_formula_batches(ordinary)
12437 {
12438 self.formula_parse_diagnostics.truncate(diagnostics_len);
12439 for (sheet, staged) in collected {
12440 self.restore_staged_sheet(sheet, staged);
12441 }
12442 self.staged_formula_index = staged_index_snapshot;
12443 return Err(error);
12444 }
12445 if !compressed.is_empty() {
12446 let _ = self.ingest_compressed_formula_source_batches(compressed)?;
12447 }
12448 if !direct.is_empty() {
12449 let _ = self.finish_compressed_formula_sources(direct)?;
12450 }
12451 self.dedup_formula_parse_diagnostics_since(diagnostics_len);
12452 Ok(())
12453 }
12454
12455 fn prepare_staged_formula_batches(
12456 &mut self,
12457 collected: &mut StagedFormulaBatches,
12458 share_parse_cache_across_sheets: bool,
12459 ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
12460 let mut ordinary = Vec::new();
12461 let mut compressed = Vec::new();
12462 let mut direct = Vec::new();
12463 let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
12464 rustc_hash::FxHashMap::default();
12465 cache.reserve(4096);
12466
12467 for (sheet, staged) in collected {
12468 if !share_parse_cache_across_sheets {
12469 cache.clear();
12470 }
12471 let mut entries: Vec<_> = staged
12472 .entries
12473 .iter()
12474 .cloned()
12475 .map(|(row, col, text)| (row, col, text, None))
12476 .collect();
12477 let mut deferred_source = None;
12478 let mut deferred_fallback = None;
12479 if let Some(package) = staged.deferred_package.as_mut() {
12480 if package.sheet_name != *sheet {
12481 return Err(ExcelError::new(ExcelErrorKind::Value)
12482 .with_message("deferred formula package sheet mismatch"));
12483 }
12484 let eligible: Vec<_> = package
12485 .families
12486 .iter()
12487 .filter(|family| !package.invalidated.contains(&family.source_id))
12488 .cloned()
12489 .collect();
12490 let eligible_partitions: Vec<_> = package
12491 .partitioned_families
12492 .iter()
12493 .filter(|family| !package.invalidated.contains(&family.source_id))
12494 .cloned()
12495 .collect();
12496 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
12497 let mut preparation = self.prepare_source_formula_proposals(
12498 sheet,
12499 &eligible,
12500 &eligible_partitions,
12501 &package.partitioned_families,
12502 package.report.source_formula_records_spooled,
12503 Arc::clone(&package.replay),
12504 &package.suppressed,
12505 )?;
12506 let replay_records = std::mem::take(&mut preparation.eager_replay);
12507 let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
12508 let accepted: BTreeMap<_, _> = preparation
12509 .fragmented
12510 .iter()
12511 .map(|fragment| (fragment.source.source_id, &fragment.source))
12512 .collect();
12513 replay_records.into_iter().partition(|record| {
12514 record.partition_owner.is_some_and(|owner| {
12515 accepted.get(&owner).is_some_and(|source| {
12516 Self::exact_replay_record_is_prepared_partition_legacy(
12517 source, record,
12518 )
12519 })
12520 })
12521 })
12522 };
12523 preparation.eager_replay = fragment_legacy;
12524 entries.extend(ordered_fallback.into_iter().map(|record| {
12525 (
12526 record.row,
12527 record.col,
12528 record.text,
12529 Some((record.source_order, record.family, record.partition_owner)),
12530 )
12531 }));
12532 deferred_source = Some((package.report.clone(), preparation));
12533 } else {
12534 let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
12535 for partition in &eligible_partitions {
12536 replay_disposition
12537 .register_partition(partition, false)
12538 .map_err(|reason| {
12539 ExcelError::new(ExcelErrorKind::Value).with_message(reason)
12540 })?;
12541 }
12542 replay_disposition
12543 .extend_suppressed_excel_coords(package.suppressed.iter().copied());
12544 let replayed = package
12545 .replay
12546 .lock()
12547 .map_err(|_| {
12548 ExcelError::new(ExcelErrorKind::Value)
12549 .with_message("deferred formula spool lock poisoned")
12550 })?
12551 .replay(&replay_disposition)
12552 .map_err(|message| {
12553 ExcelError::new(ExcelErrorKind::Value).with_message(message)
12554 })?;
12555 entries.extend(replayed.into_iter().map(|record| {
12556 (
12557 record.row,
12558 record.col,
12559 record.text,
12560 Some((record.source_order, record.family, record.partition_owner)),
12561 )
12562 }));
12563 let mut report = package.report.clone();
12564 report.source_spool_replays = report.source_spool_replays.saturating_add(1);
12565 deferred_fallback =
12566 Some((report, package.families.clone(), eligible_partitions));
12567 }
12568 }
12569
12570 let mut formulas = Vec::new();
12571 let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
12572 for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
12573 let key = if txt.starts_with('=') {
12574 txt
12575 } else {
12576 format!("={txt}")
12577 };
12578 let ast_id = if let Some(cached) = cache.get(&key) {
12579 *cached
12580 } else {
12581 let parsed = match formualizer_parse::parser::parse(&key) {
12582 Ok(parsed) => Some(parsed),
12583 Err(error) => self.handle_formula_parse_error(
12584 sheet,
12585 row,
12586 col,
12587 &key,
12588 error.to_string(),
12589 )?,
12590 };
12591 let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
12592 cache.insert(key.clone(), ast_id);
12593 ast_id
12594 };
12595
12596 if let Some(ast_id) = ast_id {
12597 let mut formula = FormulaIngestRecord::new(
12598 row,
12599 col,
12600 ast_id,
12601 Some(Arc::<str>::from(key.clone())),
12602 );
12603 if let Some((order, family, owner)) = source_proof {
12604 formula = formula.with_source_proof(order, family, owner);
12605 } else if deferred_source.is_some() {
12606 formula = formula.with_source_proof(
12607 crate::engine::SourceFormulaOrder::new(
12608 staged_order_base.saturating_add(entry_index as u64),
12609 ),
12610 None,
12611 None,
12612 );
12613 }
12614 formulas.push(formula);
12615 }
12616 }
12617
12618 let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
12619 if let Some((report, preparation)) = deferred_source {
12620 direct.push((batch, report, preparation));
12621 } else if let Some((report, families, partitions)) = deferred_fallback {
12622 let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
12623 batch.sheet_name.clone(),
12624 report,
12625 families,
12626 partitions,
12627 );
12628 compressed.push((batch, source_batch));
12629 } else if !batch.is_empty() {
12630 ordinary.push(batch);
12631 }
12632 }
12633 Ok((ordinary, compressed, direct))
12634 }
12635
12636 pub fn begin_bulk_ingest_arrow(
12638 &mut self,
12639 ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
12640 crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
12641 }
12642
12643 pub fn begin_bulk_update_arrow(
12645 &mut self,
12646 ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
12647 crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
12648 }
12649
12650 fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
12651 self.graph.sheet_id(sheet).ok_or(
12652 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
12653 name: sheet.to_string(),
12654 reason: "Unknown sheet".to_string(),
12655 },
12656 )
12657 }
12658
12659 fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
12660 if row_1based == 0 {
12661 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
12662 }
12663 Ok(row_1based - 1)
12664 }
12665
12666 fn normalize_row_range_1based(
12667 start_row_1based: u32,
12668 end_row_1based: u32,
12669 ) -> Result<(u32, u32), crate::engine::EditorError> {
12670 if start_row_1based == 0 || end_row_1based == 0 {
12671 return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
12672 }
12673 if start_row_1based > end_row_1based {
12674 return Err(crate::engine::EditorError::TransactionFailed {
12675 reason: "Row range start is greater than end".to_string(),
12676 });
12677 }
12678 Ok((start_row_1based - 1, end_row_1based - 1))
12679 }
12680
12681 fn invalidate_row_visibility_mask_cache(&self) {
12682 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12683 cache.clear();
12684 }
12685 }
12686
12687 fn set_row_hidden_by_sheet_id(
12688 &mut self,
12689 sheet_id: SheetId,
12690 row0: u32,
12691 hidden: bool,
12692 source: RowVisibilitySource,
12693 ) -> bool {
12694 let changed = {
12695 let state = self.row_visibility.entry(sheet_id).or_default();
12696 state.set_row_hidden(row0, hidden, source)
12697 };
12698
12699 let remove_entry = self
12700 .row_visibility
12701 .get(&sheet_id)
12702 .map(|state| state.is_empty())
12703 .unwrap_or(false);
12704 if remove_entry {
12705 self.row_visibility.remove(&sheet_id);
12706 }
12707
12708 if changed {
12709 self.invalidate_row_visibility_mask_cache();
12710 }
12711
12712 changed
12713 }
12714
12715 fn set_rows_hidden_by_sheet_id(
12716 &mut self,
12717 sheet_id: SheetId,
12718 start_row0: u32,
12719 end_row0: u32,
12720 hidden: bool,
12721 source: RowVisibilitySource,
12722 ) -> bool {
12723 let changed = {
12724 let state = self.row_visibility.entry(sheet_id).or_default();
12725 state.set_rows_hidden(start_row0, end_row0, hidden, source)
12726 };
12727
12728 let remove_entry = self
12729 .row_visibility
12730 .get(&sheet_id)
12731 .map(|state| state.is_empty())
12732 .unwrap_or(false);
12733 if remove_entry {
12734 self.row_visibility.remove(&sheet_id);
12735 }
12736
12737 if changed {
12738 self.invalidate_row_visibility_mask_cache();
12739 }
12740
12741 changed
12742 }
12743
12744 fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
12745 if count == 0 {
12746 return;
12747 }
12748 let mut changed = false;
12749 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12750 changed = state.insert_rows(before0, count);
12751 state.is_empty()
12752 } else {
12753 false
12754 };
12755 if remove_entry {
12756 self.row_visibility.remove(&sheet_id);
12757 }
12758 if changed {
12759 self.invalidate_row_visibility_mask_cache();
12760 }
12761 }
12762
12763 fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
12764 if count == 0 {
12765 return;
12766 }
12767 let mut changed = false;
12768 let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12769 changed = state.delete_rows(start0, count);
12770 state.is_empty()
12771 } else {
12772 false
12773 };
12774 if remove_entry {
12775 self.row_visibility.remove(&sheet_id);
12776 }
12777 if changed {
12778 self.invalidate_row_visibility_mask_cache();
12779 }
12780 }
12781
12782 fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12783 if let crate::engine::ChangeEvent::SetRowVisibility {
12784 sheet_id,
12785 row0,
12786 source,
12787 old_hidden,
12788 ..
12789 } = event
12790 {
12791 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12792 }
12793 }
12794
12795 fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12796 if let crate::engine::ChangeEvent::SetRowVisibility {
12797 sheet_id,
12798 row0,
12799 source,
12800 new_hidden,
12801 ..
12802 } = event
12803 {
12804 let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12805 }
12806 }
12807
12808 fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12809 for event in events.iter().rev() {
12810 self.apply_inverse_row_visibility_event(event);
12811 }
12812 }
12813
12814 fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12815 for event in events {
12816 self.apply_forward_row_visibility_event(event);
12817 }
12818 }
12819
12820 fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12821 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12822 sheet,
12823 row,
12824 col,
12825 old,
12826 ..
12827 } = event
12828 {
12829 self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12830 }
12831 }
12832
12833 fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12834 if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12835 sheet,
12836 row,
12837 col,
12838 new,
12839 ..
12840 } = event
12841 {
12842 self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12843 }
12844 }
12845
12846 fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12849 match target {
12850 Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12851 None => {
12852 self.clear_staged_formula_text(sheet, row, col);
12853 }
12854 }
12855 }
12856
12857 pub fn set_row_hidden(
12858 &mut self,
12859 sheet: &str,
12860 row_1based: u32,
12861 hidden: bool,
12862 source: RowVisibilitySource,
12863 ) -> Result<(), crate::engine::EditorError> {
12864 self.observe_function_semantic_epoch()
12865 .map_err(crate::engine::EditorError::Excel)?;
12866 self.observe_function_semantic_epoch()
12867 .map_err(crate::engine::EditorError::Excel)?;
12868 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12869 let row0 = Self::normalize_row_1based(row_1based)?;
12870 if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12871 self.record_formula_plane_structural_change(StructuralScope::Region(
12872 Region::whole_row(sheet_id, row0),
12873 ));
12874 self.mark_data_edited();
12875 }
12876 Ok(())
12877 }
12878
12879 pub fn set_rows_hidden(
12880 &mut self,
12881 sheet: &str,
12882 start_row_1based: u32,
12883 end_row_1based: u32,
12884 hidden: bool,
12885 source: RowVisibilitySource,
12886 ) -> Result<(), crate::engine::EditorError> {
12887 let sheet_id = self.ensure_known_sheet_id(sheet)?;
12888 let (start_row0, end_row0) =
12889 Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12890 if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12891 if start_row0 == end_row0 {
12892 self.record_formula_plane_structural_change(StructuralScope::Region(
12893 Region::whole_row(sheet_id, start_row0),
12894 ));
12895 } else {
12896 self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12897 }
12898 self.mark_data_edited();
12899 }
12900 Ok(())
12901 }
12902
12903 pub fn is_row_hidden(
12904 &self,
12905 sheet: &str,
12906 row_1based: u32,
12907 source: Option<RowVisibilitySource>,
12908 ) -> Option<bool> {
12909 let sheet_id = self.graph.sheet_id(sheet)?;
12910 let row0 = row_1based.checked_sub(1)?;
12911 Some(
12912 self.row_visibility
12913 .get(&sheet_id)
12914 .map(|state| state.is_row_hidden(row0, source))
12915 .unwrap_or(false),
12916 )
12917 }
12918
12919 pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12920 let sheet_id = self.graph.sheet_id(sheet)?;
12921 Some(
12922 self.row_visibility
12923 .get(&sheet_id)
12924 .map(|state| state.version())
12925 .unwrap_or(0),
12926 )
12927 }
12928
12929 fn build_row_visibility_mask_for_view(
12930 &self,
12931 view: &RangeView<'_>,
12932 mode: VisibilityMaskMode,
12933 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12934 let sheet_rows = view.sheet().nrows as usize;
12935 if sheet_rows == 0 || view.start_row() >= sheet_rows {
12936 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12937 }
12938
12939 let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12940 let start_row0 = view.start_row() as u32;
12941 let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12942 let version = self
12943 .row_visibility
12944 .get(&sheet_id)
12945 .map(|state| state.version())
12946 .unwrap_or(0);
12947 let key = VisibilityMaskCacheKey {
12948 sheet_id,
12949 start_row0,
12950 end_row0,
12951 mode,
12952 version,
12953 };
12954
12955 if let Ok(cache) = self.row_visibility_mask_cache.read()
12956 && let Some(mask) = cache.get(&key)
12957 {
12958 #[cfg(test)]
12959 visibility_mask_test_hooks::inc_hit();
12960 return Some(mask.clone());
12961 }
12962
12963 #[cfg(test)]
12964 visibility_mask_test_hooks::inc_miss();
12965
12966 let state = self.row_visibility.get(&sheet_id);
12967 let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12968 for row0 in start_row0..=end_row0 {
12969 let manual_hidden = state
12970 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12971 .unwrap_or(false);
12972 let filter_hidden = state
12973 .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12974 .unwrap_or(false);
12975
12976 let include = match mode {
12977 VisibilityMaskMode::IncludeAll => true,
12978 VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12979 VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12980 VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12981 !(manual_hidden || filter_hidden)
12982 }
12983 };
12984 out.push(include);
12985 }
12986
12987 let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12988 if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12989 const MAX_CACHE_ENTRIES: usize = 4096;
12990 if cache.len() >= MAX_CACHE_ENTRIES {
12991 cache.clear();
12992 #[cfg(test)]
12993 visibility_mask_test_hooks::inc_eviction();
12994 }
12995 cache.insert(key, mask.clone());
12996 }
12997
12998 Some(mask)
12999 }
13000
13001 fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
13002 match error {
13003 crate::engine::EditorError::Excel(error) => error,
13004 other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
13005 }
13006 }
13007
13008 fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
13009 let changes =
13010 crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
13011 let global_changed = changes.epoch != self.function_semantic_epoch_seen;
13012 let provider_revision = self.resolver.planning_semantic_revision();
13013 let provider_changed = provider_revision != self.function_provider_revision_seen;
13014 if !global_changed && !provider_changed {
13015 return Ok(false);
13016 }
13017
13018 let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
13019 let sheets: BTreeSet<_> = self
13020 .graph
13021 .formula_authority()
13022 .plane
13023 .spans
13024 .active_spans()
13025 .filter_map(|span| {
13026 let Some(template) = self
13027 .graph
13028 .formula_authority()
13029 .plane
13030 .templates
13031 .get(span.template_id)
13032 else {
13033 return (provider_changed || global_changed && !changes.complete)
13034 .then_some(span.domain.sheet_id());
13035 };
13036 let Some(ast) = self
13037 .graph
13038 .data_store()
13039 .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
13040 else {
13041 return (provider_changed || global_changed && !changes.complete)
13042 .then_some(span.domain.sheet_id());
13043 };
13044 let global_affected = global_changed
13045 && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
13046 let provider_affected = provider_changed && Self::ast_contains_function(&ast);
13047 (global_affected || provider_affected).then_some(span.domain.sheet_id())
13048 })
13049 .collect();
13050 let invalidated = !sheets.is_empty();
13051 for sheet_id in sheets {
13052 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
13053 .map_err(Self::editor_error_to_excel)?;
13054 }
13055 if global_changed && !changed.is_empty() || provider_changed {
13056 self.cached_static_schedule = None;
13057 }
13058 self.function_semantic_epoch_seen = changes.epoch;
13059 self.function_provider_revision_seen = provider_revision;
13060 Ok(invalidated)
13061 }
13062
13063 pub(crate) fn ast_uses_changed_function(
13064 ast: &ASTNode,
13065 changed: &BTreeSet<(String, String)>,
13066 ) -> bool {
13067 match &ast.node_type {
13068 ASTNodeType::Function { name, args } => {
13069 let normalized = name.to_uppercase();
13070 let mut spellings = vec![(String::new(), normalized.clone())];
13071 let mut stripped = normalized.as_str();
13072 loop {
13073 let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
13074 .iter()
13075 .find_map(|prefix| stripped.strip_prefix(prefix))
13076 else {
13077 break;
13078 };
13079 stripped = rest;
13080 spellings.push((String::new(), stripped.to_string()));
13081 }
13082 let resolved = crate::function_registry::resolve("", name);
13083 let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
13084 || resolved.as_ref().is_some_and(|resolved| {
13085 changed.contains(&(
13086 resolved.namespace.clone(),
13087 resolved.canonical_name.clone(),
13088 ))
13089 });
13090 directly_changed
13091 || resolved.is_none()
13092 || args
13093 .iter()
13094 .any(|arg| Self::ast_uses_changed_function(arg, changed))
13095 }
13096 ASTNodeType::Call { callee, args } => {
13097 Self::ast_uses_changed_function(callee, changed)
13098 || args
13099 .iter()
13100 .any(|arg| Self::ast_uses_changed_function(arg, changed))
13101 }
13102 ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
13103 ASTNodeType::BinaryOp { left, right, .. } => {
13104 Self::ast_uses_changed_function(left, changed)
13105 || Self::ast_uses_changed_function(right, changed)
13106 }
13107 ASTNodeType::Array(rows) => rows
13108 .iter()
13109 .flatten()
13110 .any(|node| Self::ast_uses_changed_function(node, changed)),
13111 _ => false,
13112 }
13113 }
13114
13115 fn ast_contains_function(ast: &ASTNode) -> bool {
13116 match &ast.node_type {
13117 ASTNodeType::Function { .. } => true,
13118 ASTNodeType::Call { callee, args } => {
13119 Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
13120 }
13121 ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
13122 ASTNodeType::BinaryOp { left, right, .. } => {
13123 Self::ast_contains_function(left) || Self::ast_contains_function(right)
13124 }
13125 ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
13126 ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
13127 }
13128 }
13129
13130 fn demote_span_containing_cell_for_write(
13131 &mut self,
13132 sheet_id: SheetId,
13133 row0: u32,
13134 col0: u32,
13135 ) -> Result<(), crate::engine::EditorError> {
13136 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
13137 return Ok(());
13138 }
13139 let placement = PlacementCoord::new(sheet_id, row0, col0);
13140 let inside_active_span = self
13141 .graph
13142 .formula_authority()
13143 .plane
13144 .spans
13145 .find_at(placement)
13146 .is_some();
13147 if inside_active_span {
13148 self.demote_spans_preserving_computed_overlays(
13149 sheet_id,
13150 Region::point(sheet_id, row0, col0),
13151 )?;
13152 }
13153 Ok(())
13154 }
13155
13156 fn demote_spans_preserving_computed_overlays(
13157 &mut self,
13158 _sheet_id: SheetId,
13159 affected_region: Region,
13160 ) -> Result<(), crate::engine::EditorError> {
13161 self.demote_spans_for_structural_op_impl(None, affected_region, false)
13165 }
13166
13167 fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
13168 Region::rows_from(sheet_id, start_row0)
13169 }
13170
13171 fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
13172 Region::cols_from(sheet_id, start_col0)
13173 }
13174
13175 fn structural_dirty_region_for_domain(
13176 domain: &PlacementDomain,
13177 op: StructuralOp,
13178 ) -> Option<Region> {
13179 let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
13180 PlacementDomain::RowRun {
13181 sheet_id,
13182 row_start,
13183 row_end,
13184 col,
13185 } => (*sheet_id, *row_start, *row_end, *col, *col),
13186 PlacementDomain::ColRun {
13187 sheet_id,
13188 row,
13189 col_start,
13190 col_end,
13191 } => (*sheet_id, *row, *row, *col_start, *col_end),
13192 PlacementDomain::Rect {
13193 sheet_id,
13194 row_start,
13195 row_end,
13196 col_start,
13197 col_end,
13198 } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
13199 };
13200 match op {
13201 StructuralOp::InsertRows {
13202 sheet_id: edited,
13203 before,
13204 ..
13205 } if sheet_id == edited && row_end >= before => Some(Region::rect(
13206 sheet_id,
13207 row_start.max(before),
13208 row_end,
13209 col_start,
13210 col_end,
13211 )),
13212 StructuralOp::DeleteRows {
13213 sheet_id: edited,
13214 start,
13215 ..
13216 } if sheet_id == edited && row_end >= start => Some(Region::rect(
13217 sheet_id,
13218 row_start.max(start),
13219 row_end,
13220 col_start,
13221 col_end,
13222 )),
13223 StructuralOp::InsertColumns {
13224 sheet_id: edited,
13225 before,
13226 ..
13227 } if sheet_id == edited && col_end >= before => Some(Region::rect(
13228 sheet_id,
13229 row_start,
13230 row_end,
13231 col_start.max(before),
13232 col_end,
13233 )),
13234 StructuralOp::DeleteColumns {
13235 sheet_id: edited,
13236 start,
13237 ..
13238 } if sheet_id == edited && col_end >= start => Some(Region::rect(
13239 sheet_id,
13240 row_start,
13241 row_end,
13242 col_start.max(start),
13243 col_end,
13244 )),
13245 _ => None,
13246 }
13247 }
13248
13249 fn span_result_region_intersects_affected(
13250 span: &crate::formula_plane::runtime::FormulaSpan,
13251 affected_region: &Region,
13252 ) -> bool {
13253 Region::from_domain(span.result_region.domain()).intersects(affected_region)
13254 }
13255
13256 fn span_any_read_region_intersects_affected(
13257 plane: &FormulaPlane,
13258 span: &crate::formula_plane::runtime::FormulaSpan,
13259 affected_region: &Region,
13260 ) -> bool {
13261 span.read_summary_id
13262 .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
13263 .is_some_and(|summary| {
13264 summary
13265 .dependencies
13266 .iter()
13267 .any(|dependency| dependency.read_region.intersects(affected_region))
13268 })
13269 }
13270
13271 fn insert_formula_plane_dirty_coords_for_span(
13272 &self,
13273 span_ref: FormulaSpanRef,
13274 dirty: ProducerDirtyDomain,
13275 out: &mut FxHashSet<(SheetId, u32, u32)>,
13276 ) -> Result<(), crate::engine::EditorError> {
13277 let authority = self.graph.formula_authority();
13278 let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
13279 ExcelError::new(ExcelErrorKind::NImpl)
13280 .with_message("FormulaPlane dirty transfer referenced a stale span")
13281 })?;
13282 match dirty {
13283 ProducerDirtyDomain::Whole => {
13284 out.extend(
13285 span.domain
13286 .iter()
13287 .map(|coord| (coord.sheet_id, coord.row, coord.col)),
13288 );
13289 }
13290 ProducerDirtyDomain::Cells(cells) => {
13291 out.extend(cells.into_iter().filter_map(|key| {
13292 let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
13293 span.domain
13294 .contains(coord)
13295 .then_some((coord.sheet_id, coord.row, coord.col))
13296 }));
13297 }
13298 ProducerDirtyDomain::Regions(regions) => {
13299 out.extend(span.domain.iter().filter_map(|coord| {
13300 let key = crate::formula_plane::region_index::RegionKey::from(coord);
13301 regions
13302 .iter()
13303 .any(|region| region.contains_key(key))
13304 .then_some((coord.sheet_id, coord.row, coord.col))
13305 }));
13306 }
13307 }
13308 Ok(())
13309 }
13310
13311 fn compute_current_formula_plane_dirty_result_coords(
13312 &self,
13313 ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
13314 use crate::formula_plane::producer::compute_dirty_closure;
13315
13316 let authority = self.graph.formula_authority();
13317 let span_refs = authority.active_span_refs();
13318 let span_refs_by_id = span_refs
13319 .iter()
13320 .copied()
13321 .map(|span_ref| (span_ref.id, span_ref))
13322 .collect::<BTreeMap<_, _>>();
13323 let mut dirty_coords = FxHashSet::default();
13324
13325 for span_ref in self.graph.pending_formula_dirty_whole_spans() {
13326 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
13327 self.insert_formula_plane_dirty_coords_for_span(
13328 span_ref,
13329 ProducerDirtyDomain::Whole,
13330 &mut dirty_coords,
13331 )?;
13332 }
13333 }
13334 for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
13335 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
13336 self.insert_formula_plane_dirty_coords_for_span(
13337 span_ref,
13338 ProducerDirtyDomain::Regions(vec![region]),
13339 &mut dirty_coords,
13340 )?;
13341 }
13342 }
13343
13344 let pending_changed_regions = self
13345 .graph
13346 .pending_formula_dirty_regions()
13347 .collect::<Vec<_>>();
13348 if pending_changed_regions.is_empty() {
13349 return Ok(dirty_coords);
13350 }
13351
13352 let closure = compute_dirty_closure(
13353 &authority.consumer_reads,
13354 pending_changed_regions,
13355 |producer| authority.producer_results.producer_result_region(producer),
13356 );
13357 if closure.incomplete {
13358 for span_ref in span_refs {
13359 self.insert_formula_plane_dirty_coords_for_span(
13360 span_ref,
13361 ProducerDirtyDomain::Whole,
13362 &mut dirty_coords,
13363 )?;
13364 }
13365 return Ok(dirty_coords);
13366 }
13367 for work in closure.work {
13368 let FormulaProducerId::Span(span_id) = work.producer else {
13369 continue;
13370 };
13371 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
13372 continue;
13373 };
13374 self.insert_formula_plane_dirty_coords_for_span(
13375 span_ref,
13376 work.dirty,
13377 &mut dirty_coords,
13378 )?;
13379 }
13380 for fallback in closure.fallbacks {
13381 let FormulaProducerId::Span(span_id) = fallback.consumer else {
13382 continue;
13383 };
13384 let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
13385 continue;
13386 };
13387 self.insert_formula_plane_dirty_coords_for_span(
13388 span_ref,
13389 ProducerDirtyDomain::Whole,
13390 &mut dirty_coords,
13391 )?;
13392 }
13393
13394 Ok(dirty_coords)
13395 }
13396
13397 fn demote_spans_for_structural_op(
13407 &mut self,
13408 op: StructuralOp,
13409 affected_region: Region,
13410 ) -> Result<(), crate::engine::EditorError> {
13411 if op.count() == 0 {
13412 return Ok(());
13413 }
13414 self.legacy_island_structural_summaries_trusted = false;
13417 self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
13418 }
13419
13420 fn indexed_structural_candidate_span_refs(
13421 &self,
13422 affected_region: Region,
13423 ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
13424 use crate::formula_plane::region_index::BoundedRegionQueryResult;
13425
13426 let authority = self.graph.formula_authority();
13427 if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
13428 return Err(ExcelError::new(ExcelErrorKind::NImpl)
13429 .with_message("FormulaPlane structural candidate indexes are stale")
13430 .into());
13431 }
13432 let indexed_region = match affected_region.axis_ranges() {
13433 (
13434 crate::formula_plane::region_index::AxisRange::From(start),
13435 crate::formula_plane::region_index::AxisRange::All,
13436 ) => Region {
13437 sheet_id: affected_region.sheet_id(),
13438 rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
13439 cols: crate::formula_plane::region_index::AxisRange::All,
13440 },
13441 (
13442 crate::formula_plane::region_index::AxisRange::All,
13443 crate::formula_plane::region_index::AxisRange::From(start),
13444 ) => Region {
13445 sheet_id: affected_region.sheet_id(),
13446 rows: crate::formula_plane::region_index::AxisRange::All,
13447 cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
13448 },
13449 _ => affected_region,
13450 };
13451 let max_candidates = self.config.max_formula_plane_cache_candidates;
13452 let result_query = match authority
13453 .producer_results
13454 .query_bounded(indexed_region, max_candidates)
13455 {
13456 BoundedRegionQueryResult::Complete(result) => result,
13457 BoundedRegionQueryResult::Incomplete {
13458 observed_candidates,
13459 } => {
13460 return Err(ExcelError::new(ExcelErrorKind::NImpl)
13461 .with_message(format!(
13462 "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
13463 ))
13464 .into());
13465 }
13466 };
13467 let remaining = max_candidates.saturating_sub(result_query.matches.len());
13468 let read_query = match authority
13469 .consumer_reads
13470 .query_changed_region_bounded(indexed_region, remaining)
13471 {
13472 BoundedRegionQueryResult::Complete(result) => result,
13473 BoundedRegionQueryResult::Incomplete { .. } => {
13474 return Err(ExcelError::new(ExcelErrorKind::NImpl)
13475 .with_message("FormulaPlane structural read candidate limit exceeded")
13476 .into());
13477 }
13478 };
13479
13480 let mut ids = BTreeSet::new();
13481 for matched in result_query.matches {
13482 if let FormulaProducerId::Span(id) = matched.value.producer {
13483 ids.insert(id);
13484 }
13485 }
13486 for matched in read_query.matches {
13487 if let FormulaProducerId::Span(id) = matched.value.consumer {
13488 ids.insert(id);
13489 }
13490 }
13491
13492 ids.into_iter()
13493 .map(|id| {
13494 authority.plane.spans.current_ref(id).ok_or_else(|| {
13495 crate::engine::EditorError::Excel(
13496 ExcelError::new(ExcelErrorKind::NImpl)
13497 .with_message("FormulaPlane structural index referenced a stale span"),
13498 )
13499 })
13500 })
13501 .collect()
13502 }
13503
13504 fn demote_spans_for_structural_op_impl(
13505 &mut self,
13506 op: Option<StructuralOp>,
13507 affected_region: Region,
13508 clear_computed_overlays: bool,
13509 ) -> Result<(), crate::engine::EditorError> {
13510 struct SpanPlan {
13511 span_ref: FormulaSpanRef,
13512 sheet_id: SheetId,
13513 ast: ASTNode,
13514 origin_row: u32,
13515 origin_col: u32,
13516 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13517 placements: Vec<(u32, u32)>,
13518 }
13519
13520 fn substitute_literal_slots_for_template_placement(
13521 ast: &ASTNode,
13522 binding: &[LiteralValue],
13523 ) -> ASTNode {
13524 fn clone_with_slots(
13525 ast: &ASTNode,
13526 binding: &[LiteralValue],
13527 next: &mut usize,
13528 in_array: bool,
13529 ) -> ASTNode {
13530 let node_type = match &ast.node_type {
13531 ASTNodeType::Literal(_) if !in_array => {
13532 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
13533 *next = next.saturating_add(1);
13534 ASTNodeType::Literal(value)
13535 }
13536 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
13537 ASTNodeType::Omitted => ASTNodeType::Omitted,
13538 ASTNodeType::Reference {
13539 original,
13540 reference,
13541 } => ASTNodeType::Reference {
13542 original: original.clone(),
13543 reference: reference.clone(),
13544 },
13545 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
13546 op: op.clone(),
13547 expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
13548 },
13549 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
13550 op: op.clone(),
13551 left: Box::new(clone_with_slots(left, binding, next, in_array)),
13552 right: Box::new(clone_with_slots(right, binding, next, in_array)),
13553 },
13554 ASTNodeType::Function { name, args } => ASTNodeType::Function {
13555 name: name.clone(),
13556 args: args
13557 .iter()
13558 .map(|arg| clone_with_slots(arg, binding, next, in_array))
13559 .collect(),
13560 },
13561 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
13562 callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
13563 args: args
13564 .iter()
13565 .map(|arg| clone_with_slots(arg, binding, next, in_array))
13566 .collect(),
13567 },
13568 ASTNodeType::Array(rows) => ASTNodeType::Array(
13569 rows.iter()
13570 .map(|row| {
13571 row.iter()
13572 .map(|cell| clone_with_slots(cell, binding, next, true))
13573 .collect()
13574 })
13575 .collect(),
13576 ),
13577 };
13578 ASTNode::new(node_type, ast.source_token.clone())
13579 }
13580 let mut next = 0usize;
13581 clone_with_slots(ast, binding, &mut next, false)
13582 }
13583
13584 let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
13585 self.formula_plane_structural_span_candidates = self
13586 .formula_plane_structural_span_candidates
13587 .saturating_add(span_refs.len() as u64);
13588 if span_refs.is_empty() {
13589 return Ok(());
13590 }
13591 let dirty_span_coords = if clear_computed_overlays {
13592 FxHashSet::default()
13593 } else {
13594 self.compute_current_formula_plane_dirty_result_coords()?
13595 };
13596
13597 struct SpanTemplateRewrite {
13604 adjusted_ast: ASTNode,
13605 exact_canonical_key: Arc<str>,
13606 parameterized_canonical_key: Arc<str>,
13607 formula_text: Option<Arc<str>>,
13608 canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
13614 value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
13615 }
13616
13617 struct ShiftPlan {
13618 span_ref: FormulaSpanRef,
13619 template_id: crate::formula_plane::ids::FormulaTemplateId,
13620 new_origin_row: u32,
13621 new_origin_col: u32,
13622 new_domain: crate::formula_plane::runtime::PlacementDomain,
13623 new_read_summary: Option<SpanReadSummary>,
13624 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13625 force_binding_residual_axes: bool,
13626 rewrite: Option<SpanTemplateRewrite>,
13627 dirty_region: Option<Region>,
13628 }
13629
13630 fn shift_operation_for(
13633 op: StructuralOp,
13634 ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
13635 use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
13636 match op {
13637 StructuralOp::InsertRows {
13638 sheet_id,
13639 before,
13640 count,
13641 } => ShiftOperation::InsertRows {
13642 sheet_id,
13643 before,
13644 count,
13645 },
13646 StructuralOp::DeleteRows {
13647 sheet_id,
13648 start,
13649 count,
13650 } => ShiftOperation::DeleteRows {
13651 sheet_id,
13652 start,
13653 count,
13654 },
13655 StructuralOp::InsertColumns {
13656 sheet_id,
13657 before,
13658 count,
13659 } => ShiftOperation::InsertColumns {
13660 sheet_id,
13661 before,
13662 count,
13663 },
13664 StructuralOp::DeleteColumns {
13665 sheet_id,
13666 start,
13667 count,
13668 } => ShiftOperation::DeleteColumns {
13669 sheet_id,
13670 start,
13671 count,
13672 },
13673 }
13674 }
13675
13676 struct SplitPlan {
13681 span_ref: FormulaSpanRef,
13682 sheet_id: SheetId,
13683 template_id: crate::formula_plane::ids::FormulaTemplateId,
13684 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13685 is_constant_result: bool,
13686 upper_domain: crate::formula_plane::runtime::PlacementDomain,
13687 upper_read_summary: Option<SpanReadSummary>,
13688 lower_new_origin_row: u32,
13689 lower_new_origin_col: u32,
13690 lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
13691 lower_new_read_summary: Option<SpanReadSummary>,
13692 lower_force_binding_residual_axes: bool,
13693 lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
13694 upper_dirty_region: Option<Region>,
13695 lower_dirty_region: Option<Region>,
13696 }
13697
13698 fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
13699 u32::try_from(i64::from(value).checked_add(delta)?).ok()
13700 }
13701
13702 fn shifted_read_summary(
13713 read_summary: &SpanReadSummary,
13714 new_result_region: Region,
13715 op: StructuralOp,
13716 row_delta: i64,
13717 col_delta: i64,
13718 origin_row_delta: i64,
13719 origin_col_delta: i64,
13720 ) -> Option<SpanReadSummary> {
13721 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13722 for dependency in &read_summary.dependencies {
13723 let read_region = match op.classify_region(dependency.read_region) {
13724 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13725 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13726 dependency.read_region
13727 }
13728 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13729 ..
13730 } => dependency
13731 .read_region
13732 .project_through_axis_shift(row_delta, col_delta)?,
13733 crate::formula_plane::structural_shift::AxisShiftCase::Straddles
13734 | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13735 return None;
13736 }
13737 };
13738 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13739 read_region,
13740 projection: dependency
13741 .projection
13742 .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
13743 });
13744 }
13745 Some(SpanReadSummary {
13746 result_region: new_result_region,
13747 dependencies,
13748 })
13749 }
13750
13751 fn compact_axis_through_delete(
13752 min: u32,
13753 max: u32,
13754 start: u32,
13755 count: u32,
13756 ) -> Option<(u32, u32)> {
13757 let end = start.saturating_add(count);
13758 if max < start || min >= end {
13759 return Some((min.saturating_sub(count), max.saturating_sub(count)));
13760 }
13761 let keeps_left = min < start;
13762 let keeps_right = max >= end;
13763 match (keeps_left, keeps_right) {
13764 (false, false) => None,
13765 (true, false) => Some((min, start.checked_sub(1)?)),
13766 (false, true) => Some((start, max.checked_sub(count)?)),
13767 (true, true) => Some((min, max.checked_sub(count)?)),
13768 }
13769 }
13770
13771 fn compact_domain_through_delete(
13772 domain: &PlacementDomain,
13773 op: StructuralOp,
13774 ) -> Option<PlacementDomain> {
13775 match (domain, op) {
13776 (
13777 PlacementDomain::RowRun {
13778 sheet_id,
13779 row_start,
13780 row_end,
13781 col,
13782 },
13783 StructuralOp::DeleteRows { start, count, .. },
13784 ) => {
13785 let (row_start, row_end) =
13786 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13787 Some(PlacementDomain::row_run(
13788 *sheet_id, row_start, row_end, *col,
13789 ))
13790 }
13791 (
13792 PlacementDomain::Rect {
13793 sheet_id,
13794 row_start,
13795 row_end,
13796 col_start,
13797 col_end,
13798 },
13799 StructuralOp::DeleteRows { start, count, .. },
13800 ) => {
13801 let (row_start, row_end) =
13802 compact_axis_through_delete(*row_start, *row_end, start, count)?;
13803 Some(PlacementDomain::rect(
13804 *sheet_id, row_start, row_end, *col_start, *col_end,
13805 ))
13806 }
13807 (
13808 PlacementDomain::ColRun {
13809 sheet_id,
13810 row,
13811 col_start,
13812 col_end,
13813 },
13814 StructuralOp::DeleteColumns { start, count, .. },
13815 ) => {
13816 let (col_start, col_end) =
13817 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13818 Some(PlacementDomain::col_run(
13819 *sheet_id, *row, col_start, col_end,
13820 ))
13821 }
13822 (
13823 PlacementDomain::Rect {
13824 sheet_id,
13825 row_start,
13826 row_end,
13827 col_start,
13828 col_end,
13829 },
13830 StructuralOp::DeleteColumns { start, count, .. },
13831 ) => {
13832 let (col_start, col_end) =
13833 compact_axis_through_delete(*col_start, *col_end, start, count)?;
13834 Some(PlacementDomain::rect(
13835 *sheet_id, *row_start, *row_end, col_start, col_end,
13836 ))
13837 }
13838 _ => None,
13839 }
13840 }
13841
13842 fn compact_axis_range_through_delete(
13843 axis: crate::formula_plane::region_index::AxisRange,
13844 start: u32,
13845 count: u32,
13846 ) -> Option<crate::formula_plane::region_index::AxisRange> {
13847 use crate::formula_plane::region_index::AxisRange;
13848 match axis {
13849 AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13850 .map(|(point, _)| AxisRange::Point(point)),
13851 AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13852 .map(|(min, max)| AxisRange::Span(min, max)),
13853 AxisRange::All => Some(AxisRange::All),
13854 AxisRange::From(_) | AxisRange::To(_) => None,
13855 }
13856 }
13857
13858 fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13859 let (rows, cols) = region.axis_ranges();
13860 match op {
13861 StructuralOp::DeleteRows {
13862 sheet_id,
13863 start,
13864 count,
13865 } if region.sheet_id() == sheet_id => Some(Region {
13866 sheet_id,
13867 rows: compact_axis_range_through_delete(rows, start, count)?,
13868 cols,
13869 }),
13870 StructuralOp::DeleteColumns {
13871 sheet_id,
13872 start,
13873 count,
13874 } if region.sheet_id() == sheet_id => Some(Region {
13875 sheet_id,
13876 rows,
13877 cols: compact_axis_range_through_delete(cols, start, count)?,
13878 }),
13879 _ => Some(region),
13880 }
13881 }
13882
13883 fn compact_read_summary_through_delete(
13884 read_summary: &SpanReadSummary,
13885 new_result_region: Region,
13886 op: StructuralOp,
13887 ) -> Option<SpanReadSummary> {
13888 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13889 for dependency in &read_summary.dependencies {
13890 let read_region = match op.classify_region(dependency.read_region) {
13891 crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13892 | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13893 dependency.read_region
13894 }
13895 crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13896 ..
13897 } => {
13898 let (row_delta, col_delta) = op.axis_shift_delta();
13899 dependency
13900 .read_region
13901 .project_through_axis_shift(row_delta, col_delta)?
13902 }
13903 crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13904 compact_region_through_delete(dependency.read_region, op)?
13905 }
13906 crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13907 return None;
13908 }
13909 };
13910 dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13911 read_region,
13912 projection: dependency.projection,
13913 });
13914 }
13915 Some(SpanReadSummary {
13916 result_region: new_result_region,
13917 dependencies,
13918 })
13919 }
13920
13921 fn rederive_read_summary_for_result(
13928 read_summary: &SpanReadSummary,
13929 result_region: Region,
13930 ) -> Option<SpanReadSummary> {
13931 let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13932 for dependency in &read_summary.dependencies {
13933 let sheet_id = dependency.read_region.sheet_id();
13934 let read_regions = dependency
13935 .projection
13936 .read_regions_for_result(sheet_id, result_region)
13937 .ok()?;
13938 for read_region in read_regions {
13939 let dependency = crate::formula_plane::producer::SpanReadDependency {
13940 read_region,
13941 projection: dependency.projection,
13942 };
13943 if !dependencies.contains(&dependency) {
13944 dependencies.push(dependency);
13945 }
13946 }
13947 }
13948 Some(SpanReadSummary {
13949 result_region,
13950 dependencies,
13951 })
13952 }
13953
13954 fn plan_span_split(
13964 authority: &crate::formula_plane::authority::FormulaAuthority,
13965 span: &crate::formula_plane::runtime::FormulaSpan,
13966 span_ref: FormulaSpanRef,
13967 read_summary: Option<&SpanReadSummary>,
13968 op: StructuralOp,
13969 ) -> Option<SplitPlan> {
13970 use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13971
13972 if span.intrinsic_mask_id.is_some() {
13973 return None;
13975 }
13976 let binding_set = span
13977 .binding_set_id
13978 .and_then(|id| authority.plane.binding_sets.get(id));
13979 if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13980 return None;
13984 }
13985
13986 let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13987 let upper_result_region = Region::from_domain(&upper_domain);
13988 let lower_result_region = Region::from_domain(&lower_domain);
13989 let (upper_read_summary, lower_read_summary) = match read_summary {
13990 Some(summary) => (
13991 Some(rederive_read_summary_for_result(
13992 summary,
13993 upper_result_region,
13994 )?),
13995 Some(rederive_read_summary_for_result(
13996 summary,
13997 lower_result_region,
13998 )?),
13999 ),
14000 None => (None, None),
14001 };
14002
14003 let upper_span = crate::formula_plane::runtime::FormulaSpan {
14007 domain: upper_domain.clone(),
14008 result_region: ResultRegion::scalar_cells(upper_domain.clone()),
14009 ..span.clone()
14010 };
14011 if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
14012 != SpanShiftPlan::NoOp
14013 {
14014 return None;
14015 }
14016 let lower_span = crate::formula_plane::runtime::FormulaSpan {
14017 domain: lower_domain.clone(),
14018 result_region: ResultRegion::scalar_cells(lower_domain.clone()),
14019 ..span.clone()
14020 };
14021 let SpanShiftPlan::Shift {
14022 row_delta,
14023 col_delta,
14024 origin_row_delta,
14025 origin_col_delta,
14026 rewrite_absolute_reads,
14027 } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
14028 else {
14029 return None;
14030 };
14031 if rewrite_absolute_reads {
14032 return None;
14038 }
14039
14040 let lower_new_origin_row =
14044 checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
14045 let lower_new_origin_col =
14046 checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
14047 let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
14048 let lower_new_result_region = Region::from_domain(&lower_new_domain);
14049 let lower_new_read_summary = match lower_read_summary.as_ref() {
14050 Some(summary) => Some(shifted_read_summary(
14051 summary,
14052 lower_new_result_region,
14053 op,
14054 row_delta,
14055 col_delta,
14056 origin_row_delta,
14057 origin_col_delta,
14058 )?),
14059 None => None,
14060 };
14061
14062 let mut lower_overlay_refs = Vec::new();
14067 for overlay_ref in authority
14068 .plane
14069 .formula_overlay
14070 .refs_for_source_span(span_ref)
14071 {
14072 let entry = authority.plane.formula_overlay.get(overlay_ref)?;
14073 match op.classify_region(Region::from_domain(&entry.domain)) {
14074 AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
14075 AxisShiftCase::EntirelyAboveShift { .. } => {
14076 lower_overlay_refs.push(overlay_ref);
14077 }
14078 AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
14079 return None;
14080 }
14081 }
14082 }
14083
14084 let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
14085 !binding_set.value_ref_slots.is_empty()
14086 && (origin_row_delta != 0 || origin_col_delta != 0)
14087 });
14088
14089 Some(SplitPlan {
14090 span_ref,
14091 sheet_id: span.sheet_id,
14092 template_id: span.template_id,
14093 binding_set_id: span.binding_set_id,
14094 is_constant_result: span.is_constant_result,
14095 upper_domain,
14096 upper_read_summary,
14097 lower_new_origin_row,
14098 lower_new_origin_col,
14099 lower_new_domain,
14100 lower_new_read_summary,
14101 lower_force_binding_residual_axes,
14102 lower_overlay_refs,
14103 upper_dirty_region: None,
14104 lower_dirty_region: None,
14105 })
14106 }
14107
14108 fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
14114 let origin0 = origin.checked_sub(1)?;
14115 let end = start.saturating_add(count);
14116 if origin0 < start {
14117 Some(origin)
14118 } else if origin0 >= end {
14119 Some(origin - count)
14120 } else {
14121 None
14122 }
14123 }
14124
14125 fn origin_through_delete(
14132 origin_row: u32,
14133 origin_col: u32,
14134 op: StructuralOp,
14135 ) -> Option<(u32, u32)> {
14136 match op {
14137 StructuralOp::DeleteRows { start, count, .. } => Some((
14138 origin_coord_through_delete(origin_row, start, count)?,
14139 origin_col,
14140 )),
14141 StructuralOp::DeleteColumns { start, count, .. } => Some((
14142 origin_row,
14143 origin_coord_through_delete(origin_col, start, count)?,
14144 )),
14145 StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
14146 }
14147 }
14148
14149 let mut shift_plans = Vec::new();
14150 let mut split_plans = Vec::new();
14151 let mut remove_refs = Vec::new();
14152 let mut demote_refs = Vec::new();
14153 for span_ref in span_refs {
14154 let authority = self.graph.formula_authority();
14155 let Some(span) = authority.plane.spans.get(span_ref) else {
14156 continue;
14157 };
14158 let read_summary = span
14159 .read_summary_id
14160 .and_then(|id| authority.plane.span_read_summaries.get(id));
14161 let Some(op) = op else {
14162 let result_region_affected =
14167 Self::span_result_region_intersects_affected(span, &affected_region);
14168 let read_region_affected = Self::span_any_read_region_intersects_affected(
14169 &authority.plane,
14170 span,
14171 &affected_region,
14172 );
14173 if result_region_affected || read_region_affected {
14174 demote_refs.push(span_ref);
14175 }
14176 continue;
14177 };
14178 match classify_span_for_op(span, read_summary, op) {
14179 SpanShiftPlan::NoOp => {}
14180 SpanShiftPlan::Remove => {
14181 remove_refs.push(span_ref);
14182 }
14183 SpanShiftPlan::Demote {
14184 reason:
14185 crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
14186 } => {
14187 let Some(template) = authority.plane.templates.get(span.template_id) else {
14188 return Err(ExcelError::new(ExcelErrorKind::Ref)
14189 .with_message(
14190 "FormulaPlane delete compaction found a span with a missing template",
14191 )
14192 .into());
14193 };
14194 let binding_compaction_safe = span
14195 .binding_set_id
14196 .and_then(|id| authority.plane.binding_sets.get(id))
14197 .is_none_or(|binding_set| binding_set.is_single_literal_binding());
14198 let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
14203 !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
14204 summary, op,
14205 )
14206 });
14207 let compaction_frame_sound = read_summary.is_none_or(|summary| {
14213 crate::formula_plane::structural_shift::delete_compaction_frame_is_sound(
14214 summary,
14215 &span.domain,
14216 template.origin_row,
14217 template.origin_col,
14218 op,
14219 )
14220 });
14221 let mut compaction_plan = None;
14222 if binding_compaction_safe
14223 && absolute_reads_compaction_safe
14224 && compaction_frame_sound
14225 && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
14226 {
14227 let new_result_region = Region::from_domain(&new_domain);
14228 let new_read_summary = if let Some(summary) = read_summary {
14229 compact_read_summary_through_delete(summary, new_result_region, op)
14230 } else {
14231 None
14232 };
14233 let carried_origin =
14234 origin_through_delete(template.origin_row, template.origin_col, op);
14235 if (read_summary.is_none() || new_read_summary.is_some())
14236 && let Some((new_origin_row, new_origin_col)) = carried_origin
14237 {
14238 let force_binding_residual_axes = span
14239 .binding_set_id
14240 .and_then(|id| authority.plane.binding_sets.get(id))
14241 .is_some_and(|binding_set| {
14242 !binding_set.value_ref_slots.is_empty()
14243 && (new_origin_row != template.origin_row
14244 || new_origin_col != template.origin_col)
14245 });
14246 let dirty_region =
14247 Self::structural_dirty_region_for_domain(&new_domain, op);
14248 compaction_plan = Some(ShiftPlan {
14249 span_ref,
14250 template_id: span.template_id,
14251 new_origin_row,
14252 new_origin_col,
14253 new_domain,
14254 new_read_summary,
14255 binding_set_id: span.binding_set_id,
14256 force_binding_residual_axes,
14257 rewrite: None,
14258 dirty_region,
14259 });
14260 }
14261 }
14262 match compaction_plan {
14263 Some(plan) => shift_plans.push(plan),
14264 None => match plan_span_split(authority, span, span_ref, read_summary, op) {
14271 Some(mut plan) => {
14272 plan.upper_dirty_region =
14273 Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
14274 plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
14275 &plan.lower_new_domain,
14276 op,
14277 );
14278 split_plans.push(plan);
14279 }
14280 None => demote_refs.push(span_ref),
14281 },
14282 }
14283 }
14284 SpanShiftPlan::Demote { .. } => {
14285 demote_refs.push(span_ref);
14286 }
14287 SpanShiftPlan::Split => {
14288 match plan_span_split(authority, span, span_ref, read_summary, op) {
14289 Some(mut plan) => {
14290 plan.upper_dirty_region =
14291 Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
14292 plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
14293 &plan.lower_new_domain,
14294 op,
14295 );
14296 split_plans.push(plan);
14297 }
14298 None => demote_refs.push(span_ref),
14301 }
14302 }
14303 SpanShiftPlan::Shift {
14304 row_delta,
14305 col_delta,
14306 origin_row_delta,
14307 origin_col_delta,
14308 rewrite_absolute_reads,
14309 } => {
14310 let Some(template) = authority.plane.templates.get(span.template_id) else {
14311 return Err(ExcelError::new(ExcelErrorKind::Ref)
14312 .with_message("FormulaPlane shift found a span with a missing template")
14313 .into());
14314 };
14315 let Some(new_origin_row) =
14316 checked_shift_u32(template.origin_row, origin_row_delta)
14317 else {
14318 return Err(ExcelError::new(ExcelErrorKind::Ref)
14319 .with_message("FormulaPlane shift overflowed template origin row")
14320 .into());
14321 };
14322 let Some(new_origin_col) =
14323 checked_shift_u32(template.origin_col, origin_col_delta)
14324 else {
14325 return Err(ExcelError::new(ExcelErrorKind::Ref)
14326 .with_message("FormulaPlane shift overflowed template origin column")
14327 .into());
14328 };
14329 let Some(new_domain) =
14330 span.domain.project_through_axis_shift(row_delta, col_delta)
14331 else {
14332 return Err(ExcelError::new(ExcelErrorKind::Ref)
14333 .with_message("FormulaPlane shift overflowed span domain")
14334 .into());
14335 };
14336 let new_result_region = Region::from_domain(&new_domain);
14337 let force_binding_residual_axes = span
14338 .binding_set_id
14339 .and_then(|id| authority.plane.binding_sets.get(id))
14340 .is_some_and(|binding_set| {
14341 !binding_set.value_ref_slots.is_empty()
14342 && (origin_row_delta != 0 || origin_col_delta != 0)
14343 });
14344 let rewrite_and_summary = if rewrite_absolute_reads {
14345 let frame_sound = read_summary.is_some_and(|summary| {
14366 crate::formula_plane::structural_shift::rewrite_frame_is_sound(
14367 summary,
14368 template.origin_row,
14369 template.origin_col,
14370 op,
14371 )
14372 });
14373 if !frame_sound {
14374 demote_refs.push(span_ref);
14375 continue;
14376 }
14377 let Some(ast) = self
14378 .graph
14379 .data_store()
14380 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
14381 else {
14382 demote_refs.push(span_ref);
14383 continue;
14384 };
14385 let shift_op = shift_operation_for(op);
14386 let adjuster =
14387 crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
14388 let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
14389 span.sheet_id,
14390 self.graph.sheet_reg(),
14391 );
14392 let Some(adjusted_ast) =
14393 adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
14394 else {
14395 demote_refs.push(span_ref);
14400 continue;
14401 };
14402 let canonical = crate::formula_plane::template_canonical::canonicalize_template(
14403 &adjusted_ast,
14404 new_origin_row,
14405 new_origin_col,
14406 );
14407 if !canonical.labels.is_authority_supported() {
14408 demote_refs.push(span_ref);
14409 continue;
14410 }
14411 let literal_slots_compatible = span
14416 .binding_set_id
14417 .and_then(|id| authority.plane.binding_sets.get(id))
14418 .is_none_or(|binding_set| {
14419 binding_set.literal_slots.as_ref()
14420 == canonical.literal_slot_descriptors.as_ref()
14421 });
14422 if !literal_slots_compatible {
14423 demote_refs.push(span_ref);
14424 continue;
14425 }
14426 let value_ref_slots = Arc::from(
14434 crate::formula_plane::placement::value_ref_slot_descriptors(
14435 &canonical.expr,
14436 )
14437 .into_boxed_slice(),
14438 );
14439 let dependency_summary =
14450 crate::formula_plane::dependency_summary::summarize_canonical_template(
14451 &canonical,
14452 );
14453 let rewritten_result_region =
14454 ResultRegion::scalar_cells(new_domain.clone());
14455 let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
14456 span.sheet_id,
14457 &rewritten_result_region,
14458 &dependency_summary,
14459 self.graph.sheet_reg(),
14460 ) else {
14461 demote_refs.push(span_ref);
14462 continue;
14463 };
14464 let formula_text = Some(Arc::<str>::from(
14465 formualizer_parse::pretty::canonical_formula(&adjusted_ast),
14466 ));
14467 Some((
14468 SpanTemplateRewrite {
14469 exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
14470 parameterized_canonical_key: Arc::<str>::from(
14471 canonical.parameterized_key.payload(),
14472 ),
14473 formula_text,
14474 canonical_expr: canonical.expr,
14475 value_ref_slots,
14476 adjusted_ast,
14477 },
14478 rederived_summary,
14479 ))
14480 } else {
14481 None
14482 };
14483 let (rewrite, new_read_summary) = match rewrite_and_summary {
14484 Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
14485 None => {
14486 let new_read_summary = if let Some(summary) = read_summary {
14487 Some(
14488 shifted_read_summary(
14489 summary,
14490 new_result_region,
14491 op,
14492 row_delta,
14493 col_delta,
14494 origin_row_delta,
14495 origin_col_delta,
14496 )
14497 .ok_or_else(|| {
14498 ExcelError::new(ExcelErrorKind::Ref).with_message(
14499 "FormulaPlane shift could not project read summary",
14500 )
14501 })?,
14502 )
14503 } else {
14504 None
14505 };
14506 (None, new_read_summary)
14507 }
14508 };
14509 let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
14510 shift_plans.push(ShiftPlan {
14511 span_ref,
14512 template_id: span.template_id,
14513 new_origin_row,
14514 new_origin_col,
14515 new_domain,
14516 new_read_summary,
14517 binding_set_id: span.binding_set_id,
14518 force_binding_residual_axes,
14519 rewrite,
14520 dirty_region,
14521 });
14522 }
14523 }
14524 }
14525 let span_geometry_changed =
14526 !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
14527 let mut span_dirty_deltas = Vec::new();
14528 if span_geometry_changed {
14529 let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
14533 for plan in shift_plans {
14534 let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
14535 let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
14536 let template_slot_map =
14537 crate::formula_plane::placement::build_template_slot_map(
14538 ast_id,
14539 self.graph.data_store(),
14540 &rewrite.canonical_expr,
14541 );
14542 (ast_id, template_slot_map)
14543 });
14544 prepared_shift_plans.push((plan, rewrite_prepared));
14545 }
14546 let authority = self.graph.formula_authority_mut();
14547 for span_ref in remove_refs {
14548 authority.plane.remove_overlays_for_source_span(span_ref);
14549 authority.plane.remove_span(span_ref);
14550 }
14551 for (plan, rewrite_prepared) in prepared_shift_plans {
14552 let dirty_region = plan.dirty_region;
14553 let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
14554 let Some((ast_id, template_slot_map)) = rewrite_prepared else {
14555 return Err(ExcelError::new(ExcelErrorKind::Ref)
14556 .with_message("FormulaPlane shift lost a rewritten template AST")
14557 .into());
14558 };
14559 let Some(template_id) = authority.plane.intern_rewritten_template(
14560 plan.template_id,
14561 ast_id,
14562 rewrite.exact_canonical_key,
14563 rewrite.parameterized_canonical_key,
14564 rewrite.formula_text,
14565 plan.new_origin_row,
14566 plan.new_origin_col,
14567 ) else {
14568 return Err(ExcelError::new(ExcelErrorKind::Ref)
14569 .with_message("FormulaPlane shift could not intern rewritten template")
14570 .into());
14571 };
14572 (
14573 template_id,
14574 Some((template_slot_map, rewrite.value_ref_slots)),
14575 )
14576 } else {
14577 let Some(template_id) = authority.plane.intern_shifted_template_origin(
14578 plan.template_id,
14579 plan.new_origin_row,
14580 plan.new_origin_col,
14581 ) else {
14582 return Err(ExcelError::new(ExcelErrorKind::Ref)
14583 .with_message("FormulaPlane shift could not clone template origin")
14584 .into());
14585 };
14586 (template_id, None)
14587 };
14588 if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
14589 && let Some(binding_set_id) = plan.binding_set_id
14590 {
14591 authority.plane.set_binding_template_slots(
14592 binding_set_id,
14593 template_slot_map,
14594 value_ref_slots,
14595 );
14596 }
14597 if let Some(binding_set_id) = plan.binding_set_id {
14598 let Some(template) = authority.plane.templates.get(template_id) else {
14599 return Err(ExcelError::new(ExcelErrorKind::Ref)
14600 .with_message("FormulaPlane shift could not find shifted template")
14601 .into());
14602 };
14603 let (ast_id, origin_row, origin_col) =
14604 (template.ast_id, template.origin_row, template.origin_col);
14605 authority.plane.set_binding_template_anchor(
14606 binding_set_id,
14607 ast_id,
14608 origin_row,
14609 origin_col,
14610 );
14611 }
14612 let read_summary_id = plan
14613 .new_read_summary
14614 .map(|summary| authority.plane.insert_span_read_summary(summary));
14615 let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
14616 if !authority.plane.replace_span_geometry(
14617 plan.span_ref,
14618 template_id,
14619 plan.new_domain,
14620 result_region,
14621 read_summary_id,
14622 ) {
14623 return Err(ExcelError::new(ExcelErrorKind::Ref)
14624 .with_message("FormulaPlane shift could not update span geometry")
14625 .into());
14626 }
14627 if let Some(region) = dirty_region {
14628 span_dirty_deltas.push((plan.span_ref, region));
14629 }
14630 if plan.force_binding_residual_axes
14631 && let Some(binding_set_id) = plan.binding_set_id
14632 {
14633 authority.plane.force_binding_residual_axes(binding_set_id);
14642 }
14643 }
14644 for plan in split_plans {
14645 let upper_dirty_region = plan.upper_dirty_region;
14646 let lower_dirty_region = plan.lower_dirty_region;
14647 let Some(split_relocation) = authority
14648 .plane
14649 .spans
14650 .get(plan.span_ref)
14651 .map(|span| span.ast_relocation)
14652 else {
14653 return Err(ExcelError::new(ExcelErrorKind::Ref)
14654 .with_message("FormulaPlane split lost its anchor AST state")
14655 .into());
14656 };
14657 let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
14660 plan.template_id,
14661 plan.lower_new_origin_row,
14662 plan.lower_new_origin_col,
14663 ) else {
14664 return Err(ExcelError::new(ExcelErrorKind::Ref)
14665 .with_message("FormulaPlane split could not clone template origin")
14666 .into());
14667 };
14668 let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
14669 let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
14670 return Err(ExcelError::new(ExcelErrorKind::Ref)
14671 .with_message(
14672 "FormulaPlane split found a span with a missing binding set",
14673 )
14674 .into());
14675 };
14676 let clone = source.clone();
14680 let new_binding_set_id = authority.plane.insert_binding_set(clone);
14681 let Some(template) = authority.plane.templates.get(lower_template_id) else {
14682 return Err(ExcelError::new(ExcelErrorKind::Ref)
14683 .with_message("FormulaPlane split could not find shifted template")
14684 .into());
14685 };
14686 let (ast_id, origin_row, origin_col) =
14687 (template.ast_id, template.origin_row, template.origin_col);
14688 authority.plane.set_binding_template_anchor(
14689 new_binding_set_id,
14690 ast_id,
14691 origin_row,
14692 origin_col,
14693 );
14694 Some(new_binding_set_id)
14695 } else {
14696 None
14697 };
14698 let lower_read_summary_id = plan
14699 .lower_new_read_summary
14700 .map(|summary| authority.plane.insert_span_read_summary(summary));
14701 let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
14702 let lower_ref = authority.plane.insert_span_with_ast_relocation(
14703 crate::formula_plane::runtime::NewFormulaSpan {
14704 sheet_id: plan.sheet_id,
14705 template_id: lower_template_id,
14706 domain: plan.lower_new_domain,
14707 result_region: lower_result_region,
14708 intrinsic_mask_id: None,
14709 read_summary_id: lower_read_summary_id,
14710 binding_set_id: lower_binding_set_id,
14711 is_constant_result: plan.is_constant_result,
14712 },
14713 crate::formula_plane::runtime::SpanAstRelocation {
14714 ast_id: split_relocation.ast_id,
14715 anchor_row: plan.lower_new_origin_row,
14716 anchor_col: plan.lower_new_origin_col,
14717 },
14718 );
14719 if let Some(region) = lower_dirty_region {
14720 span_dirty_deltas.push((lower_ref, region));
14721 }
14722 if let Some(binding_set_id) = lower_binding_set_id {
14723 authority
14724 .plane
14725 .set_binding_span_ref(binding_set_id, lower_ref);
14726 if plan.lower_force_binding_residual_axes {
14727 authority.plane.force_binding_residual_axes(binding_set_id);
14731 }
14732 }
14733 let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
14737 if !name_keys.is_empty() {
14738 authority
14739 .plane
14740 .register_span_name_dependents(lower_ref, &name_keys);
14741 }
14742 for overlay_ref in plan.lower_overlay_refs {
14743 if !authority
14744 .plane
14745 .set_overlay_source_span(overlay_ref, Some(lower_ref))
14746 {
14747 return Err(ExcelError::new(ExcelErrorKind::Ref)
14748 .with_message(
14749 "FormulaPlane split could not re-point overlay provenance",
14750 )
14751 .into());
14752 }
14753 }
14754 let upper_read_summary_id = plan
14757 .upper_read_summary
14758 .map(|summary| authority.plane.insert_span_read_summary(summary));
14759 let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
14760 if !authority.plane.replace_span_geometry_with_ast_relocation(
14761 plan.span_ref,
14762 plan.template_id,
14763 split_relocation,
14764 plan.upper_domain,
14765 upper_result_region,
14766 upper_read_summary_id,
14767 ) {
14768 return Err(ExcelError::new(ExcelErrorKind::Ref)
14769 .with_message("FormulaPlane split could not update span geometry")
14770 .into());
14771 }
14772 if let Some(region) = upper_dirty_region {
14773 span_dirty_deltas.push((plan.span_ref, region));
14774 }
14775 }
14776 authority.rebuild_indexes();
14777 }
14778 for (span_ref, region) in span_dirty_deltas {
14779 self.graph.mark_formula_span_region_dirty(span_ref, region);
14780 }
14781
14782 let mut span_plans = Vec::new();
14783 for span_ref in demote_refs {
14784 let authority = self.graph.formula_authority();
14785 let Some(span) = authority.plane.spans.get(span_ref) else {
14786 continue;
14787 };
14788 let relocation = span.ast_relocation;
14789 let ast = self
14790 .graph
14791 .data_store()
14792 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14793 .ok_or_else(|| {
14794 ExcelError::new(ExcelErrorKind::Ref).with_message(
14795 "FormulaPlane demotion could not retrieve the span anchor AST",
14796 )
14797 })?;
14798 let placements = span
14799 .domain
14800 .iter()
14801 .map(|placement| (placement.row + 1, placement.col + 1))
14802 .collect();
14803 span_plans.push(SpanPlan {
14804 span_ref,
14805 sheet_id: span.sheet_id,
14806 ast,
14807 origin_row: relocation.anchor_row,
14808 origin_col: relocation.anchor_col,
14809 binding_set_id: span.binding_set_id,
14810 placements,
14811 });
14812 }
14813 if span_plans.is_empty() {
14814 return Ok(());
14815 }
14816
14817 let mut relocated = Vec::new();
14818 let mut placement_cells = Vec::new();
14819 for plan in &span_plans {
14820 for &(row, col) in &plan.placements {
14821 let row_delta = i64::from(row) - i64::from(plan.origin_row);
14822 let col_delta = i64::from(col) - i64::from(plan.origin_col);
14823 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14824 let authority = self.graph.formula_authority();
14825 if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14826 if binding_set.is_single_literal_binding() {
14827 plan.ast.clone()
14828 } else {
14829 let placement = crate::formula_plane::runtime::PlacementCoord::new(
14830 plan.sheet_id,
14831 row.saturating_sub(1),
14832 col.saturating_sub(1),
14833 );
14834 let binding =
14835 authority.plane.spans.get(plan.span_ref).and_then(|span| {
14836 binding_set
14837 .literal_bindings_for_placement(&span.domain, placement)
14838 });
14839 if let Some(binding) = binding {
14840 substitute_literal_slots_for_template_placement(
14841 &plan.ast,
14842 binding.as_ref(),
14843 )
14844 } else {
14845 plan.ast.clone()
14846 }
14847 }
14848 } else {
14849 plan.ast.clone()
14850 }
14851 } else {
14852 plan.ast.clone()
14853 };
14854 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14855 relocated.push((plan.sheet_id, row, col, ast));
14856 placement_cells.push((plan.sheet_id, row, col));
14857 }
14858 }
14859 let planned_by_sheet = {
14860 let mut pipeline = self.ingest_pipeline();
14861 let mut planned_by_sheet: BTreeMap<
14862 SheetId,
14863 Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14864 > = BTreeMap::new();
14865 for (formula_sheet_id, row, col, ast) in relocated {
14866 let placement =
14867 CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14868 let ingested =
14869 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14870 planned_by_sheet.entry(formula_sheet_id).or_default().push((
14871 row,
14872 col,
14873 ingested.ast_id,
14874 ingested.dep_plan,
14875 ));
14876 }
14877 planned_by_sheet
14878 };
14879 {
14880 let authority = self.graph.formula_authority_mut();
14881 for plan in &span_plans {
14882 authority
14883 .plane
14884 .remove_overlays_for_source_span(plan.span_ref);
14885 authority.plane.remove_span(plan.span_ref);
14886 }
14887 authority.rebuild_indexes();
14888 }
14889 if clear_computed_overlays {
14890 self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14897 }
14898 for (formula_sheet_id, planned) in planned_by_sheet {
14899 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14900 self.graph
14901 .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14902 }
14903 if !clear_computed_overlays {
14904 for (formula_sheet_id, row, col) in &placement_cells {
14905 let row0 = row.saturating_sub(1);
14906 let col0 = col.saturating_sub(1);
14907 if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14908 continue;
14909 }
14910 let cell =
14911 CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14912 if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14913 self.graph.set_dirty(vertex_id, false);
14914 }
14915 }
14916 }
14917 Ok(())
14918 }
14919
14920 pub(crate) fn prepare_formula_span_demotion(
14924 &mut self,
14925 span_refs: &[FormulaSpanRef],
14926 ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14927 #[cfg(test)]
14928 let fault = self
14929 .formula_span_demotion_fault_for_test
14930 .take()
14931 .unwrap_or_default();
14932 #[cfg(not(test))]
14933 let fault = FormulaSpanDemotionFault::None;
14934
14935 if fault == FormulaSpanDemotionFault::AstPreparation {
14936 return Err(FormulaSpanDemotionError::Injected(fault));
14937 }
14938
14939 fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14940 fn visit(
14941 ast: &ASTNode,
14942 binding: &[LiteralValue],
14943 next: &mut usize,
14944 in_array: bool,
14945 ) -> ASTNode {
14946 let node_type = match &ast.node_type {
14947 ASTNodeType::Literal(_) if !in_array => {
14948 let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14949 *next = next.saturating_add(1);
14950 ASTNodeType::Literal(value)
14951 }
14952 ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14953 ASTNodeType::Omitted => ASTNodeType::Omitted,
14954 ASTNodeType::Reference {
14955 original,
14956 reference,
14957 } => ASTNodeType::Reference {
14958 original: original.clone(),
14959 reference: reference.clone(),
14960 },
14961 ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14962 op: op.clone(),
14963 expr: Box::new(visit(expr, binding, next, in_array)),
14964 },
14965 ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14966 op: op.clone(),
14967 left: Box::new(visit(left, binding, next, in_array)),
14968 right: Box::new(visit(right, binding, next, in_array)),
14969 },
14970 ASTNodeType::Function { name, args } => ASTNodeType::Function {
14971 name: name.clone(),
14972 args: args
14973 .iter()
14974 .map(|arg| visit(arg, binding, next, in_array))
14975 .collect(),
14976 },
14977 ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14978 callee: Box::new(visit(callee, binding, next, in_array)),
14979 args: args
14980 .iter()
14981 .map(|arg| visit(arg, binding, next, in_array))
14982 .collect(),
14983 },
14984 ASTNodeType::Array(rows) => ASTNodeType::Array(
14985 rows.iter()
14986 .map(|row| {
14987 row.iter()
14988 .map(|cell| visit(cell, binding, next, true))
14989 .collect()
14990 })
14991 .collect(),
14992 ),
14993 };
14994 ASTNode::new(node_type, ast.source_token.clone())
14995 }
14996 let mut next = 0;
14997 visit(ast, binding, &mut next, false)
14998 }
14999
15000 struct SpanMaterialization {
15001 span_ref: FormulaSpanRef,
15002 sheet_id: SheetId,
15003 ast: ASTNode,
15004 origin_row: u32,
15005 origin_col: u32,
15006 binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
15007 domain: PlacementDomain,
15008 }
15009
15010 let request_id = self
15011 .active_evaluation_resource_request
15012 .as_ref()
15013 .map(|request| request.request_id);
15014 let resource_error =
15015 |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
15016 FormulaSpanDemotionError::Resource(
15017 crate::engine::ResourceLedgerError::Exhausted(
15018 formualizer_common::ResourceExhaustionDetail {
15019 reason,
15020 limit,
15021 observed,
15022 request_id,
15023 },
15024 )
15025 .into_excel_error(),
15026 )
15027 };
15028 let authority = self.graph.formula_authority();
15029 let expected_plane_epoch = authority.plane.epoch().0;
15030 let expected_indexes_epoch = authority.indexes_epoch();
15031 let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
15032 let mut unique = Vec::new();
15033 unique.try_reserve_exact(span_refs.len()).map_err(|_| {
15034 resource_error(
15035 formualizer_common::ResourceExhaustionReason::ScratchMemory,
15036 u64::MAX,
15037 span_refs.len() as u64,
15038 )
15039 })?;
15040 let mut plans = Vec::new();
15041 plans.try_reserve_exact(span_refs.len()).map_err(|_| {
15042 resource_error(
15043 formualizer_common::ResourceExhaustionReason::ScratchMemory,
15044 u64::MAX,
15045 span_refs.len() as u64,
15046 )
15047 })?;
15048 let mut placement_count = 0usize;
15049 let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
15050 for &span_ref in span_refs {
15051 if unique.contains(&span_ref) {
15052 continue;
15053 }
15054 unique.push(span_ref);
15055 let span = authority
15056 .plane
15057 .spans
15058 .get(span_ref)
15059 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15060 let relocation = span.ast_relocation;
15061 let ast = self
15062 .graph
15063 .data_store()
15064 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
15065 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15066 let count = usize::try_from(span.domain.cell_count())
15067 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
15068 placement_count = placement_count
15069 .checked_add(count)
15070 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
15071 if u64::try_from(placement_count)
15072 .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
15073 > self.workbook_load_limits.max_formula_plane_fallback_cells
15074 {
15075 return Err(resource_error(
15076 formualizer_common::ResourceExhaustionReason::MaterializationCells,
15077 self.workbook_load_limits.max_formula_plane_fallback_cells,
15078 placement_count as u64,
15079 ));
15080 }
15081 let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
15082 *sheet_count = sheet_count
15083 .checked_add(span.domain.cell_count())
15084 .ok_or(FormulaSpanDemotionError::CountOverflow)?;
15085 if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
15086 return Err(resource_error(
15087 formualizer_common::ResourceExhaustionReason::Admission,
15088 self.workbook_load_limits.max_sheet_logical_cells,
15089 *sheet_count,
15090 ));
15091 }
15092 let domain_in_bounds = match &span.domain {
15093 PlacementDomain::RowRun { row_end, col, .. } => {
15094 *row_end < self.workbook_load_limits.max_sheet_rows
15095 && *col < self.workbook_load_limits.max_sheet_cols
15096 }
15097 PlacementDomain::ColRun { row, col_end, .. } => {
15098 *row < self.workbook_load_limits.max_sheet_rows
15099 && *col_end < self.workbook_load_limits.max_sheet_cols
15100 }
15101 PlacementDomain::Rect {
15102 row_end, col_end, ..
15103 } => {
15104 *row_end < self.workbook_load_limits.max_sheet_rows
15105 && *col_end < self.workbook_load_limits.max_sheet_cols
15106 }
15107 };
15108 if !domain_in_bounds {
15109 return Err(resource_error(
15110 formualizer_common::ResourceExhaustionReason::Admission,
15111 u64::from(self.workbook_load_limits.max_sheet_rows)
15112 .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
15113 span.domain.cell_count(),
15114 ));
15115 }
15116 plans.push(SpanMaterialization {
15117 span_ref,
15118 sheet_id: span.sheet_id,
15119 ast,
15120 origin_row: relocation.anchor_row,
15121 origin_col: relocation.anchor_col,
15122 binding_set_id: span.binding_set_id,
15123 domain: span.domain.clone(),
15124 });
15125 }
15126 let mut relocated = Vec::new();
15127 relocated.try_reserve_exact(placement_count).map_err(|_| {
15128 resource_error(
15129 formualizer_common::ResourceExhaustionReason::ScratchMemory,
15130 u64::MAX,
15131 placement_count as u64,
15132 )
15133 })?;
15134 for plan in &plans {
15135 for coord in plan.domain.iter() {
15136 let row = coord.row.saturating_add(1);
15137 let col = coord.col.saturating_add(1);
15138 let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
15139 let authority = self.graph.formula_authority();
15140 let binding_set = authority
15141 .plane
15142 .binding_sets
15143 .get(binding_set_id)
15144 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15145 if binding_set.is_single_literal_binding() {
15146 plan.ast.clone()
15147 } else {
15148 let placement = PlacementCoord::new(
15149 plan.sheet_id,
15150 row.saturating_sub(1),
15151 col.saturating_sub(1),
15152 );
15153 let span = authority
15154 .plane
15155 .spans
15156 .get(plan.span_ref)
15157 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15158 let binding = binding_set
15159 .literal_bindings_for_placement(&span.domain, placement)
15160 .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15161 substitute_literal_slots(&plan.ast, binding.as_ref())
15162 }
15163 } else {
15164 plan.ast.clone()
15165 };
15166 let row_delta = i64::from(row) - i64::from(plan.origin_row);
15167 let col_delta = i64::from(col) - i64::from(plan.origin_col);
15168 let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
15169 .map_err(FormulaSpanDemotionError::AstPreparation)?;
15170 relocated.push((plan.sheet_id, row, col, ast));
15171 }
15172 }
15173
15174 let mut analyzed = Vec::new();
15175 analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
15176 resource_error(
15177 formualizer_common::ResourceExhaustionReason::ScratchMemory,
15178 u64::MAX,
15179 relocated.len() as u64,
15180 )
15181 })?;
15182 {
15183 let mut pipeline = self.ingest_pipeline();
15184 for (sheet_id, row, col, ast) in relocated {
15185 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
15186 let ingested = pipeline
15187 .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
15188 .map_err(FormulaSpanDemotionError::AstPreparation)?;
15189 analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
15190 }
15191 }
15192
15193 if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
15194 return Err(FormulaSpanDemotionError::Injected(fault));
15195 }
15196 let legacy_graph = self
15197 .graph
15198 .prepare_legacy_graph_plan_multi_sheet(analyzed)
15199 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
15200 self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
15201 .map_err(FormulaSpanDemotionError::Resource)?;
15202
15203 Ok(PreparedFormulaSpanDemotion {
15204 span_refs: unique,
15205 expected_plane_epoch,
15206 expected_indexes_epoch,
15207 expected_indexed_plane_epoch,
15208 placement_count,
15209 legacy_graph,
15210 fault,
15211 })
15212 }
15213
15214 pub(crate) fn validate_prepared_formula_span_demotion(
15215 &self,
15216 prepared: &PreparedFormulaSpanDemotion,
15217 ) -> Result<(), FormulaSpanDemotionError> {
15218 let authority = self.graph.formula_authority();
15219 if authority.plane.epoch().0 != prepared.expected_plane_epoch
15220 || authority.indexes_epoch() != prepared.expected_indexes_epoch
15221 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
15222 || prepared
15223 .span_refs
15224 .iter()
15225 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
15226 {
15227 return Err(FormulaSpanDemotionError::StaleAuthority);
15228 }
15229 self.graph
15230 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
15231 .map_err(FormulaSpanDemotionError::LegacyGraph)
15232 }
15233
15234 pub(crate) fn commit_prepared_formula_span_demotion(
15240 &mut self,
15241 prepared: PreparedFormulaSpanDemotion,
15242 ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
15243 self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
15244 .map_err(FormulaSpanDemotionError::Resource)?;
15245 if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
15246 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15247 }
15248 self.graph
15249 .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
15250 .map_err(FormulaSpanDemotionError::LegacyGraph)?;
15251 if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
15252 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15253 }
15254 {
15255 let authority = self.graph.formula_authority();
15256 if authority.plane.epoch().0 != prepared.expected_plane_epoch
15257 || authority.indexes_epoch() != prepared.expected_indexes_epoch
15258 || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
15259 || prepared
15260 .span_refs
15261 .iter()
15262 .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
15263 {
15264 return Err(FormulaSpanDemotionError::StaleAuthority);
15265 }
15266 }
15267 if matches!(
15268 prepared.fault,
15269 FormulaSpanDemotionFault::AllocationReservation
15270 | FormulaSpanDemotionFault::BeforeFirstMutation
15271 ) {
15272 return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15273 }
15274
15275 let PreparedFormulaSpanDemotion {
15276 span_refs,
15277 placement_count,
15278 legacy_graph,
15279 ..
15280 } = prepared;
15281 let _ = self
15282 .graph
15283 .apply_prevalidated_legacy_graph_plan(legacy_graph);
15284 let authority = self.graph.formula_authority_mut();
15285 for span_ref in &span_refs {
15286 authority.plane.remove_overlays_for_source_span(*span_ref);
15287 let _ = authority.plane.remove_span(*span_ref);
15288 }
15289 let _ = authority.rebuild_indexes();
15290 self.mark_topology_edited();
15291 Ok(FormulaSpanDemotionReport {
15292 spans_demoted: span_refs.len(),
15293 placements_materialized: placement_count,
15294 })
15295 }
15296
15297 fn transition_off_mode_spans_to_legacy(&mut self) -> Result<bool, ExcelError> {
15298 if self.config.formula_plane_mode != FormulaPlaneMode::Off {
15299 return Ok(false);
15300 }
15301 let span_refs = self.graph.formula_authority().active_span_refs();
15302 if span_refs.is_empty() {
15303 return Ok(false);
15304 }
15305
15306 let formula_dirty = self.graph.lease_formula_dirty();
15307 let materialization_started = crate::instant::FzInstant::now();
15308 let map_error = |phase: &str, error: FormulaSpanDemotionError| match error {
15309 FormulaSpanDemotionError::Resource(error)
15310 | FormulaSpanDemotionError::AstPreparation(error) => error,
15311 error => ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15312 "FormulaPlane Off-mode span demotion {phase} failed: {error}"
15313 )),
15314 };
15315 let prepared = self
15316 .prepare_formula_span_demotion(&span_refs)
15317 .map_err(|error| map_error("preparation", error))?;
15318 let commit_started = self.preflight_evaluation_commit_window(prepared.placement_count)?;
15319 let report = self
15320 .commit_prepared_formula_span_demotion(prepared)
15321 .map_err(|error| map_error("commit", error))?;
15322 self.ack_formula_dirty_observed(formula_dirty);
15323 self.observe_evaluation_commit_window(commit_started);
15324 self.observe_materialization(
15325 report.placements_materialized,
15326 false,
15327 materialization_started.elapsed(),
15328 );
15329 Ok(true)
15330 }
15331
15332 #[cfg(test)]
15333 pub(crate) fn set_formula_span_demotion_fault_for_test(
15334 &mut self,
15335 fault: FormulaSpanDemotionFault,
15336 ) {
15337 self.formula_span_demotion_fault_for_test = Some(fault);
15338 }
15339
15340 #[cfg(test)]
15341 pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
15342 self.force_non_cycle_schedule_fallback_for_test = true;
15343 }
15344
15345 #[cfg(test)]
15346 pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
15347 self.mixed_topology_index_builds_for_test
15348 }
15349
15350 fn collect_cyclic_span_refs(
15355 &self,
15356 schedule: &MixedSchedule,
15357 span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
15358 ) -> Vec<FormulaSpanRef> {
15359 let mut refs = Vec::new();
15360 for fallback in &schedule.fallbacks {
15361 if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
15362 continue;
15363 }
15364 if let FormulaProducerId::Span(span_id) = fallback.producer
15365 && let Some(span_ref) = span_refs_by_id.get(&span_id)
15366 && !refs.contains(span_ref)
15367 {
15368 refs.push(*span_ref);
15369 }
15370 }
15371 refs
15372 }
15373
15374 fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
15382 let materialization_started = crate::instant::FzInstant::now();
15383 let prepared = self
15384 .prepare_formula_span_demotion(span_refs)
15385 .map_err(|error| {
15386 if let FormulaSpanDemotionError::Resource(error) = error {
15387 error
15388 } else {
15389 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15390 "FormulaPlane cycle-member span demotion preparation failed: {error}"
15391 ))
15392 }
15393 })?;
15394 let report = self
15395 .commit_prepared_formula_span_demotion(prepared)
15396 .map_err(|error| {
15397 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15398 "FormulaPlane cycle-member span demotion commit failed: {error}"
15399 ))
15400 })?;
15401 self.observe_materialization(
15402 report.placements_materialized,
15403 true,
15404 materialization_started.elapsed(),
15405 );
15406 self.formula_plane_cycle_member_span_demotions = self
15407 .formula_plane_cycle_member_span_demotions
15408 .saturating_add(report.spans_demoted as u64);
15409 Self::record_shadow_fallback_reason(
15410 &mut self.formula_ingest_report_total,
15411 PlacementFallbackReason::CycleMember,
15412 report.spans_demoted as u64,
15413 );
15414 Ok(())
15415 }
15416
15417 fn demote_array_result_span(&mut self, span_ref: FormulaSpanRef) -> Result<(), ExcelError> {
15430 let materialization_started = crate::instant::FzInstant::now();
15431 let prepared = self
15432 .prepare_formula_span_demotion(&[span_ref])
15433 .map_err(|error| {
15434 if let FormulaSpanDemotionError::Resource(error) = error {
15435 error
15436 } else {
15437 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15438 "FormulaPlane array-result span demotion preparation failed: {error}"
15439 ))
15440 }
15441 })?;
15442 let report = self
15443 .commit_prepared_formula_span_demotion(prepared)
15444 .map_err(|error| {
15445 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15446 "FormulaPlane array-result span demotion commit failed: {error}"
15447 ))
15448 })?;
15449 self.observe_materialization(
15450 report.placements_materialized,
15451 true,
15452 materialization_started.elapsed(),
15453 );
15454 self.formula_plane_array_result_span_demotions = self
15455 .formula_plane_array_result_span_demotions
15456 .saturating_add(report.spans_demoted as u64);
15457 Self::record_shadow_fallback_reason(
15458 &mut self.formula_ingest_report_total,
15459 PlacementFallbackReason::ArrayResult,
15460 report.spans_demoted as u64,
15461 );
15462 Ok(())
15463 }
15464
15465 fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
15480 let dirty = self.graph.get_evaluation_vertices();
15481 if dirty.is_empty() {
15482 return Ok(0);
15483 }
15484 let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
15485 let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
15486 let mut cycle_errors = 0usize;
15487 let mut stamped_vertices: Vec<VertexId> = Vec::new();
15488 for &unit in &schedule.units {
15489 let ScheduleUnit::Cycle(i) = unit else {
15490 continue;
15491 };
15492 let members = schedule.unit_cycle(i);
15493 let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
15494 if stamped > 0 {
15495 cycle_errors += 1;
15496 }
15497 stamped_vertices.extend(members.iter().copied());
15498 }
15499 if !stamped_vertices.is_empty() {
15504 self.resource_checkpoint(0)?;
15505 self.graph.clear_dirty_flags(&stamped_vertices);
15506 }
15507 Ok(cycle_errors)
15508 }
15509
15510 fn materialize_deferred_sheet_before_structural_edit(
15511 &mut self,
15512 sheet: &str,
15513 ) -> Result<(), crate::engine::EditorError> {
15514 if self.staged_formulas.contains_key(sheet) {
15515 self.build_graph_for_sheets([sheet])?;
15516 }
15517 Ok(())
15518 }
15519
15520 fn structural_row_occupancy(
15521 &self,
15522 sheet: &str,
15523 sheet_id: SheetId,
15524 ) -> crate::engine::graph::StructuralOccupancy {
15525 if !self.graph.has_compressed_range_dependencies() {
15526 return crate::engine::graph::StructuralOccupancy::default();
15527 }
15528 let mut occupancy = self.graph.structural_occupancy(sheet_id);
15529 if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
15530 occupancy.include_arrow_sheet(arrow_sheet);
15531 occupancy
15532 } else {
15533 crate::engine::graph::StructuralOccupancy::conservative()
15535 }
15536 }
15537
15538 fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
15539 crate::engine::graph::StructuralOccupancy::conservative()
15543 }
15544
15545 pub fn insert_rows(
15547 &mut self,
15548 sheet: &str,
15549 before: u32,
15550 count: u32,
15551 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15552 {
15553 if count == 0 {
15554 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15555 }
15556 self.observe_function_semantic_epoch()
15557 .map_err(crate::engine::EditorError::Excel)?;
15558 use crate::engine::graph::editor::vertex_editor::VertexEditor;
15559 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15560 let sheet_id = self.ensure_known_sheet_id(sheet)?;
15561 let before0 = before.saturating_sub(1);
15562 let affected_region = Self::structural_row_region(sheet_id, before0);
15563 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
15564 let op = StructuralOp::InsertRows {
15565 sheet_id,
15566 before: before0,
15567 count,
15568 };
15569 self.demote_spans_for_structural_op(op, affected_region)?;
15570 let summary = {
15571 let mut editor =
15572 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15573 editor.insert_rows(sheet_id, before0, count)?
15574 };
15575 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15576 let before0 = before0 as usize;
15577 asheet.insert_rows(before0, count as usize);
15578 }
15579 self.purge_derived_formats_after_row(sheet_id, before0);
15580 self.mark_moved_formula_vertices_dirty(&summary);
15581 self.clear_computed_overlay_after_row(sheet, before0 as usize);
15582 self.shift_row_visibility_insert(sheet_id, before0, count);
15583 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15584 self.mark_topology_edited();
15585 Ok(summary)
15586 }
15587
15588 pub fn delete_rows(
15590 &mut self,
15591 sheet: &str,
15592 start: u32,
15593 count: u32,
15594 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15595 {
15596 if count == 0 {
15597 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15598 }
15599 self.observe_function_semantic_epoch()
15600 .map_err(crate::engine::EditorError::Excel)?;
15601 use crate::engine::graph::editor::vertex_editor::VertexEditor;
15602 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15603 let sheet_id = self.ensure_known_sheet_id(sheet)?;
15604 let start0 = start.saturating_sub(1);
15605 let affected_region = Self::structural_row_region(sheet_id, start0);
15606 let occupancy = self.structural_row_occupancy(sheet, sheet_id);
15607 let op = StructuralOp::DeleteRows {
15608 sheet_id,
15609 start: start0,
15610 count,
15611 };
15612 self.demote_spans_for_structural_op(op, affected_region)?;
15613 let summary = {
15614 let mut editor =
15615 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15616 editor.delete_rows(sheet_id, start0, count)?
15617 };
15618 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15619 let start0 = start0 as usize;
15620 asheet.delete_rows(start0, count as usize);
15621 }
15622 self.purge_derived_formats_after_row(sheet_id, start0);
15623 self.mark_moved_formula_vertices_dirty(&summary);
15624 self.clear_computed_overlay_after_row(sheet, start0 as usize);
15625 self.shift_row_visibility_delete(sheet_id, start0, count);
15626 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15627 self.mark_topology_edited();
15628 Ok(summary)
15629 }
15630
15631 pub fn insert_columns(
15633 &mut self,
15634 sheet: &str,
15635 before: u32,
15636 count: u32,
15637 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15638 {
15639 if count == 0 {
15640 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15641 }
15642 self.observe_function_semantic_epoch()
15643 .map_err(crate::engine::EditorError::Excel)?;
15644 use crate::engine::graph::editor::vertex_editor::VertexEditor;
15645 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15646 let sheet_id = self.graph.sheet_id(sheet).ok_or(
15647 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
15648 name: sheet.to_string(),
15649 reason: "Unknown sheet".to_string(),
15650 },
15651 )?;
15652 let before0 = before.saturating_sub(1);
15653 let affected_region = Self::structural_col_region(sheet_id, before0);
15654 let occupancy = self.structural_column_occupancy();
15655 let op = StructuralOp::InsertColumns {
15656 sheet_id,
15657 before: before0,
15658 count,
15659 };
15660 self.demote_spans_for_structural_op(op, affected_region)?;
15661 let summary = {
15662 let mut editor =
15663 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15664 editor.insert_columns(sheet_id, before0, count)?
15665 };
15666 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15667 let before0 = before0 as usize;
15668 asheet.insert_columns(before0, count as usize);
15669 }
15670 self.purge_derived_formats_after_col(sheet_id, before0);
15671 self.mark_moved_formula_vertices_dirty(&summary);
15672 self.clear_computed_overlay_after_col(sheet, before0 as usize);
15673 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15674 self.mark_topology_edited();
15675 Ok(summary)
15676 }
15677
15678 pub fn delete_columns(
15680 &mut self,
15681 sheet: &str,
15682 start: u32,
15683 count: u32,
15684 ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15685 {
15686 if count == 0 {
15687 return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15688 }
15689 self.observe_function_semantic_epoch()
15690 .map_err(crate::engine::EditorError::Excel)?;
15691 use crate::engine::graph::editor::vertex_editor::VertexEditor;
15692 self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15693 let sheet_id = self.graph.sheet_id(sheet).ok_or(
15694 crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
15695 name: sheet.to_string(),
15696 reason: "Unknown sheet".to_string(),
15697 },
15698 )?;
15699 let start0 = start.saturating_sub(1);
15700 let affected_region = Self::structural_col_region(sheet_id, start0);
15701 let occupancy = self.structural_column_occupancy();
15702 let op = StructuralOp::DeleteColumns {
15703 sheet_id,
15704 start: start0,
15705 count,
15706 };
15707 self.demote_spans_for_structural_op(op, affected_region)?;
15708 let summary = {
15709 let mut editor =
15710 VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15711 editor.delete_columns(sheet_id, start0, count)?
15712 };
15713 if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15714 let start0 = start0 as usize;
15715 asheet.delete_columns(start0, count as usize);
15716 }
15717 self.purge_derived_formats_after_col(sheet_id, start0);
15718 self.mark_moved_formula_vertices_dirty(&summary);
15719 self.clear_computed_overlay_after_col(sheet, start0 as usize);
15720 self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15721 self.mark_topology_edited();
15722 Ok(summary)
15723 }
15724 fn arrow_used_row_bounds(
15726 &self,
15727 sheet: &str,
15728 start_col: u32,
15729 end_col: u32,
15730 ) -> Option<(u32, u32)> {
15731 let a = self.sheet_store().sheet(sheet)?;
15732 if a.columns.is_empty() {
15733 return None;
15734 }
15735 let sc0 = start_col.saturating_sub(1) as usize;
15736 let ec0 = end_col.saturating_sub(1) as usize;
15737 let col_hi = a.columns.len().saturating_sub(1);
15738 if sc0 > col_hi {
15739 return None;
15740 }
15741 let ec0 = ec0.min(col_hi);
15742 let snap = self.data_snapshot_id();
15744 let mut min_r0: Option<usize> = None;
15745 for ci in sc0..=ec0 {
15746 let sheet_id = self.graph.sheet_id(sheet)?;
15747 if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
15748 g.as_ref()
15749 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
15750 }) {
15751 let mv = mv as usize;
15752 min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
15753 continue;
15754 }
15755 let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
15757 if let Ok(mut g) = self.row_bounds_cache.write() {
15758 g.get_or_insert_with(|| RowBoundsCache::new(snap))
15759 .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
15760 }
15761 if let Some(m) = min_c {
15762 min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
15763 }
15764 }
15765 min_r0?;
15766 let mut max_r0: Option<usize> = None;
15767 for ci in sc0..=ec0 {
15768 let sheet_id = self.graph.sheet_id(sheet)?;
15769 if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
15770 g.as_ref()
15771 .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
15772 }) {
15773 let mv = mv as usize;
15774 max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
15775 continue;
15776 }
15777 let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
15778 if let Ok(mut g) = self.row_bounds_cache.write() {
15779 g.get_or_insert_with(|| RowBoundsCache::new(snap))
15780 .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
15781 }
15782 if let Some(m) = max_c {
15783 max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
15784 }
15785 }
15786 match (min_r0, max_r0) {
15787 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15788 _ => None,
15789 }
15790 }
15791
15792 fn scan_column_used_bounds(
15793 a: &crate::arrow_store::ArrowSheet,
15794 ci: usize,
15795 ) -> (Option<u32>, Option<u32>) {
15796 let col = &a.columns[ci];
15797
15798 let mut min_r0: Option<u32> = None;
15800 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15801 let tags = chunk.type_tag.values();
15802 for (off, &t) in tags.iter().enumerate() {
15803 let overlay_non_empty = chunk
15804 .overlay
15805 .get(off)
15806 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15807 .unwrap_or(false)
15808 || chunk
15809 .computed_overlay
15810 .get(off)
15811 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15812 .unwrap_or(false);
15813 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15814 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15815 break;
15816 };
15817 let row0 = chunk_start + off;
15818 min_r0 = Some(row0 as u32);
15819 break;
15820 }
15821 }
15822 if min_r0.is_some() {
15823 break;
15824 }
15825 }
15826 if min_r0.is_none() && !col.sparse_chunks.is_empty() {
15827 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
15828 sparse_idxs.sort_unstable();
15829 for chunk_idx in sparse_idxs {
15830 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
15831 continue;
15832 };
15833 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15834 continue;
15835 };
15836 let tags = chunk.type_tag.values();
15837 for (off, &t) in tags.iter().enumerate() {
15838 let overlay_non_empty = chunk
15839 .overlay
15840 .get(off)
15841 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15842 .unwrap_or(false)
15843 || chunk
15844 .computed_overlay
15845 .get(off)
15846 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15847 .unwrap_or(false);
15848 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15849 let row0 = chunk_start + off;
15850 min_r0 = Some(row0 as u32);
15851 break;
15852 }
15853 }
15854 if min_r0.is_some() {
15855 break;
15856 }
15857 }
15858 }
15859
15860 let mut max_r0: Option<u32> = None;
15862 if !col.sparse_chunks.is_empty() {
15863 let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
15864 sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
15865 for chunk_idx in sparse_idxs {
15866 let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
15867 continue;
15868 };
15869 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15870 continue;
15871 };
15872 let tags = chunk.type_tag.values();
15873 for (rev_idx, &t) in tags.iter().enumerate().rev() {
15874 let overlay_non_empty = chunk
15875 .overlay
15876 .get(rev_idx)
15877 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15878 .unwrap_or(false)
15879 || chunk
15880 .computed_overlay
15881 .get(rev_idx)
15882 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15883 .unwrap_or(false);
15884 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15885 let row0 = chunk_start + rev_idx;
15886 max_r0 = Some(row0 as u32);
15887 break;
15888 }
15889 }
15890 if max_r0.is_some() {
15891 break;
15892 }
15893 }
15894 }
15895 if max_r0.is_none() {
15896 for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
15897 let tags = chunk.type_tag.values();
15898 for (rev_idx, &t) in tags.iter().enumerate().rev() {
15899 let overlay_non_empty = chunk
15900 .overlay
15901 .get(rev_idx)
15902 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15903 .unwrap_or(false)
15904 || chunk
15905 .computed_overlay
15906 .get(rev_idx)
15907 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15908 .unwrap_or(false);
15909 if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15910 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15911 break;
15912 };
15913 let row0 = chunk_start + rev_idx;
15914 max_r0 = Some(row0 as u32);
15915 break;
15916 }
15917 }
15918 if max_r0.is_some() {
15919 break;
15920 }
15921 }
15922 }
15923
15924 (min_r0, max_r0)
15925 }
15926
15927 fn arrow_used_col_bounds(
15929 &self,
15930 sheet: &str,
15931 start_row: u32,
15932 end_row: u32,
15933 ) -> Option<(u32, u32)> {
15934 let a = self.sheet_store().sheet(sheet)?;
15935 if a.columns.is_empty() {
15936 return None;
15937 }
15938 let sr0 = start_row.saturating_sub(1) as usize;
15939 let er0 = end_row.saturating_sub(1) as usize;
15940 if sr0 > er0 {
15941 return None;
15942 }
15943 let mut min_c0: Option<usize> = None;
15946 let mut max_c0: Option<usize> = None;
15947 for (ci, col) in a.columns.iter().enumerate() {
15949 let mut any_in_range = false;
15950
15951 let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15952 let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15953 return false;
15954 };
15955 let chunk_len = chunk.type_tag.len();
15956 if chunk_len == 0 {
15957 return false;
15958 }
15959 let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15960 if sr0 > chunk_end || er0 < chunk_start {
15962 return false;
15963 }
15964 let start_off = sr0.max(chunk_start) - chunk_start;
15965 let end_off = er0.min(chunk_end) - chunk_start;
15966 let tags = chunk.type_tag.values();
15967 for off in start_off..=end_off {
15968 let overlay_non_empty = chunk
15969 .overlay
15970 .get(off)
15971 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15972 .unwrap_or(false)
15973 || chunk
15974 .computed_overlay
15975 .get(off)
15976 .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15977 .unwrap_or(false);
15978 if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15979 return true;
15980 }
15981 }
15982 false
15983 };
15984
15985 for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15986 if scan_chunk(chunk_idx, chunk) {
15987 any_in_range = true;
15988 break;
15989 }
15990 }
15991
15992 if !any_in_range && !col.sparse_chunks.is_empty() {
15993 for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15994 if scan_chunk(chunk_idx, chunk) {
15995 any_in_range = true;
15996 break;
15997 }
15998 }
15999 }
16000
16001 if any_in_range {
16002 min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
16003 max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
16004 }
16005 }
16006 match (min_c0, max_c0) {
16007 (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
16008 _ => None,
16009 }
16010 }
16011
16012 fn formula_row_bounds_for_columns(
16013 &self,
16014 sheet: &str,
16015 start_col: u32,
16016 end_col: u32,
16017 ) -> Option<(u32, u32)> {
16018 let sheet_id = self.graph.sheet_id(sheet)?;
16019 let sc0 = start_col.saturating_sub(1);
16020 let ec0 = end_col.saturating_sub(1);
16021 let mut min_r0: Option<u32> = None;
16022 let mut max_r0: Option<u32> = None;
16023
16024 if let Some(index) = self.graph.sheet_index(sheet_id) {
16025 for vid in index.vertices_in_col_range(sc0, ec0) {
16026 if !matches!(
16027 self.graph.get_vertex_kind(vid),
16028 VertexKind::FormulaScalar | VertexKind::FormulaArray
16029 ) {
16030 continue;
16031 }
16032 let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
16033 continue;
16034 };
16035 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
16036 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
16037 }
16038 } else {
16039 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
16040 if !matches!(
16041 self.graph.get_vertex_kind(vid),
16042 VertexKind::FormulaScalar | VertexKind::FormulaArray
16043 ) {
16044 continue;
16045 }
16046 let col0 = coord.col();
16047 if col0 < sc0 || col0 > ec0 {
16048 continue;
16049 }
16050 let row0 = coord.row();
16051 min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
16052 max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
16053 }
16054 }
16055
16056 match (min_r0, max_r0) {
16057 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
16058 _ => None,
16059 }
16060 }
16061
16062 fn formula_col_bounds_for_rows(
16063 &self,
16064 sheet: &str,
16065 start_row: u32,
16066 end_row: u32,
16067 ) -> Option<(u32, u32)> {
16068 let sheet_id = self.graph.sheet_id(sheet)?;
16069 let sr0 = start_row.saturating_sub(1);
16070 let er0 = end_row.saturating_sub(1);
16071 let mut min_c0: Option<u32> = None;
16072 let mut max_c0: Option<u32> = None;
16073
16074 if let Some(index) = self.graph.sheet_index(sheet_id) {
16075 for vid in index.vertices_in_row_range(sr0, er0) {
16076 if !matches!(
16077 self.graph.get_vertex_kind(vid),
16078 VertexKind::FormulaScalar | VertexKind::FormulaArray
16079 ) {
16080 continue;
16081 }
16082 let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
16083 continue;
16084 };
16085 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
16086 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
16087 }
16088 } else {
16089 for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
16090 if !matches!(
16091 self.graph.get_vertex_kind(vid),
16092 VertexKind::FormulaScalar | VertexKind::FormulaArray
16093 ) {
16094 continue;
16095 }
16096 let row0 = coord.row();
16097 if row0 < sr0 || row0 > er0 {
16098 continue;
16099 }
16100 let col0 = coord.col();
16101 min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
16102 max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
16103 }
16104 }
16105
16106 match (min_c0, max_c0) {
16107 (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
16108 _ => None,
16109 }
16110 }
16111
16112 fn union_used_bounds(
16113 first: Option<(u32, u32)>,
16114 second: Option<(u32, u32)>,
16115 ) -> Option<(u32, u32)> {
16116 match (first, second) {
16117 (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
16118 (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
16119 (None, None) => None,
16120 }
16121 }
16122
16123 fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
16126 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16127 return;
16128 }
16129 if self.arrow_sheets.sheet(sheet).is_none() {
16130 self.arrow_sheets
16131 .sheets
16132 .push(crate::arrow_store::ArrowSheet {
16133 name: std::sync::Arc::<str>::from(sheet),
16134 date_system: self.config.date_system,
16135 columns: Vec::new(),
16136 nrows: 0,
16137 chunk_starts: Vec::new(),
16138 chunk_rows: 32 * 1024,
16139 });
16140 }
16141
16142 let row0 = row.saturating_sub(1) as usize;
16143 let col0 = col.saturating_sub(1) as usize;
16144
16145 let asheet = self
16146 .arrow_sheets
16147 .sheet_mut(sheet)
16148 .expect("ArrowSheet must exist");
16149
16150 let cur_cols = asheet.columns.len();
16151 if col0 >= cur_cols {
16152 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16153 }
16154
16155 if row0 >= asheet.nrows as usize {
16156 if asheet.columns.is_empty() {
16157 asheet.insert_columns(0, 1);
16158 }
16159 asheet.ensure_row_capacity(row0 + 1);
16160 }
16161 if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
16162 let ov =
16163 crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
16164 let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
16165 let _ = ch.overlay.set(in_off, ov);
16166 let format = match value {
16167 LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
16168 LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
16169 LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
16170 LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
16171 _ => None,
16172 };
16173 ch.overlay.set_format(in_off, format);
16174 ch.computed_overlay.remove(in_off)
16179 } else {
16180 return;
16181 };
16182 let abs_threshold = 1024usize;
16184 let frac_den = 50usize;
16185 let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
16186 if freed > 0 {
16187 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16188 }
16189 self.adjust_computed_overlay_bytes(computed_delta);
16190 }
16191 }
16192
16193 fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
16198 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16199 return;
16200 }
16201 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16202 return;
16203 };
16204 let row0 = row.saturating_sub(1) as usize;
16205 let col0 = col.saturating_sub(1) as usize;
16206 if row0 >= asheet.nrows as usize {
16207 return;
16208 }
16209 if col0 >= asheet.columns.len() {
16210 return;
16211 }
16212 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16213 return;
16214 };
16215 if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
16216 let _ = ch.overlay.remove(in_off);
16217 }
16218 }
16219
16220 fn clear_computed_overlay_col_row_range(
16221 &mut self,
16222 sheet: &str,
16223 col0: usize,
16224 start_row0: usize,
16225 end_row0_exclusive: usize,
16226 ) {
16227 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16228 return;
16229 }
16230 if start_row0 >= end_row0_exclusive {
16231 return;
16232 }
16233
16234 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16235 return;
16236 };
16237 if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
16238 return;
16239 }
16240 let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
16241 if start_row0 >= end_row0_exclusive {
16242 return;
16243 }
16244
16245 let starts = asheet.chunk_starts.clone();
16246 let nrows = asheet.nrows as usize;
16247 let mut delta = 0isize;
16248 let Some(col) = asheet.columns.get_mut(col0) else {
16249 return;
16250 };
16251 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
16252 let Some(&chunk_start) = starts.get(chunk_idx) else {
16253 continue;
16254 };
16255 let chunk_end = starts
16256 .get(chunk_idx + 1)
16257 .copied()
16258 .unwrap_or(nrows)
16259 .min(chunk_start.saturating_add(ch.len()));
16260 let clear_start = start_row0.max(chunk_start);
16261 let clear_end = end_row0_exclusive.min(chunk_end);
16262 if clear_start >= clear_end {
16263 continue;
16264 }
16265 if clear_start == chunk_start && clear_end == chunk_end {
16266 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16267 } else {
16268 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
16269 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
16270 delta = delta.saturating_add(
16271 ch.computed_overlay
16272 .remove_range(start_in_chunk..end_in_chunk),
16273 );
16274 }
16275 }
16276 for (chunk_idx, ch) in &mut col.sparse_chunks {
16277 let Some(&chunk_start) = starts.get(*chunk_idx) else {
16278 continue;
16279 };
16280 let chunk_end = starts
16281 .get(*chunk_idx + 1)
16282 .copied()
16283 .unwrap_or(nrows)
16284 .min(chunk_start.saturating_add(ch.len()));
16285 let clear_start = start_row0.max(chunk_start);
16286 let clear_end = end_row0_exclusive.min(chunk_end);
16287 if clear_start >= clear_end {
16288 continue;
16289 }
16290 if clear_start == chunk_start && clear_end == chunk_end {
16291 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16292 } else {
16293 let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
16294 let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
16295 delta = delta.saturating_add(
16296 ch.computed_overlay
16297 .remove_range(start_in_chunk..end_in_chunk),
16298 );
16299 }
16300 }
16301 self.adjust_computed_overlay_bytes(delta);
16302 }
16303
16304 fn clear_computed_overlay_cells_in_region(
16305 &mut self,
16306 cells: &[(SheetId, u32, u32)],
16307 affected_region: &Region,
16308 ) {
16309 let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
16310 for (formula_sheet_id, row, col) in cells {
16311 let row0 = row.saturating_sub(1);
16312 let col0 = col.saturating_sub(1);
16313 let placement_region = Region::point(*formula_sheet_id, row0, col0);
16314 if placement_region.intersects(affected_region) {
16315 by_col
16316 .entry((*formula_sheet_id, col0))
16317 .or_default()
16318 .push(row0);
16319 }
16320 }
16321
16322 for ((formula_sheet_id, col0), mut rows) in by_col {
16323 rows.sort_unstable();
16324 rows.dedup();
16325 let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
16326 let mut start = rows[0];
16327 let mut prev = rows[0];
16328 for row in rows.into_iter().skip(1) {
16329 if row == prev.saturating_add(1) {
16330 prev = row;
16331 continue;
16332 }
16333 self.clear_computed_overlay_col_row_range(
16334 &sheet_name,
16335 col0 as usize,
16336 start as usize,
16337 prev.saturating_add(1) as usize,
16338 );
16339 start = row;
16340 prev = row;
16341 }
16342 self.clear_computed_overlay_col_row_range(
16343 &sheet_name,
16344 col0 as usize,
16345 start as usize,
16346 prev.saturating_add(1) as usize,
16347 );
16348 }
16349 }
16350
16351 fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
16352 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16353 return;
16354 }
16355
16356 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16357 return;
16358 };
16359 if start_row0 >= asheet.nrows as usize {
16360 return;
16361 }
16362
16363 let starts = asheet.chunk_starts.clone();
16364 let nrows = asheet.nrows as usize;
16365 let mut delta = 0isize;
16366 for col in &mut asheet.columns {
16367 for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
16368 let Some(&chunk_start) = starts.get(chunk_idx) else {
16369 continue;
16370 };
16371 let chunk_end = starts
16372 .get(chunk_idx + 1)
16373 .copied()
16374 .unwrap_or(nrows)
16375 .min(chunk_start.saturating_add(ch.len()));
16376 if chunk_end <= start_row0 {
16377 continue;
16378 }
16379 if chunk_start >= start_row0 {
16380 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16381 } else {
16382 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
16383 delta = delta
16384 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
16385 }
16386 }
16387
16388 for (chunk_idx, ch) in &mut col.sparse_chunks {
16389 let Some(&chunk_start) = starts.get(*chunk_idx) else {
16390 continue;
16391 };
16392 let chunk_end = starts
16393 .get(*chunk_idx + 1)
16394 .copied()
16395 .unwrap_or(nrows)
16396 .min(chunk_start.saturating_add(ch.len()));
16397 if chunk_end <= start_row0 {
16398 continue;
16399 }
16400 if chunk_start >= start_row0 {
16401 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16402 } else {
16403 let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
16404 delta = delta
16405 .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
16406 }
16407 }
16408 }
16409 self.adjust_computed_overlay_bytes(delta);
16410 }
16411
16412 fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
16413 if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16414 return;
16415 }
16416
16417 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16418 return;
16419 };
16420 if start_col0 >= asheet.columns.len() {
16421 return;
16422 }
16423
16424 let mut delta = 0isize;
16425 for col in asheet.columns.iter_mut().skip(start_col0) {
16426 for ch in &mut col.chunks {
16427 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16428 }
16429 for ch in col.sparse_chunks.values_mut() {
16430 delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16431 }
16432 }
16433 self.adjust_computed_overlay_bytes(delta);
16434 }
16435
16436 #[inline]
16437 fn literal_to_overlay_value(
16438 value: &LiteralValue,
16439 date_system: crate::engine::DateSystem,
16440 ) -> crate::arrow_store::OverlayValue {
16441 crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
16442 }
16443
16444 fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
16445 self.arrow_sheets
16446 .sheet(sheet)
16447 .map(|sheet| sheet.date_system)
16448 .unwrap_or(self.config.date_system)
16449 }
16450
16451 fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16454 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16455 return None;
16456 }
16457 let asheet = self.arrow_sheets.sheet(sheet)?;
16458 let row0 = row.saturating_sub(1) as usize;
16459 let col0 = col.saturating_sub(1) as usize;
16460 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
16461 return None;
16462 }
16463 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
16464 let ch = asheet.columns[col0].chunk(ch_idx)?;
16465 ch.overlay
16466 .get_scalar(in_off)
16467 .map(|ov| ov.to_literal_for(asheet.date_system))
16468 }
16469
16470 fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16473 if !(self.config.arrow_storage_enabled
16474 && self.config.delta_overlay_enabled
16475 && self.config.write_formula_overlay_enabled)
16476 {
16477 return None;
16478 }
16479 let asheet = self.arrow_sheets.sheet(sheet)?;
16480 let row0 = row.saturating_sub(1) as usize;
16481 let col0 = col.saturating_sub(1) as usize;
16482 if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
16483 return None;
16484 }
16485 let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
16486 let ch = asheet.columns[col0].chunk(ch_idx)?;
16487 ch.computed_overlay
16488 .get_scalar(in_off)
16489 .map(|ov| ov.to_literal_for(asheet.date_system))
16490 }
16491
16492 fn set_delta_overlay_cell_raw(
16493 &mut self,
16494 sheet: &str,
16495 row: u32,
16496 col: u32,
16497 value: Option<LiteralValue>,
16498 ) {
16499 if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16500 return;
16501 }
16502
16503 self.ensure_arrow_sheet(sheet);
16504 let date_system = self.arrow_sheet_date_system(sheet);
16505 let ov_opt = value
16506 .as_ref()
16507 .map(|value| Self::literal_to_overlay_value(value, date_system));
16508 let row0 = row.saturating_sub(1) as usize;
16509 let col0 = col.saturating_sub(1) as usize;
16510 let asheet = self
16511 .arrow_sheets
16512 .sheet_mut(sheet)
16513 .expect("ArrowSheet must exist");
16514
16515 let cur_cols = asheet.columns.len();
16516 if col0 >= cur_cols {
16517 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16518 }
16519 if row0 >= asheet.nrows as usize {
16520 if asheet.columns.is_empty() {
16521 asheet.insert_columns(0, 1);
16522 }
16523 asheet.ensure_row_capacity(row0 + 1);
16524 }
16525
16526 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16527 return;
16528 };
16529 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
16530 return;
16531 };
16532
16533 if let Some(ov) = ov_opt {
16534 let _ = ch.overlay.set(in_off, ov);
16535 } else {
16536 let _ = ch.overlay.remove(in_off);
16537 }
16538 }
16539
16540 fn set_computed_overlay_cell_raw(
16541 &mut self,
16542 sheet: &str,
16543 row: u32,
16544 col: u32,
16545 value: Option<LiteralValue>,
16546 ) {
16547 if !(self.config.arrow_storage_enabled
16548 && self.config.delta_overlay_enabled
16549 && self.config.write_formula_overlay_enabled)
16550 {
16551 return;
16552 }
16553
16554 self.ensure_arrow_sheet(sheet);
16555 let date_system = self.arrow_sheet_date_system(sheet);
16556 let ov_opt = value
16557 .as_ref()
16558 .map(|value| Self::literal_to_overlay_value(value, date_system));
16559 let row0 = row.saturating_sub(1) as usize;
16560 let col0 = col.saturating_sub(1) as usize;
16561 let asheet = self
16562 .arrow_sheets
16563 .sheet_mut(sheet)
16564 .expect("ArrowSheet must exist");
16565
16566 let cur_cols = asheet.columns.len();
16567 if col0 >= cur_cols {
16568 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16569 }
16570 if row0 >= asheet.nrows as usize {
16571 if asheet.columns.is_empty() {
16572 asheet.insert_columns(0, 1);
16573 }
16574 asheet.ensure_row_capacity(row0 + 1);
16575 }
16576
16577 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16578 return;
16579 };
16580 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
16581 return;
16582 };
16583
16584 let delta = if let Some(ov) = ov_opt {
16585 ch.computed_overlay.set(in_off, ov)
16586 } else {
16587 ch.computed_overlay.remove(in_off)
16588 };
16589 self.adjust_computed_overlay_bytes(delta);
16590 }
16591
16592 fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
16593 use crate::engine::ArrowOp;
16594
16595 let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
16596 Box::new(batch.ops.iter().rev())
16597 } else {
16598 Box::new(batch.ops.iter())
16599 };
16600
16601 for op in iter {
16602 match op {
16603 ArrowOp::SetDeltaCell {
16604 sheet_id,
16605 row0,
16606 col0,
16607 old,
16608 new,
16609 } => {
16610 let sheet = self.graph.sheet_name(*sheet_id).to_string();
16611 let v = if undo { old.clone() } else { new.clone() };
16612 self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
16613 }
16614 ArrowOp::SetComputedCell {
16615 sheet_id,
16616 row0,
16617 col0,
16618 old,
16619 new,
16620 } => {
16621 let sheet = self.graph.sheet_name(*sheet_id).to_string();
16622 let v = if undo { old.clone() } else { new.clone() };
16623 self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
16624 }
16625 ArrowOp::RestoreComputedRect {
16626 sheet_id,
16627 sr0,
16628 sc0,
16629 er0,
16630 ec0,
16631 old,
16632 new,
16633 } => {
16634 let sheet = self.graph.sheet_name(*sheet_id).to_string();
16635 let vals = if undo { old } else { new };
16636 let height = (*er0).saturating_sub(*sr0) as usize + 1;
16637 let width = (*ec0).saturating_sub(*sc0) as usize + 1;
16638 for r in 0..height {
16639 for c in 0..width {
16640 let v = vals
16641 .get(r)
16642 .and_then(|row| row.get(c))
16643 .cloned()
16644 .unwrap_or(LiteralValue::Empty);
16645 self.set_computed_overlay_cell_raw(
16646 &sheet,
16647 *sr0 + 1 + r as u32,
16648 *sc0 + 1 + c as u32,
16649 Some(v),
16650 );
16651 }
16652 }
16653 }
16654 ArrowOp::InsertRows {
16655 sheet_id,
16656 before0,
16657 count,
16658 } => {
16659 let sheet = self.graph.sheet_name(*sheet_id).to_string();
16660 self.ensure_arrow_sheet(&sheet);
16661 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
16662 if undo {
16663 asheet.delete_rows(*before0 as usize, *count as usize);
16664 } else {
16665 asheet.insert_rows(*before0 as usize, *count as usize);
16666 }
16667 }
16668 self.purge_derived_formats_after_row(*sheet_id, *before0);
16669 }
16670 ArrowOp::InsertCols {
16671 sheet_id,
16672 before0,
16673 count,
16674 } => {
16675 let sheet = self.graph.sheet_name(*sheet_id).to_string();
16676 self.ensure_arrow_sheet(&sheet);
16677 if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
16678 if undo {
16679 asheet.delete_columns(*before0 as usize, *count as usize);
16680 } else {
16681 asheet.insert_columns(*before0 as usize, *count as usize);
16682 }
16683 }
16684 self.purge_derived_formats_after_col(*sheet_id, *before0);
16685 }
16686 }
16687 }
16688 }
16689
16690 fn record_spill_ops_into_arrow_undo(
16691 &mut self,
16692 undo: &mut crate::engine::ArrowUndoBatch,
16693 events: &[crate::engine::ChangeEvent],
16694 ) {
16695 use crate::engine::ChangeEvent;
16696 use formualizer_common::LiteralValue;
16697
16698 #[allow(clippy::type_complexity)]
16699 let rect_from_snapshot =
16700 |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
16701 -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
16702 if snap.target_cells.is_empty() {
16703 return None;
16704 }
16705 let sheet_id = snap.target_cells[0].sheet_id;
16706 let sr0 = snap.target_cells[0].coord.row();
16707 let sc0 = snap.target_cells[0].coord.col();
16708 if snap.values.is_empty() || snap.values[0].is_empty() {
16709 return None;
16710 }
16711 let h = snap.values.len() as u32;
16712 let w = snap.values[0].len() as u32;
16713 let er0 = sr0.saturating_add(h.saturating_sub(1));
16714 let ec0 = sc0.saturating_add(w.saturating_sub(1));
16715 Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
16716 };
16717
16718 for ev in events {
16719 match ev {
16720 ChangeEvent::SpillCommitted { old, new, .. } => {
16721 if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
16722 let old_vals = if let Some(old_snap) = old {
16723 rect_from_snapshot(old_snap)
16724 .map(|(_, _, _, _, _, v)| v)
16725 .unwrap_or_else(|| {
16726 vec![
16727 vec![LiteralValue::Empty; new_vals[0].len()];
16728 new_vals.len()
16729 ]
16730 })
16731 } else {
16732 vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
16733 };
16734 undo.record_restore_computed_rect(
16735 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
16736 );
16737 }
16738 }
16739 ChangeEvent::SpillCleared { old, .. } => {
16740 if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
16741 let new_vals =
16742 vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
16743 undo.record_restore_computed_rect(
16744 sid, sr0, sc0, er0, ec0, old_vals, new_vals,
16745 );
16746 }
16747 }
16748 _ => {}
16749 }
16750 }
16751 }
16752
16753 fn mirror_value_to_computed_overlay(
16758 &mut self,
16759 sheet: &str,
16760 row: u32,
16761 col: u32,
16762 value: &LiteralValue,
16763 ) {
16764 if !(self.config.arrow_storage_enabled
16765 && self.config.delta_overlay_enabled
16766 && self.config.write_formula_overlay_enabled)
16767 {
16768 return;
16769 }
16770 if self.computed_overlay_mirroring_disabled {
16771 return;
16772 }
16773
16774 let date_system = self.arrow_sheet_date_system(sheet);
16775 let ov = Self::literal_to_overlay_value(value, date_system);
16776 self.write_computed_overlay_value_0based(
16777 sheet,
16778 row.saturating_sub(1),
16779 col.saturating_sub(1),
16780 ov,
16781 );
16782 }
16783
16784 fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
16785 if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
16786 self.record_derived_format_at(cell, format);
16787 }
16788 }
16789
16790 fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
16791 #[cfg(test)]
16792 self.derived_format_operations_for_test
16793 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16794 let mut formats = self.derived_formats.write().unwrap();
16795 match format.filter(|id| *id != crate::format::FormatId::GENERAL) {
16796 Some(format) => {
16797 formats.insert(cell, format);
16798 }
16799 None => {
16800 formats.remove(&cell);
16801 }
16802 }
16803 }
16804
16805 fn prepare_formula_plane_derived_format_sync(&self) -> Option<DerivedFormatPlaneSyncPlan> {
16806 let plane_epoch = self.graph.formula_authority().plane.epoch().0;
16807 if self.derived_formats_plane_epoch == plane_epoch {
16808 return None;
16809 }
16810
16811 if self.derived_formats.read().unwrap().is_empty() {
16812 return Some(DerivedFormatPlaneSyncPlan {
16813 plane_epoch,
16814 active_regions: Vec::new(),
16815 overlay_regions: Vec::new(),
16816 });
16817 }
16818
16819 let authority = self.graph.formula_authority();
16820 let active_regions = authority
16821 .plane
16822 .spans
16823 .active_spans()
16824 .map(|span| {
16825 crate::formula_plane::region_index::Region::from_domain(span.result_region.domain())
16826 })
16827 .collect();
16828 let overlay_regions = authority
16829 .plane
16830 .formula_overlay
16831 .active_entries()
16832 .map(|(entry, _)| {
16833 crate::formula_plane::region_index::Region::from_domain(&entry.domain)
16834 })
16835 .collect();
16836 Some(DerivedFormatPlaneSyncPlan {
16837 plane_epoch,
16838 active_regions,
16839 overlay_regions,
16840 })
16841 }
16842
16843 fn commit_formula_plane_derived_format_sync(&mut self, plan: DerivedFormatPlaneSyncPlan) {
16844 let DerivedFormatPlaneSyncPlan {
16845 plane_epoch,
16846 active_regions,
16847 overlay_regions,
16848 } = plan;
16849 if !active_regions.is_empty() {
16850 self.derived_formats.write().unwrap().retain(|cell, _| {
16851 let key = crate::formula_plane::region_index::RegionKey::new(
16852 cell.sheet_id,
16853 cell.coord.row(),
16854 cell.coord.col(),
16855 );
16856 let plane_owned = active_regions.iter().any(|region| region.contains_key(key))
16857 && overlay_regions
16858 .iter()
16859 .all(|region| !region.contains_key(key));
16860 !plane_owned
16861 });
16862 }
16863 self.derived_formats_plane_epoch = plane_epoch;
16864
16865 #[cfg(debug_assertions)]
16866 {
16867 let formats = self.derived_formats.read().unwrap();
16868 debug_assert!(formats.keys().all(|cell| {
16869 let key = crate::formula_plane::region_index::RegionKey::new(
16870 cell.sheet_id,
16871 cell.coord.row(),
16872 cell.coord.col(),
16873 );
16874 active_regions
16875 .iter()
16876 .all(|region| !region.contains_key(key))
16877 || overlay_regions
16878 .iter()
16879 .any(|region| region.contains_key(key))
16880 }));
16881 }
16882 }
16883
16884 fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
16885 self.derived_formats.write().unwrap().remove(&cell);
16886 self.derived_format_results
16887 .write()
16888 .unwrap()
16889 .retain(|vertex, _| self.graph.get_cell_ref(*vertex) != Some(cell));
16890 if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
16891 arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
16892 }
16893 }
16894
16895 fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
16896 let cells = events
16897 .iter()
16898 .filter_map(|event| match event {
16899 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16900 Some(*addr)
16901 }
16902 _ => None,
16903 })
16904 .collect::<FxHashSet<_>>();
16905 for cell in cells {
16906 let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
16907 self.clear_cell_format_state(&sheet, cell);
16908 }
16909 }
16910
16911 fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
16912 self.derived_formats
16913 .write()
16914 .unwrap()
16915 .retain(|cell, _| cell.sheet_id != sheet_id || cell.coord.row() < start0);
16916 }
16917
16918 fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
16919 self.derived_formats
16920 .write()
16921 .unwrap()
16922 .retain(|cell, _| cell.sheet_id != sheet_id || cell.coord.col() < start0);
16923 }
16924
16925 fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
16926 self.derived_formats
16927 .write()
16928 .unwrap()
16929 .retain(|cell, _| cell.sheet_id != sheet_id);
16930 }
16931
16932 #[cfg(test)]
16933 pub(crate) fn debug_computed_overlay_format_0based(
16934 &self,
16935 sheet: &str,
16936 row0: u32,
16937 col0: u32,
16938 ) -> Option<crate::format::FormatId> {
16939 let sheet = self.arrow_sheets.sheet(sheet)?;
16940 let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
16941 sheet
16942 .columns
16943 .get(col0 as usize)?
16944 .chunk(chunk_idx)?
16945 .computed_overlay
16946 .get_format(row_in_chunk)
16947 }
16948
16949 #[cfg(test)]
16950 pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
16951 &self,
16952 sheet: &str,
16953 row0: u32,
16954 col0: u32,
16955 ) -> bool {
16956 let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
16957 return false;
16958 };
16959 let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
16960 return false;
16961 };
16962 sheet
16963 .columns
16964 .get(col0 as usize)
16965 .and_then(|column| column.chunk(chunk_idx))
16966 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
16967 }
16968
16969 #[cfg(test)]
16970 pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
16971 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
16972 self.derived_formats
16973 .write()
16974 .unwrap()
16975 .remove(&CellRef::new_absolute(sheet_id, row0, col0));
16976 }
16977 }
16978
16979 #[cfg(test)]
16980 pub(crate) fn debug_record_derived_format_0based(
16981 &self,
16982 sheet: &str,
16983 row0: u32,
16984 col0: u32,
16985 format: Option<crate::format::FormatId>,
16986 ) {
16987 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
16988 self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
16989 }
16990 }
16991
16992 #[cfg(test)]
16993 pub(crate) fn debug_derived_format_0based(
16994 &self,
16995 sheet: &str,
16996 row0: u32,
16997 col0: u32,
16998 ) -> Option<crate::format::FormatId> {
16999 let sheet_id = self.graph.sheet_id(sheet)?;
17000 self.derived_formats
17001 .read()
17002 .unwrap()
17003 .get(&CellRef::new_absolute(sheet_id, row0, col0))
17004 .copied()
17005 }
17006
17007 #[cfg(test)]
17008 pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
17009 self.derived_format_operations_for_test
17010 .store(0, std::sync::atomic::Ordering::Relaxed);
17011 self.computed_overlay_set_explicit_entry_operations_for_test = 0;
17012 self.computed_overlay_stale_clear_range_effects_for_test = 0;
17013 self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
17014 self.computed_format_vector_allocations_for_test
17015 .store(0, std::sync::atomic::Ordering::Relaxed);
17016 }
17017
17018 #[cfg(test)]
17019 pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
17020 (
17021 self.derived_format_operations_for_test
17022 .load(std::sync::atomic::Ordering::Relaxed),
17023 self.computed_overlay_set_explicit_entry_operations_for_test,
17024 self.computed_format_vector_allocations_for_test
17025 .load(std::sync::atomic::Ordering::Relaxed),
17026 self.computed_overlay_stale_clear_range_effects_for_test,
17027 self.computed_overlay_stale_clear_offset_attempts_for_test,
17028 )
17029 }
17030
17031 fn write_computed_overlay_format_0based(
17032 &mut self,
17033 sheet: &str,
17034 row0: u32,
17035 col0: u32,
17036 format: Option<crate::format::FormatId>,
17037 ) {
17038 self.ensure_arrow_sheet(sheet);
17039 let (row0, col0) = (row0 as usize, col0 as usize);
17040 let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
17041 return;
17042 };
17043 if col0 >= asheet.columns.len() {
17044 asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
17045 }
17046 if row0 >= asheet.nrows as usize {
17047 asheet.ensure_row_capacity(row0 + 1);
17048 }
17049 let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
17050 return;
17051 };
17052 if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
17053 chunk.computed_overlay.set_format(offset, format);
17054 }
17055 }
17056
17057 fn write_computed_overlay_value_0based(
17058 &mut self,
17059 sheet: &str,
17060 row0: u32,
17061 col0: u32,
17062 value: OverlayValue,
17063 ) {
17064 if !(self.config.arrow_storage_enabled
17065 && self.config.delta_overlay_enabled
17066 && self.config.write_formula_overlay_enabled)
17067 {
17068 return;
17069 }
17070 if self.computed_overlay_mirroring_disabled {
17071 return;
17072 }
17073
17074 self.ensure_arrow_sheet(sheet);
17075
17076 let row0 = row0 as usize;
17077 let col0 = col0 as usize;
17078 let asheet = self
17079 .arrow_sheets
17080 .sheet_mut(sheet)
17081 .expect("ArrowSheet must exist");
17082
17083 let cur_cols = asheet.columns.len();
17084 if col0 >= cur_cols {
17085 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
17086 }
17087
17088 if row0 >= asheet.nrows as usize {
17089 if asheet.columns.is_empty() {
17090 asheet.insert_columns(0, 1);
17091 }
17092 asheet.ensure_row_capacity(row0 + 1);
17093 }
17094
17095 let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
17096 return;
17097 };
17098 let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
17099 return;
17100 };
17101
17102 let delta = ch.computed_overlay.set_scalar(in_off, value);
17103 self.adjust_computed_overlay_bytes(delta);
17104
17105 if let Some(cap) = self.config.max_overlay_memory_bytes
17106 && self.computed_overlay_bytes_estimate > cap
17107 {
17108 self.disable_computed_overlay_mirroring_due_to_budget(cap);
17109 }
17110 }
17111
17112 pub(crate) fn plan_computed_write_coalescing(
17113 &self,
17114 buffer: &ComputedWriteBuffer,
17115 ) -> ComputedWriteCoalescingPlan {
17116 self.plan_computed_write_coalescing_from_writes(
17117 buffer.writes().iter().cloned(),
17118 buffer.formats_present,
17119 )
17120 }
17121
17122 fn plan_owned_computed_write_coalescing(
17123 &self,
17124 writes: Vec<ComputedWrite>,
17125 formats_present: bool,
17126 ) -> ComputedWriteCoalescingPlan {
17127 self.plan_computed_write_coalescing_from_writes(writes, formats_present)
17128 }
17129
17130 fn plan_computed_write_coalescing_from_writes(
17131 &self,
17132 writes: impl IntoIterator<Item = ComputedWrite>,
17133 formats_present: bool,
17134 ) -> ComputedWriteCoalescingPlan {
17135 let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
17136 BTreeMap::new();
17137 let mut input_cells = 0usize;
17138
17139 for write in writes {
17140 match write {
17141 ComputedWrite::Cell {
17142 seq,
17143 sheet_id,
17144 row0,
17145 col0,
17146 value,
17147 format_id,
17148 } => {
17149 input_cells = input_cells.saturating_add(1);
17150 self.push_computed_write_plan_entry(
17151 &mut groups,
17152 seq,
17153 sheet_id,
17154 row0,
17155 col0,
17156 value,
17157 format_id,
17158 );
17159 }
17160 ComputedWrite::Rect {
17161 seq,
17162 sheet_id,
17163 sr0,
17164 sc0,
17165 values,
17166 } => {
17167 for (r_off, row) in values.into_iter().enumerate() {
17168 for (c_off, value) in row.into_iter().enumerate() {
17169 input_cells = input_cells.saturating_add(1);
17170 self.push_computed_write_plan_entry(
17171 &mut groups,
17172 seq,
17173 sheet_id,
17174 sr0.saturating_add(r_off as u32),
17175 sc0.saturating_add(c_off as u32),
17176 value,
17177 None,
17178 );
17179 }
17180 }
17181 }
17182 }
17183 }
17184
17185 let mut plan = ComputedWriteCoalescingPlan {
17186 chunks: Vec::with_capacity(groups.len()),
17187 input_cells,
17188 coalesced_cells: 0,
17189 overwritten_cells: 0,
17190 };
17191 for (key, entries) in groups {
17192 let computed_lane_has_formats =
17193 self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
17194 let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
17195 key,
17196 entries,
17197 formats_present,
17198 computed_lane_has_formats,
17199 );
17200 #[cfg(test)]
17201 if matches!(
17202 &chunk_plan.format_effect,
17203 ComputedWriteChunkFormatEffect::SetExplicit(_)
17204 ) {
17205 self.computed_format_vector_allocations_for_test
17206 .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
17207 }
17208 plan.coalesced_cells = plan
17209 .coalesced_cells
17210 .saturating_add(chunk_plan.entries.len());
17211 plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
17212 plan.chunks.push(chunk_plan);
17213 }
17214 debug_assert_eq!(
17215 plan.input_cells,
17216 plan.coalesced_cells.saturating_add(plan.overwritten_cells)
17217 );
17218 plan
17219 }
17220
17221 fn push_computed_write_plan_entry(
17222 &self,
17223 groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
17224 seq: u64,
17225 sheet_id: SheetId,
17226 row0: u32,
17227 col0: u32,
17228 value: OverlayValue,
17229 format_id: Option<crate::format::FormatId>,
17230 ) {
17231 let (chunk_idx, chunk_start_row0, row_in_chunk) =
17232 self.locate_computed_write_chunk(sheet_id, row0);
17233 let key = ComputedWriteChunkKey {
17234 sheet_id,
17235 col0,
17236 chunk_idx,
17237 chunk_start_row0,
17238 };
17239 groups
17240 .entry(key)
17241 .or_default()
17242 .push(ComputedWriteChunkEntryPlan {
17243 row_in_chunk,
17244 seq,
17245 value,
17246 format_id,
17247 });
17248 }
17249
17250 fn computed_overlay_chunk_has_formats(
17251 &self,
17252 sheet_id: SheetId,
17253 col0: u32,
17254 chunk_idx: usize,
17255 ) -> bool {
17256 let sheet_name = self.graph.sheet_name(sheet_id);
17257 self.arrow_sheets
17258 .sheet(sheet_name)
17259 .and_then(|sheet| sheet.columns.get(col0 as usize))
17260 .and_then(|column| column.chunk(chunk_idx))
17261 .is_some_and(|chunk| chunk.computed_overlay.has_formats())
17262 }
17263
17264 fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
17265 let sheet_name = self.graph.sheet_name(sheet_id);
17266 if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
17267 return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
17268 }
17269 Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
17270 }
17271
17272 fn locate_row_in_sheet_for_computed_write_plan(
17273 sheet: &crate::arrow_store::ArrowSheet,
17274 row0: usize,
17275 ) -> (usize, u32, usize) {
17276 if row0 < sheet.nrows as usize
17277 && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
17278 {
17279 let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
17280 return (chunk_idx, chunk_start as u32, row_in_chunk);
17281 }
17282
17283 let chunk_rows = sheet.chunk_rows.max(1);
17284 if sheet.chunk_starts.is_empty() {
17285 return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
17286 }
17287
17288 let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
17289 let mut chunk_start = sheet.chunk_starts[chunk_idx];
17290 while chunk_start.saturating_add(chunk_rows) <= row0 {
17291 chunk_idx = chunk_idx.saturating_add(1);
17292 chunk_start = chunk_start.saturating_add(chunk_rows);
17293 }
17294 (
17295 chunk_idx,
17296 chunk_start as u32,
17297 row0.saturating_sub(chunk_start),
17298 )
17299 }
17300
17301 fn locate_row_in_empty_sheet_for_computed_write_plan(
17302 row0: usize,
17303 chunk_rows: usize,
17304 ) -> (usize, u32, usize) {
17305 let chunk_rows = chunk_rows.max(1);
17306 let chunk_idx = row0 / chunk_rows;
17307 let chunk_start = chunk_idx.saturating_mul(chunk_rows);
17308 (
17309 chunk_idx,
17310 chunk_start as u32,
17311 row0.saturating_sub(chunk_start),
17312 )
17313 }
17314
17315 #[cfg(test)]
17316 pub(crate) fn debug_plan_computed_write_coalescing(
17317 &self,
17318 buffer: &ComputedWriteBuffer,
17319 ) -> ComputedWriteCoalescingPlan {
17320 self.plan_computed_write_coalescing(buffer)
17321 }
17322
17323 fn record_computed_write_buffer_delta(
17324 &self,
17325 buffer: &ComputedWriteBuffer,
17326 delta: &mut DeltaCollector,
17327 ) {
17328 if delta.mode == DeltaMode::Off {
17329 return;
17330 }
17331 let mut final_values = BTreeMap::new();
17332 for write in buffer.writes() {
17333 match write {
17334 ComputedWrite::Cell {
17335 sheet_id,
17336 row0,
17337 col0,
17338 value,
17339 ..
17340 } => {
17341 final_values.insert((*sheet_id, *row0, *col0), value.clone());
17342 }
17343 ComputedWrite::Rect {
17344 sheet_id,
17345 sr0,
17346 sc0,
17347 values,
17348 ..
17349 } => {
17350 for (row_offset, row) in values.iter().enumerate() {
17351 for (col_offset, value) in row.iter().enumerate() {
17352 final_values.insert(
17353 (
17354 *sheet_id,
17355 sr0.saturating_add(row_offset as u32),
17356 sc0.saturating_add(col_offset as u32),
17357 ),
17358 value.clone(),
17359 );
17360 }
17361 }
17362 }
17363 }
17364 }
17365 for ((sheet_id, row0, col0), value) in final_values {
17366 let old = self
17367 .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
17368 .unwrap_or(LiteralValue::Empty);
17369 if old != value.to_literal_for(self.config.date_system) {
17370 delta.record_cell(sheet_id, row0, col0);
17371 }
17372 }
17373 }
17374
17375 pub(crate) fn flush_computed_write_buffer(
17376 &mut self,
17377 buffer: &mut ComputedWriteBuffer,
17378 ) -> Result<(), ExcelError> {
17379 self.flush_computed_write_buffer_inner(buffer, false)
17380 }
17381
17382 fn flush_authoritative_computed_write_buffer(
17383 &mut self,
17384 buffer: &mut ComputedWriteBuffer,
17385 ) -> Result<(), ExcelError> {
17386 self.flush_computed_write_buffer_inner(buffer, true)
17387 }
17388
17389 fn flush_computed_write_buffer_inner(
17390 &mut self,
17391 buffer: &mut ComputedWriteBuffer,
17392 synchronize_formula_plane_formats: bool,
17393 ) -> Result<(), ExcelError> {
17394 if buffer.is_empty() {
17395 return Ok(());
17400 }
17401
17402 #[cfg(test)]
17403 if synchronize_formula_plane_formats
17404 && std::mem::take(&mut self.cancel_before_formula_plane_layer_commit_once_for_test)
17405 {
17406 self.active_cancel_flag
17407 .as_ref()
17408 .expect("test cancellation hook requires a cancellable evaluation")
17409 .cancel();
17410 self.cancellation_checkpoint("Evaluation cancelled before FormulaPlane layer commit")?;
17411 }
17412
17413 self.resource_checkpoint(0)?;
17418 let format_sync = synchronize_formula_plane_formats
17419 .then(|| self.prepare_formula_plane_derived_format_sync())
17420 .flatten();
17421 let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
17422 if let Some(format_sync) = format_sync {
17423 self.commit_formula_plane_derived_format_sync(format_sync);
17424 }
17425 let (writes, formats_present) = buffer.take_writes();
17426 let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
17427 self.flush_computed_write_plan(plan);
17428 self.observe_evaluation_commit_window(commit_started);
17429
17430 Ok(())
17431 }
17432
17433 fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
17434 for chunk in plan.chunks {
17435 self.flush_computed_write_chunk_plan(chunk);
17436 }
17437 }
17438
17439 fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
17440 match &chunk.shape {
17441 ComputedWriteChunkPlanShape::Point => {
17442 self.flush_computed_write_chunk_plan_as_points(chunk);
17443 }
17444 ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
17445 self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
17446 }
17447 ComputedWriteChunkPlanShape::DenseRange { .. } => {
17448 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
17449 }
17450 ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
17451 if Self::should_emit_computed_run_fragment(*len, *runs) {
17452 self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
17453 } else {
17454 self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
17455 }
17456 }
17457 }
17458 }
17459
17460 #[inline]
17461 fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
17462 runs <= len / 2
17463 }
17464
17465 fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
17466 let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
17467 for entry in chunk.entries {
17468 let row0 = chunk
17469 .chunk_start_row0
17470 .saturating_add(entry.row_in_chunk as u32);
17471 self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
17472 }
17473 self.apply_computed_overlay_format_effect(
17474 chunk.sheet_id,
17475 chunk.col0,
17476 chunk.chunk_idx,
17477 chunk.format_effect,
17478 );
17479 }
17480
17481 fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
17482 &mut self,
17483 chunk: ComputedWriteChunkPlan,
17484 ) {
17485 let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
17486 let sheet_id = chunk.sheet_id;
17487 let col0 = chunk.col0;
17488 let chunk_idx = chunk.chunk_idx;
17489 let chunk_start_row0 = chunk.chunk_start_row0;
17490 let format_effect = chunk.format_effect;
17491 let items: Vec<(usize, OverlayValue)> = chunk
17492 .entries
17493 .into_iter()
17494 .map(|entry| (entry.row_in_chunk, entry.value))
17495 .collect();
17496 match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
17497 items,
17498 point_estimate,
17499 ) {
17500 Some(Ok(fragment)) => {
17501 self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
17502 }
17503 Some(Err(cells)) => {
17504 self.flush_computed_overlay_cells_as_points(
17505 sheet_id,
17506 col0,
17507 chunk_start_row0,
17508 cells,
17509 );
17510 }
17511 None => {}
17512 }
17513 self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
17514 }
17515
17516 #[inline]
17517 fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
17518 chunk
17519 .entries
17520 .iter()
17521 .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
17522 .fold(0usize, usize::saturating_add)
17523 }
17524
17525 fn flush_computed_overlay_cells_as_points(
17526 &mut self,
17527 sheet_id: SheetId,
17528 col0: u32,
17529 chunk_start_row0: u32,
17530 cells: Vec<(usize, OverlayValue)>,
17531 ) {
17532 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
17533 for (row_in_chunk, value) in cells {
17534 let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
17535 self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
17536 }
17537 }
17538
17539 fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
17540 if chunk.entries.is_empty() {
17541 return;
17542 }
17543 let start = chunk.entries[0].row_in_chunk;
17544 let values: Vec<OverlayValue> =
17545 chunk.entries.into_iter().map(|entry| entry.value).collect();
17546 if let Some(fragment) = OverlayFragment::dense_range(start, values) {
17547 self.apply_computed_overlay_fragment(
17548 chunk.sheet_id,
17549 chunk.col0,
17550 chunk.chunk_idx,
17551 fragment,
17552 );
17553 }
17554 self.apply_computed_overlay_format_effect(
17555 chunk.sheet_id,
17556 chunk.col0,
17557 chunk.chunk_idx,
17558 chunk.format_effect,
17559 );
17560 }
17561
17562 fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
17563 if chunk.entries.is_empty() {
17564 return;
17565 }
17566 let start = chunk.entries[0].row_in_chunk;
17567 let values: Vec<OverlayValue> =
17568 chunk.entries.into_iter().map(|entry| entry.value).collect();
17569 if let Some(fragment) = OverlayFragment::run_range(start, values) {
17570 self.apply_computed_overlay_fragment(
17571 chunk.sheet_id,
17572 chunk.col0,
17573 chunk.chunk_idx,
17574 fragment,
17575 );
17576 }
17577 self.apply_computed_overlay_format_effect(
17578 chunk.sheet_id,
17579 chunk.col0,
17580 chunk.chunk_idx,
17581 chunk.format_effect,
17582 );
17583 }
17584
17585 fn apply_computed_overlay_format_effect(
17586 &mut self,
17587 sheet_id: SheetId,
17588 col0: u32,
17589 chunk_idx: usize,
17590 effect: ComputedWriteChunkFormatEffect,
17591 ) {
17592 if !(self.config.arrow_storage_enabled
17593 && self.config.delta_overlay_enabled
17594 && self.config.write_formula_overlay_enabled)
17595 || self.computed_overlay_mirroring_disabled
17596 {
17597 return;
17598 }
17599
17600 let sheet_name = self.graph.sheet_name(sheet_id);
17601 let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
17602 return;
17603 };
17604 let Some(chunk) = sheet
17605 .columns
17606 .get_mut(col0 as usize)
17607 .and_then(|column| column.chunk_mut(chunk_idx))
17608 else {
17609 return;
17610 };
17611 match effect {
17612 ComputedWriteChunkFormatEffect::NoFormatWork => {}
17613 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
17614 start,
17615 end,
17616 }) => {
17617 #[cfg(test)]
17618 {
17619 self.computed_overlay_stale_clear_range_effects_for_test = self
17620 .computed_overlay_stale_clear_range_effects_for_test
17621 .saturating_add(1);
17622 }
17623 chunk.computed_overlay.clear_format_range(start, end);
17624 }
17625 ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
17626 offsets,
17627 )) => {
17628 #[cfg(test)]
17629 {
17630 self.computed_overlay_stale_clear_offset_attempts_for_test = self
17631 .computed_overlay_stale_clear_offset_attempts_for_test
17632 .saturating_add(offsets.len() as u64);
17633 }
17634 chunk.computed_overlay.clear_format_offsets(&offsets);
17635 }
17636 ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
17637 #[cfg(test)]
17638 {
17639 self.computed_overlay_set_explicit_entry_operations_for_test = self
17640 .computed_overlay_set_explicit_entry_operations_for_test
17641 .saturating_add(formats.len() as u64);
17642 }
17643 for (row_in_chunk, format_id) in formats {
17644 chunk.computed_overlay.set_format(row_in_chunk, format_id);
17645 }
17646 }
17647 }
17648 }
17649
17650 fn apply_computed_overlay_fragment(
17651 &mut self,
17652 sheet_id: SheetId,
17653 col0: u32,
17654 chunk_idx: usize,
17655 fragment: OverlayFragment,
17656 ) {
17657 if !(self.config.arrow_storage_enabled
17658 && self.config.delta_overlay_enabled
17659 && self.config.write_formula_overlay_enabled)
17660 {
17661 return;
17662 }
17663 if self.computed_overlay_mirroring_disabled {
17664 return;
17665 }
17666
17667 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
17668 self.ensure_arrow_sheet(&sheet_name);
17669
17670 let col0 = col0 as usize;
17671 let asheet = self
17672 .arrow_sheets
17673 .sheet_mut(&sheet_name)
17674 .expect("ArrowSheet must exist");
17675
17676 let cur_cols = asheet.columns.len();
17677 if col0 >= cur_cols {
17678 asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
17679 }
17680
17681 let start_row0 = asheet
17682 .chunk_starts
17683 .get(chunk_idx)
17684 .copied()
17685 .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
17686 let required_rows =
17687 start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
17688 if required_rows > asheet.nrows as usize {
17689 if asheet.columns.is_empty() {
17690 asheet.insert_columns(0, 1);
17691 }
17692 asheet.ensure_row_capacity(required_rows);
17693 }
17694
17695 let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
17696 return;
17697 };
17698 let delta = ch.computed_overlay.apply_fragment(fragment);
17699 self.adjust_computed_overlay_bytes(delta);
17700
17701 if let Some(cap) = self.config.max_overlay_memory_bytes
17702 && self.computed_overlay_bytes_estimate > cap
17703 {
17704 self.disable_computed_overlay_mirroring_due_to_budget(cap);
17705 }
17706 }
17707
17708 #[inline]
17709 fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
17710 if delta >= 0 {
17711 self.computed_overlay_bytes_estimate = self
17712 .computed_overlay_bytes_estimate
17713 .saturating_add(delta as usize);
17714 } else {
17715 self.computed_overlay_bytes_estimate = self
17716 .computed_overlay_bytes_estimate
17717 .saturating_sub((-delta) as usize);
17718 }
17719 }
17720
17721 fn clear_all_computed_overlays(&mut self) {
17722 let mut freed_total = 0usize;
17723 for sh in self.arrow_sheets.sheets.iter_mut() {
17724 for col in sh.columns.iter_mut() {
17725 for ch in col.chunks.iter_mut() {
17726 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
17727 }
17728 for ch in col.sparse_chunks.values_mut() {
17729 freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
17730 }
17731 }
17732 }
17733 self.computed_overlay_bytes_estimate = self
17734 .computed_overlay_bytes_estimate
17735 .saturating_sub(freed_total);
17736 }
17737
17738 fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
17739 self.compact_all_computed_overlays();
17742 }
17743
17744 fn compact_all_computed_overlays(&mut self) {
17747 let mut freed_total = 0usize;
17748 for sheet in self.arrow_sheets.sheets.iter_mut() {
17749 for col_idx in 0..sheet.columns.len() {
17750 let num_dense = sheet.columns[col_idx].chunks.len();
17752 for ch_idx in 0..num_dense {
17753 freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
17754 }
17755 let sparse_keys: Vec<usize> = sheet.columns[col_idx]
17757 .sparse_chunks
17758 .keys()
17759 .copied()
17760 .collect();
17761 for ch_idx in sparse_keys {
17762 freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
17763 }
17764 }
17765 }
17766 self.computed_overlay_bytes_estimate = self
17767 .computed_overlay_bytes_estimate
17768 .saturating_sub(freed_total);
17769 self.overlay_compactions = self.overlay_compactions.saturating_add(1);
17770 }
17771
17772 fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
17773 let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
17774 }
17775
17776 fn record_vertex_value_to_overlay(
17777 &mut self,
17778 vertex_id: VertexId,
17779 value: &LiteralValue,
17780 computed_writes: Option<&mut ComputedWriteBuffer>,
17781 ) -> Result<(), ExcelError> {
17782 if !(self.config.arrow_storage_enabled
17783 && self.config.delta_overlay_enabled
17784 && self.config.write_formula_overlay_enabled)
17785 {
17786 return Ok(());
17787 }
17788 if self.computed_overlay_mirroring_disabled {
17789 return Ok(());
17790 }
17791 if !matches!(
17792 self.graph.get_vertex_kind(vertex_id),
17793 VertexKind::FormulaScalar | VertexKind::FormulaArray
17794 ) {
17795 return Ok(());
17796 }
17797 let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
17798 return Ok(());
17799 };
17800 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17801 let date_system = self.arrow_sheet_date_system(&sheet_name);
17802 let ov = Self::literal_to_overlay_value(value, date_system);
17803 if let Some(buffer) = computed_writes {
17804 let format_id = self.derived_formats.read().unwrap().get(&cell).copied();
17805 buffer.push_cell_with_format(
17806 cell.sheet_id,
17807 cell.coord.row(),
17808 cell.coord.col(),
17809 ov,
17810 format_id,
17811 );
17812 if self.should_flush_computed_write_buffer(buffer) {
17813 self.flush_computed_write_buffer(buffer)?;
17814 }
17815 } else {
17816 self.write_computed_overlay_value_0based(
17817 &sheet_name,
17818 cell.coord.row(),
17819 cell.coord.col(),
17820 ov,
17821 );
17822 }
17823 Ok(())
17824 }
17825
17826 #[inline]
17827 fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
17828 self.config.max_overlay_memory_bytes.is_some_and(|cap| {
17829 if cap == 0 {
17830 return false;
17831 }
17832 self.computed_overlay_bytes_estimate
17833 .saturating_add(buffer.estimated_bytes())
17834 > cap
17835 })
17836 }
17837
17838 pub fn overlay_memory_usage(&self) -> usize {
17840 self.computed_overlay_bytes_estimate
17841 }
17842
17843 #[cfg(test)]
17844 pub(crate) fn debug_overlay_compactions(&self) -> u64 {
17845 self.overlay_compactions
17846 }
17847
17848 #[cfg(test)]
17849 pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
17850 let mut total = 0usize;
17851 for sheet in &self.arrow_sheets.sheets {
17852 for column in &sheet.columns {
17853 for chunk in &column.chunks {
17854 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
17855 }
17856 for chunk in column.sparse_chunks.values() {
17857 total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
17858 }
17859 }
17860 }
17861 self.computed_overlay_bytes_estimate = total;
17862 total
17863 }
17864
17865 fn resolve_sheet_locator_for_write(
17866 &mut self,
17867 loc: formualizer_common::SheetLocator<'_>,
17868 current_sheet: &str,
17869 ) -> Result<SheetId, ExcelError> {
17870 Ok(match loc {
17871 formualizer_common::SheetLocator::Id(id) => id,
17872 formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
17873 formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
17874 })
17875 }
17876
17877 fn resolve_sheet_locator_for_read(
17878 &self,
17879 loc: formualizer_common::SheetLocator<'_>,
17880 current_sheet: &str,
17881 ) -> Result<SheetId, ExcelError> {
17882 match loc {
17883 formualizer_common::SheetLocator::Id(id) => Ok(id),
17884 formualizer_common::SheetLocator::Name(name) => self
17885 .graph
17886 .sheet_id(name.as_ref())
17887 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
17888 formualizer_common::SheetLocator::Current => self
17889 .graph
17890 .sheet_id(current_sheet)
17891 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
17892 }
17893 }
17894
17895 pub fn set_cell_value(
17897 &mut self,
17898 sheet: &str,
17899 row: u32,
17900 col: u32,
17901 value: LiteralValue,
17902 ) -> Result<(), ExcelError> {
17903 self.observe_function_semantic_epoch()?;
17904 let sheet_existed = self.graph.sheet_id(sheet).is_some();
17905 let sheet_id = self.graph.sheet_id_mut(sheet);
17906 let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
17907 let replaced_formula =
17908 self.graph
17909 .get_vertex_id_for_address(&cell_ref)
17910 .is_some_and(|vertex| {
17911 matches!(
17912 self.graph.get_vertex_kind(*vertex),
17913 VertexKind::FormulaScalar | VertexKind::FormulaArray
17914 )
17915 });
17916 self.demote_span_containing_cell_for_write(
17917 sheet_id,
17918 row.saturating_sub(1),
17919 col.saturating_sub(1),
17920 )
17921 .map_err(Self::editor_error_to_excel)?;
17922 self.graph.set_cell_value(sheet, row, col, value.clone())?;
17923 self.clear_cell_format_state(sheet, cell_ref);
17924 self.record_formula_plane_changed_cell(sheet, row, col);
17925 if !sheet_existed || replaced_formula {
17926 self.mark_topology_edited();
17927 }
17928 self.mirror_value_to_overlay(sheet, row, col, &value);
17930 self.mark_data_edited();
17932 Ok(())
17933 }
17934
17935 fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
17940 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
17941 return;
17942 }
17943 let sheet_id = self.graph.sheet_id_mut(sheet);
17944 self.record_formula_plane_structural_change(StructuralScope::Cell {
17945 sheet: sheet_id,
17946 row: row.saturating_sub(1),
17947 col: col.saturating_sub(1),
17948 });
17949 }
17950
17951 fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
17952 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
17953 return;
17954 }
17955
17956 match event {
17957 ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
17958 self.record_formula_plane_structural_change(StructuralScope::Cell {
17959 sheet: addr.sheet_id,
17960 row: addr.coord.row(),
17961 col: addr.coord.col(),
17962 });
17963 }
17964 ChangeEvent::SpillCommitted { new, .. } => {
17965 if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
17966 self.record_formula_plane_structural_change(scope);
17967 }
17968 }
17969 ChangeEvent::SpillCleared { old, .. } => {
17970 if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
17971 self.record_formula_plane_structural_change(scope);
17972 }
17973 }
17974 ChangeEvent::DefineName { .. }
17975 | ChangeEvent::UpdateName { .. }
17976 | ChangeEvent::DeleteName { .. }
17977 | ChangeEvent::NamedRangeAdjusted { .. } => {
17978 self.record_formula_plane_structural_change(StructuralScope::AllSheets);
17982 }
17983 ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
17984 }
17987 ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
17988 self.record_formula_plane_structural_change(StructuralScope::Region(
17989 Region::whole_row(*sheet_id, *row0),
17990 ));
17991 }
17992 ChangeEvent::AddVertex { .. }
17993 | ChangeEvent::RemoveVertex { .. }
17994 | ChangeEvent::EdgeAdded { .. }
17995 | ChangeEvent::EdgeRemoved { .. }
17996 | ChangeEvent::CompoundStart { .. }
17997 | ChangeEvent::CompoundEnd { .. }
17998 | ChangeEvent::StagedFormulaCellChanged { .. } => {}
17999 }
18000 }
18001
18002 fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
18003 if self.config.formula_plane_mode == FormulaPlaneMode::Off {
18004 return;
18005 }
18006
18007 match scope {
18008 StructuralScope::Cell { sheet, row, col } => {
18009 self.graph
18010 .mark_formula_region_dirty(Region::point(sheet, row, col));
18011 }
18012 StructuralScope::Region(region) => {
18013 self.graph.mark_formula_region_dirty(region);
18014 }
18015 StructuralScope::Sheet(sheet_id) => {
18016 self.graph
18017 .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
18018 }
18019 StructuralScope::RemovedSheet(sheet_id) => {
18020 let removed_refs = {
18021 let authority = self.graph.formula_authority();
18022 authority
18023 .active_span_refs()
18024 .into_iter()
18025 .filter(|span_ref| {
18026 authority
18027 .plane
18028 .spans
18029 .get(*span_ref)
18030 .map(|span| span.sheet_id == sheet_id)
18031 .unwrap_or(false)
18032 })
18033 .collect::<Vec<_>>()
18034 };
18035
18036 {
18037 let authority = self.graph.formula_authority_mut();
18038 for span_ref in removed_refs {
18039 authority.plane.remove_span(span_ref);
18040 }
18041 let _ = authority.rebuild_indexes();
18042 }
18043 self.graph
18044 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
18045 }
18046 StructuralScope::OpaqueGlobal => {
18047 let _ = self.graph.formula_authority_mut().rebuild_indexes();
18048 self.graph
18049 .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
18050 }
18051 StructuralScope::AllSheets => {
18052 let _ = self.graph.formula_authority_mut().rebuild_indexes();
18056 }
18057 }
18058 }
18059
18060 fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
18061 let first = cells.first()?;
18062 let sheet_id = first.sheet_id;
18063 if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
18064 return Some(StructuralScope::OpaqueGlobal);
18065 }
18066 let mut row_start = first.coord.row();
18067 let mut row_end = row_start;
18068 let mut col_start = first.coord.col();
18069 let mut col_end = col_start;
18070 for cell in cells.iter().skip(1) {
18071 row_start = row_start.min(cell.coord.row());
18072 row_end = row_end.max(cell.coord.row());
18073 col_start = col_start.min(cell.coord.col());
18074 col_end = col_end.max(cell.coord.col());
18075 }
18076 Some(StructuralScope::Region(Region::rect(
18077 sheet_id, row_start, row_end, col_start, col_end,
18078 )))
18079 }
18080
18081 pub fn set_cell_value_ref(
18082 &mut self,
18083 cell: formualizer_common::SheetCellRef<'_>,
18084 current_sheet: &str,
18085 value: LiteralValue,
18086 ) -> Result<(), ExcelError> {
18087 let owned = cell.into_owned();
18088 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
18089 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
18090 self.set_cell_value(
18091 &sheet_name,
18092 owned.coord.row() + 1,
18093 owned.coord.col() + 1,
18094 value,
18095 )
18096 }
18097
18098 pub fn set_cell_formula_ref(
18099 &mut self,
18100 cell: formualizer_common::SheetCellRef<'_>,
18101 current_sheet: &str,
18102 ast: ASTNode,
18103 ) -> Result<(), ExcelError> {
18104 let owned = cell.into_owned();
18105 let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
18106 let sheet_name = self.graph.sheet_name(sheet_id).to_string();
18107 self.set_cell_formula(
18108 &sheet_name,
18109 owned.coord.row() + 1,
18110 owned.coord.col() + 1,
18111 ast,
18112 )
18113 }
18114
18115 pub fn get_cell_value_ref(
18116 &self,
18117 cell: formualizer_common::SheetCellRef<'_>,
18118 current_sheet: &str,
18119 ) -> Result<Option<LiteralValue>, ExcelError> {
18120 let owned = cell.into_owned();
18121 let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
18122 let sheet_name = self.graph.sheet_name(sheet_id);
18123 Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
18124 }
18125
18126 pub fn resolve_range_view_sheet_ref<'c>(
18127 &'c self,
18128 r: &formualizer_common::SheetRef<'_>,
18129 current_sheet: &str,
18130 ) -> Result<RangeView<'c>, ExcelError> {
18131 use formualizer_common::SheetLocator;
18132
18133 let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
18134 match loc {
18135 SheetLocator::Current => Ok(None),
18136 SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
18137 SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
18138 }
18139 };
18140
18141 let rt = match r {
18142 formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
18143 sheet: sheet_to_opt_name(cell.sheet.clone())?,
18144 row: cell.coord.row() + 1,
18145 col: cell.coord.col() + 1,
18146 row_abs: cell.coord.row_abs(),
18147 col_abs: cell.coord.col_abs(),
18148 },
18149 formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
18150 sheet: sheet_to_opt_name(range.sheet.clone())?,
18151 start_row: range.start_row.map(|b| b.index + 1),
18152 start_col: range.start_col.map(|b| b.index + 1),
18153 end_row: range.end_row.map(|b| b.index + 1),
18154 end_col: range.end_col.map(|b| b.index + 1),
18155 start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
18156 start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
18157 end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
18158 end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
18159 },
18160 };
18161
18162 crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
18163 }
18164
18165 pub fn set_cell_formula(
18167 &mut self,
18168 sheet: &str,
18169 row: u32,
18170 col: u32,
18171 ast: ASTNode,
18172 ) -> Result<(), ExcelError> {
18173 self.observe_function_semantic_epoch()?;
18174 let sheet_id = self.graph.sheet_id_mut(sheet);
18175 self.demote_span_containing_cell_for_write(
18176 sheet_id,
18177 row.saturating_sub(1),
18178 col.saturating_sub(1),
18179 )
18180 .map_err(Self::editor_error_to_excel)?;
18181 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18182 let ingested = {
18183 let mut pipeline = self.ingest_pipeline();
18184 pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
18185 };
18186 self.graph.set_cell_formula_with_plan(
18187 sheet,
18188 row,
18189 col,
18190 ingested.ast_id,
18191 &ingested.dep_plan,
18192 ingested.dep_plan.volatile,
18193 ingested.dep_plan.dynamic,
18194 )?;
18195 self.clear_cell_format_state(sheet, placement);
18196 self.record_formula_plane_changed_cell(sheet, row, col);
18197
18198 self.clear_delta_overlay_cell(sheet, row, col);
18203
18204 self.mark_topology_edited();
18206 Ok(())
18207 }
18208
18209 pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
18211 where
18212 I: IntoIterator<Item = (u32, u32, ASTNode)>,
18213 {
18214 let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
18215 let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
18216 let sheet_id = self.graph.sheet_id_mut(sheet);
18217 let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
18218 let placement =
18219 PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
18220 self.graph
18221 .formula_authority()
18222 .plane
18223 .spans
18224 .find_at(placement)
18225 .is_some()
18226 });
18227 if writes_inside_active_span {
18228 self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
18229 .map_err(Self::editor_error_to_excel)?;
18230 }
18231 let ingested = {
18232 let mut pipeline = self.ingest_pipeline();
18233 let inputs = collected.into_iter().map(|(row, col, ast)| {
18234 let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18235 (FormulaAstInput::Tree(ast), placement, None)
18236 });
18237 pipeline.ingest_batch(inputs)?
18238 };
18239 let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
18240 .into_iter()
18241 .map(|formula| {
18242 (
18243 formula.placement.coord.row() + 1,
18244 formula.placement.coord.col() + 1,
18245 formula.ast_id,
18246 formula.dep_plan,
18247 )
18248 })
18249 .collect();
18250 let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
18251 for (row, col) in edited_cells {
18252 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18253 self.clear_cell_format_state(sheet, cell);
18254 self.record_formula_plane_changed_cell(sheet, row, col);
18255 }
18256 if n > 0 {
18258 self.mark_topology_edited();
18259 }
18260 Ok(n)
18261 }
18262
18263 #[inline]
18264 fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
18265 match v {
18266 LiteralValue::Empty => None,
18267 LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
18268 other => Some(other),
18269 }
18270 }
18271
18272 fn materialize_temporal_egress(
18273 value: LiteralValue,
18274 class: Option<&formualizer_common::numfmt::FormatClass>,
18275 policy: crate::engine::TemporalEgress,
18276 date_system: crate::engine::DateSystem,
18277 ) -> LiteralValue {
18278 use formualizer_common::numfmt::FormatClass;
18279 if policy == crate::engine::TemporalEgress::Serial {
18280 return value;
18281 }
18282 let LiteralValue::Number(serial) = value else {
18283 return value;
18284 };
18285 match class {
18286 Some(FormatClass::Date) => {
18287 formualizer_common::try_serial_to_date_for(date_system, serial)
18288 .map(LiteralValue::Date)
18289 .unwrap_or(LiteralValue::Number(serial))
18290 }
18291 Some(FormatClass::DateTime) => {
18292 formualizer_common::try_serial_to_datetime_for(date_system, serial)
18293 .map(LiteralValue::DateTime)
18294 .unwrap_or(LiteralValue::Number(serial))
18295 }
18296 Some(FormatClass::Time) => {
18297 let seconds = (serial.rem_euclid(1.0) * 86_400.0).round() as u32 % 86_400;
18298 chrono::NaiveTime::from_num_seconds_from_midnight_opt(seconds, 0)
18299 .map(LiteralValue::Time)
18300 .unwrap_or(LiteralValue::Number(serial))
18301 }
18302 Some(FormatClass::Duration) => {
18303 let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
18304 if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
18305 LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
18306 } else {
18307 LiteralValue::Number(serial)
18308 }
18309 }
18310 _ => LiteralValue::Number(serial),
18311 }
18312 }
18313
18314 pub(crate) fn effective_format_id(
18315 &self,
18316 sheet: &str,
18317 row: u32,
18318 col: u32,
18319 ) -> Option<crate::format::FormatId> {
18320 let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
18321 arrow.format_id(
18322 row.saturating_sub(1) as usize,
18323 col.saturating_sub(1) as usize,
18324 )
18325 });
18326 arrow.or_else(|| {
18327 let sheet_id = self.graph.sheet_id(sheet)?;
18328 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18329 self.derived_formats.read().unwrap().get(&cell).copied()
18330 })
18331 }
18332
18333 pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
18335 let raw = self.read_cell_value(sheet, row, col)?;
18336 let format = self.effective_format_id(sheet, row, col);
18337 let class = format.and_then(|id| self.format_registry.class(id));
18338 Self::normalize_public_cell_read(Self::materialize_temporal_egress(
18339 raw,
18340 class,
18341 self.config.temporal_egress,
18342 self.config.date_system,
18343 ))
18344 }
18345
18346 pub fn get_range_values(
18348 &self,
18349 sheet: &str,
18350 sr: u32,
18351 sc: u32,
18352 er: u32,
18353 ec: u32,
18354 ) -> Vec<Vec<LiteralValue>> {
18355 let height = er.saturating_sub(sr).saturating_add(1) as usize;
18356 let width = ec.saturating_sub(sc).saturating_add(1) as usize;
18357 let Some(asheet) = self.sheet_store().sheet(sheet) else {
18358 return vec![vec![LiteralValue::Empty; width]; height];
18359 };
18360 let view = asheet.range_view(
18361 sr.saturating_sub(1) as usize,
18362 sc.saturating_sub(1) as usize,
18363 er.saturating_sub(1) as usize,
18364 ec.saturating_sub(1) as usize,
18365 );
18366 let sheet_id = self.graph.sheet_id(sheet);
18367 let derived_formats = self.derived_formats.read().unwrap();
18368 let has_derived_formats = sheet_id
18369 .is_some_and(|sheet_id| derived_formats.keys().any(|cell| cell.sheet_id == sheet_id));
18370 let mut out = Vec::with_capacity(height);
18371 if !asheet.has_formats() && !has_derived_formats {
18372 for rr in 0..height {
18373 let mut row = Vec::with_capacity(width);
18374 for cc in 0..width {
18375 row.push(view.get_cell(rr, cc));
18376 }
18377 out.push(row);
18378 }
18379 return out;
18380 }
18381 let format_registry = &self.format_registry;
18382 for rr in 0..height {
18383 let mut row = Vec::with_capacity(width);
18384 for cc in 0..width {
18385 let raw = view.get_cell(rr, cc);
18386 let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
18387 let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
18388 let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
18389 let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
18390 derived_formats.get(&cell).copied()
18391 });
18392 let class = format.and_then(|id| format_registry.class(id));
18393 row.push(Self::materialize_temporal_egress(
18394 raw,
18395 class,
18396 self.config.temporal_egress,
18397 self.config.date_system,
18398 ));
18399 }
18400 out.push(row);
18401 }
18402 out
18403 }
18404
18405 pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
18407 let asheet = self.sheet_store().sheet(sheet)?;
18408 let r0 = row.saturating_sub(1) as usize;
18409 let c0 = col.saturating_sub(1) as usize;
18410 let v = asheet.get_cell_value(r0, c0);
18411 if matches!(v, LiteralValue::Empty) {
18412 None
18413 } else {
18414 Some(v)
18415 }
18416 }
18417
18418 pub(crate) fn read_range_values(
18420 &self,
18421 sheet: &str,
18422 sr: u32,
18423 sc: u32,
18424 er: u32,
18425 ec: u32,
18426 ) -> RangeView<'_> {
18427 let Some(asheet) = self.sheet_store().sheet(sheet) else {
18428 return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
18429 };
18430 if er < sr || ec < sc {
18431 return asheet.range_view(1, 1, 0, 0);
18432 }
18433 let sr0 = sr.saturating_sub(1) as usize;
18434 let sc0 = sc.saturating_sub(1) as usize;
18435 let er0 = er.saturating_sub(1) as usize;
18436 let ec0 = ec.saturating_sub(1) as usize;
18437 asheet.range_view(sr0, sc0, er0, ec0)
18438 }
18439
18440 pub(crate) fn formula_plane_span_ast_at(
18441 &self,
18442 span_ref: FormulaSpanRef,
18443 placement: PlacementCoord,
18444 ) -> Option<ASTNode> {
18445 let authority = self.graph.formula_authority();
18446 let span = authority.plane.spans.get(span_ref)?;
18447 let relocation = span.ast_relocation;
18448 let mut ast = self
18449 .graph
18450 .data_store()
18451 .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
18452 if let Some(binding_set_id) = span.binding_set_id {
18453 let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
18454 if !binding_set.is_single_literal_binding() {
18455 let binding =
18456 binding_set.literal_bindings_for_placement(&span.domain, placement)?;
18457 ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
18458 }
18459 }
18460 let row = placement.row.checked_add(1)?;
18461 let col = placement.col.checked_add(1)?;
18462 let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
18463 let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
18464 relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
18465 }
18466
18467 pub fn get_cell(
18469 &self,
18470 sheet: &str,
18471 row: u32,
18472 col: u32,
18473 ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
18474 let v = self.get_cell_value(sheet, row, col);
18475 let sheet_id = self.graph.sheet_id(sheet)?;
18476 let coord = Coord::from_excel(row, col, true, true);
18477 let cell = CellRef::new(sheet_id, coord);
18478 let placement =
18479 crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
18480 let handle = self
18481 .graph
18482 .formula_authority()
18483 .plane
18484 .resolve_formula_at(placement, None);
18485 match handle.resolution {
18486 crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
18487 let ast = self.formula_plane_span_ast_at(span, placement);
18491 return Some((ast, v));
18492 }
18493 crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
18494 let ast = self
18495 .graph
18496 .formula_authority()
18497 .plane
18498 .formula_overlay
18499 .get(overlay_ref)
18500 .and_then(|overlay| match overlay.kind {
18501 crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
18502 template_id,
18503 ) => self
18504 .graph
18505 .formula_authority()
18506 .plane
18507 .templates
18508 .get(template_id)
18509 .and_then(|template| {
18510 self.graph
18511 .data_store()
18512 .retrieve_ast(template.ast_id, self.graph.sheet_reg())
18513 }),
18514 _ => None,
18515 });
18516 return Some((ast, v));
18517 }
18518 _ => {}
18519 }
18520
18521 if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
18522 let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
18523 self.graph
18524 .data_store()
18525 .retrieve_ast(ast_id, self.graph.sheet_reg())
18526 });
18527 Some((ast, v))
18528 } else if v.is_some() {
18529 Some((None, v))
18530 } else {
18531 None
18532 }
18533 }
18534
18535 pub fn begin_batch(&mut self) {
18537 self.graph.begin_batch();
18538 }
18539
18540 pub fn end_batch(&mut self) {
18542 self.graph.end_batch();
18543 }
18544
18545 pub fn begin_deferred_dirty(&mut self) {
18554 self.graph.begin_deferred_dirty();
18555 }
18556
18557 pub fn end_deferred_dirty(&mut self) {
18560 let _ = self.graph.end_deferred_dirty();
18561 }
18562
18563 pub fn dirty_propagation_visits(&self) -> u64 {
18567 self.graph.dirty_propagation_visits()
18568 }
18569
18570 #[inline]
18573 fn record_cell_if_changed(
18574 delta: &mut DeltaCollector,
18575 cell: &CellRef,
18576 old: &LiteralValue,
18577 new: &LiteralValue,
18578 ) {
18579 if old != new {
18580 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
18581 }
18582 }
18583
18584 pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
18585 self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
18586 engine.observe_function_semantic_epoch()?;
18587 engine.resource_checkpoint(1)?;
18589 if !engine.graph.vertex_exists(vertex_id) {
18590 return engine.evaluate_vertex_impl(vertex_id, None);
18591 }
18592 engine.transition_off_mode_spans_to_legacy()?;
18593 let is_formula = matches!(
18594 engine.graph.get_vertex_kind(vertex_id),
18595 VertexKind::FormulaScalar | VertexKind::FormulaArray
18596 );
18597 if is_formula {
18598 engine.begin_evaluation_request();
18599 engine.graph.flush_pending_edge_deltas();
18600 let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
18601 vertex_id,
18602 )];
18603 if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
18604 && engine.graph.formula_authority().active_span_count() > 0
18605 {
18606 engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
18607 } else {
18608 engine.evaluate_legacy_target_roots(&roots, None)?;
18609 }
18610 } else if engine.config.formula_plane_mode
18611 == FormulaPlaneMode::AuthoritativeExperimental
18612 && engine.graph.formula_authority().active_span_count() > 0
18613 {
18614 engine.begin_evaluation_request();
18615 engine.graph.flush_pending_edge_deltas();
18616 engine.evaluate_authoritative_formula_plane(None, None)?;
18617 }
18618 engine.evaluate_vertex_impl(vertex_id, None)
18619 })
18620 }
18621
18622 fn evaluate_vertex_impl(
18623 &mut self,
18624 vertex_id: VertexId,
18625 delta: Option<&mut DeltaCollector>,
18626 ) -> Result<LiteralValue, ExcelError> {
18627 if !self.graph.vertex_exists(vertex_id) {
18631 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
18632 .with_message(format!("Vertex not found: {vertex_id:?}")));
18633 }
18634 if self.active_resource_ledger.is_some()
18635 && matches!(
18636 self.graph.get_vertex_kind(vertex_id),
18637 VertexKind::FormulaScalar | VertexKind::FormulaArray
18638 )
18639 {
18640 let value = self
18641 .evaluate_vertex_immutable(vertex_id)
18642 .unwrap_or_else(LiteralValue::Error);
18643 let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
18644 self.resource_checkpoint(0)?;
18646 let mut delta = delta;
18647 for effect in &effects {
18648 self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
18649 }
18650 return Ok(value);
18651 }
18652
18653 let mut delta = delta;
18654
18655 let kind = self.graph.get_vertex_kind(vertex_id);
18657 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
18658
18659 let ast_id = match kind {
18660 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
18661 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
18662 ast_id
18663 } else {
18664 return Ok(LiteralValue::Number(0.0));
18665 }
18666 }
18667 VertexKind::Empty | VertexKind::Cell => {
18668 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
18669 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
18670 let row = cell_ref.coord.row() + 1;
18671 let col = cell_ref.coord.col() + 1;
18672 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18673 return Ok(v);
18674 }
18675 }
18676 return Ok(LiteralValue::Number(0.0));
18677 }
18678 VertexKind::NamedScalar => {
18679 let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
18680 return Ok(value);
18681 }
18682 VertexKind::NamedArray => {
18683 let value = self.evaluate_named_array(vertex_id, sheet_id)?;
18684 return Ok(value);
18685 }
18686 VertexKind::InfiniteRange
18687 | VertexKind::Range
18688 | VertexKind::External
18689 | VertexKind::Table => {
18690 return Ok(LiteralValue::Number(0.0));
18692 }
18693 };
18694
18695 let sheet_name = self.graph.sheet_name(sheet_id);
18697 let cell_ref = self
18698 .graph
18699 .get_cell_ref(vertex_id)
18700 .expect("cell ref for vertex");
18701 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
18702
18703 let result =
18704 interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
18705
18706 match result {
18708 Ok(cv) => {
18709 let derived_format = cv.format_id();
18710 self.record_derived_format(vertex_id, derived_format);
18711 let result_literal =
18712 crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
18713 let output_sheet_name = sheet_name.to_string();
18714 self.write_computed_overlay_format_0based(
18715 &output_sheet_name,
18716 cell_ref.coord.row(),
18717 cell_ref.coord.col(),
18718 derived_format,
18719 );
18720 match result_literal {
18721 LiteralValue::Array(rows) => {
18722 self.graph
18724 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
18725 let anchor = self
18727 .graph
18728 .get_cell_ref(vertex_id)
18729 .expect("cell ref for vertex");
18730 let sheet_id = anchor.sheet_id;
18731 let h = rows.len() as u32;
18732 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
18733
18734 let spill_cells = (h as u64).saturating_mul(w as u64);
18736 if spill_cells > self.config.spill.max_spill_cells as u64 {
18737 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
18738 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18739 .with_message("SpillTooLarge")
18740 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18741 expected_rows: h,
18742 expected_cols: w,
18743 });
18744 let spill_val = LiteralValue::Error(spill_err.clone());
18745 if let Some(d) = delta.as_deref_mut() {
18746 let old = self
18747 .read_cell_value(
18748 self.graph.sheet_name(anchor.sheet_id),
18749 anchor.coord.row() + 1,
18750 anchor.coord.col() + 1,
18751 )
18752 .unwrap_or(LiteralValue::Empty);
18753 if old != spill_val {
18754 d.record_cell(
18755 anchor.sheet_id,
18756 anchor.coord.row(),
18757 anchor.coord.col(),
18758 );
18759 }
18760 }
18761 self.graph.update_vertex_value(vertex_id, spill_val.clone());
18762 if self.config.arrow_storage_enabled
18763 && self.config.delta_overlay_enabled
18764 && self.config.write_formula_overlay_enabled
18765 {
18766 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
18767 self.mirror_value_to_computed_overlay(
18768 &sheet_name,
18769 anchor.coord.row() + 1,
18770 anchor.coord.col() + 1,
18771 &spill_val,
18772 );
18773 }
18774 return Ok(spill_val);
18775 }
18776 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
18780 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
18781 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
18782 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
18783 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18784 .with_message("Spill exceeds sheet bounds")
18785 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18786 expected_rows: h,
18787 expected_cols: w,
18788 });
18789 let spill_val = LiteralValue::Error(spill_err.clone());
18790 if let Some(d) = delta.as_deref_mut() {
18791 let old = self
18792 .read_cell_value(
18793 self.graph.sheet_name(anchor.sheet_id),
18794 anchor.coord.row() + 1,
18795 anchor.coord.col() + 1,
18796 )
18797 .unwrap_or(LiteralValue::Empty);
18798 if old != spill_val {
18799 d.record_cell(
18800 anchor.sheet_id,
18801 anchor.coord.row(),
18802 anchor.coord.col(),
18803 );
18804 }
18805 }
18806 self.graph.update_vertex_value(vertex_id, spill_val.clone());
18807 if self.config.arrow_storage_enabled
18808 && self.config.delta_overlay_enabled
18809 && self.config.write_formula_overlay_enabled
18810 {
18811 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
18812 self.mirror_value_to_computed_overlay(
18813 &sheet_name,
18814 anchor.coord.row() + 1,
18815 anchor.coord.col() + 1,
18816 &spill_val,
18817 );
18818 }
18819 return Ok(spill_val);
18820 }
18821 let mut targets = Vec::new();
18822 for r in 0..h {
18823 for c in 0..w {
18824 targets.push(self.graph.make_cell_ref_internal(
18825 sheet_id,
18826 anchor.coord.row() + r,
18827 anchor.coord.col() + c,
18828 ));
18829 }
18830 }
18831
18832 match self.spill_mgr.reserve(
18834 vertex_id,
18835 anchor,
18836 SpillShape { rows: h, cols: w },
18837 SpillMeta {
18838 epoch: self.recalc_epoch,
18839 config: self.config.spill,
18840 },
18841 ) {
18842 Ok(()) => {
18843 if let Err(e) = self.commit_spill_and_mirror(
18847 vertex_id,
18848 &targets,
18849 rows.clone(),
18850 delta.as_deref_mut(),
18851 None,
18852 ) {
18853 self.clear_spill_projection_and_mirror(
18855 vertex_id,
18856 delta.as_deref_mut(),
18857 );
18858 if let Some(d) = delta.as_deref_mut() {
18859 let old = self
18860 .read_cell_value(
18861 self.graph.sheet_name(anchor.sheet_id),
18862 anchor.coord.row() + 1,
18863 anchor.coord.col() + 1,
18864 )
18865 .unwrap_or(LiteralValue::Empty);
18866 let new = LiteralValue::Error(e.clone());
18867 if old != new {
18868 d.record_cell(
18869 anchor.sheet_id,
18870 anchor.coord.row(),
18871 anchor.coord.col(),
18872 );
18873 }
18874 }
18875 let err_val = LiteralValue::Error(e.clone());
18876 self.graph.update_vertex_value(vertex_id, err_val.clone());
18877 if self.config.arrow_storage_enabled
18878 && self.config.delta_overlay_enabled
18879 && self.config.write_formula_overlay_enabled
18880 {
18881 let sheet_name =
18882 self.graph.sheet_name(anchor.sheet_id).to_string();
18883 self.mirror_value_to_computed_overlay(
18884 &sheet_name,
18885 anchor.coord.row() + 1,
18886 anchor.coord.col() + 1,
18887 &err_val,
18888 );
18889 }
18890 return Ok(err_val);
18891 }
18892 let top_left = rows
18894 .first()
18895 .and_then(|r| r.first())
18896 .cloned()
18897 .unwrap_or(LiteralValue::Empty);
18898 self.graph.update_vertex_value(vertex_id, top_left.clone());
18899 Ok(top_left)
18900 }
18901 Err(e) => {
18902 self.clear_spill_projection_and_mirror(
18903 vertex_id,
18904 delta.as_deref_mut(),
18905 );
18906 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18907 .with_message(
18908 e.message.unwrap_or_else(|| "Spill blocked".to_string()),
18909 )
18910 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18911 expected_rows: h,
18912 expected_cols: w,
18913 });
18914 let spill_val = LiteralValue::Error(spill_err.clone());
18915 if let Some(d) = delta.as_deref_mut() {
18916 let old = self
18917 .read_cell_value(
18918 self.graph.sheet_name(anchor.sheet_id),
18919 anchor.coord.row() + 1,
18920 anchor.coord.col() + 1,
18921 )
18922 .unwrap_or(LiteralValue::Empty);
18923 if old != spill_val {
18924 d.record_cell(
18925 anchor.sheet_id,
18926 anchor.coord.row(),
18927 anchor.coord.col(),
18928 );
18929 }
18930 }
18931 self.graph.update_vertex_value(vertex_id, spill_val.clone());
18932 if self.config.arrow_storage_enabled
18933 && self.config.delta_overlay_enabled
18934 && self.config.write_formula_overlay_enabled
18935 {
18936 let sheet_name =
18937 self.graph.sheet_name(anchor.sheet_id).to_string();
18938 self.mirror_value_to_computed_overlay(
18939 &sheet_name,
18940 anchor.coord.row() + 1,
18941 anchor.coord.col() + 1,
18942 &spill_val,
18943 );
18944 }
18945 Ok(spill_val)
18946 }
18947 }
18948 }
18949 other => {
18950 let spill_cells = self
18952 .graph
18953 .spill_cells_for_anchor(vertex_id)
18954 .map(|cells| cells.to_vec())
18955 .unwrap_or_default();
18956 if let Some(d) = delta.as_deref_mut()
18957 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
18958 {
18959 if spill_cells.is_empty() {
18960 let old = self
18961 .read_cell_value(
18962 self.graph.sheet_name(anchor.sheet_id),
18963 anchor.coord.row() + 1,
18964 anchor.coord.col() + 1,
18965 )
18966 .unwrap_or(LiteralValue::Empty);
18967 if old != other {
18968 d.record_cell(
18969 anchor.sheet_id,
18970 anchor.coord.row(),
18971 anchor.coord.col(),
18972 );
18973 }
18974 } else {
18975 for cell in spill_cells.iter() {
18976 let sheet_name = self.graph.sheet_name(cell.sheet_id);
18977 let old = self
18978 .get_cell_value(
18979 sheet_name,
18980 cell.coord.row() + 1,
18981 cell.coord.col() + 1,
18982 )
18983 .unwrap_or(LiteralValue::Empty);
18984 let new = if cell.sheet_id == anchor.sheet_id
18985 && cell.coord.row() == anchor.coord.row()
18986 && cell.coord.col() == anchor.coord.col()
18987 {
18988 other.clone()
18989 } else {
18990 LiteralValue::Empty
18991 };
18992 Self::record_cell_if_changed(d, cell, &old, &new);
18993 }
18994 }
18995 }
18996 self.graph.clear_spill_region(vertex_id);
18997 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
18998 self.record_formula_plane_structural_change(scope);
18999 }
19000 if self.config.arrow_storage_enabled
19001 && self.config.delta_overlay_enabled
19002 && self.config.write_formula_overlay_enabled
19003 {
19004 let empty = LiteralValue::Empty;
19005 for cell in spill_cells.iter() {
19006 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19007 self.mirror_value_to_computed_overlay(
19008 &sheet_name,
19009 cell.coord.row() + 1,
19010 cell.coord.col() + 1,
19011 &empty,
19012 );
19013 }
19014 }
19015 self.graph.update_vertex_value(vertex_id, other.clone());
19016 if self.config.arrow_storage_enabled
19018 && self.config.delta_overlay_enabled
19019 && self.config.write_formula_overlay_enabled
19020 {
19021 let anchor = self
19022 .graph
19023 .get_cell_ref(vertex_id)
19024 .expect("cell ref for vertex");
19025 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
19026 self.mirror_value_to_computed_overlay(
19027 &sheet_name,
19028 anchor.coord.row() + 1,
19029 anchor.coord.col() + 1,
19030 &other,
19031 );
19032 }
19033 Ok(other)
19034 }
19035 }
19036 }
19037 Err(e) => {
19038 let spill_cells = self
19041 .graph
19042 .spill_cells_for_anchor(vertex_id)
19043 .map(|cells| cells.to_vec())
19044 .unwrap_or_default();
19045 let err_val = LiteralValue::Error(e.clone());
19046 if let Some(d) = delta
19047 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
19048 {
19049 if spill_cells.is_empty() {
19050 let old = self
19051 .read_cell_value(
19052 self.graph.sheet_name(anchor.sheet_id),
19053 anchor.coord.row() + 1,
19054 anchor.coord.col() + 1,
19055 )
19056 .unwrap_or(LiteralValue::Empty);
19057 if old != err_val {
19058 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
19059 }
19060 } else {
19061 for cell in spill_cells.iter() {
19062 let sheet_name = self.graph.sheet_name(cell.sheet_id);
19063 let old = self
19064 .get_cell_value(
19065 sheet_name,
19066 cell.coord.row() + 1,
19067 cell.coord.col() + 1,
19068 )
19069 .unwrap_or(LiteralValue::Empty);
19070 let new = if cell.sheet_id == anchor.sheet_id
19071 && cell.coord.row() == anchor.coord.row()
19072 && cell.coord.col() == anchor.coord.col()
19073 {
19074 err_val.clone()
19075 } else {
19076 LiteralValue::Empty
19077 };
19078 Self::record_cell_if_changed(d, cell, &old, &new);
19079 }
19080 }
19081 }
19082 self.graph.clear_spill_region(vertex_id);
19083 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
19084 self.record_formula_plane_structural_change(scope);
19085 }
19086 if self.config.arrow_storage_enabled
19087 && self.config.delta_overlay_enabled
19088 && self.config.write_formula_overlay_enabled
19089 {
19090 let empty = LiteralValue::Empty;
19091 for cell in spill_cells.iter() {
19092 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19093 self.mirror_value_to_computed_overlay(
19094 &sheet_name,
19095 cell.coord.row() + 1,
19096 cell.coord.col() + 1,
19097 &empty,
19098 );
19099 }
19100 }
19101 self.graph.update_vertex_value(vertex_id, err_val.clone());
19102 if self.config.arrow_storage_enabled
19103 && self.config.delta_overlay_enabled
19104 && self.config.write_formula_overlay_enabled
19105 {
19106 let anchor = self
19107 .graph
19108 .get_cell_ref(vertex_id)
19109 .expect("cell ref for vertex");
19110 let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
19111 self.mirror_value_to_computed_overlay(
19112 &sheet_name,
19113 anchor.coord.row() + 1,
19114 anchor.coord.col() + 1,
19115 &err_val,
19116 );
19117 }
19118 Ok(err_val)
19119 }
19120 }
19121 }
19122
19123 fn evaluate_named_scalar(
19124 &mut self,
19125 vertex_id: VertexId,
19126 sheet_id: SheetId,
19127 ) -> Result<LiteralValue, ExcelError> {
19128 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19129 ExcelError::new(ExcelErrorKind::Name)
19130 .with_message("Named range metadata missing".to_string())
19131 })?;
19132
19133 match &named_range.definition {
19134 NamedDefinition::Cell(cell_ref) => {
19135 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19136 let row = cell_ref.coord.row() + 1;
19137 let col = cell_ref.coord.col() + 1;
19138
19139 if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
19140 && matches!(
19141 self.graph.get_vertex_kind(dep_vertex),
19142 VertexKind::FormulaScalar | VertexKind::FormulaArray
19143 )
19144 {
19145 let value = self.evaluate_vertex(dep_vertex)?;
19147 self.graph.update_vertex_value(vertex_id, value.clone());
19148 Ok(value)
19149 } else {
19150 let value = self
19151 .get_cell_value(sheet_name, row, col)
19152 .unwrap_or(LiteralValue::Empty);
19153 self.graph.update_vertex_value(vertex_id, value.clone());
19154 Ok(value)
19155 }
19156 }
19157 NamedDefinition::Literal(v) => {
19158 let out = v.clone();
19159 self.graph.update_vertex_value(vertex_id, out.clone());
19160 Ok(out)
19161 }
19162 NamedDefinition::Formula { ast, .. } => {
19163 let context_sheet = match named_range.scope {
19164 NameScope::Sheet(id) => id,
19165 NameScope::Workbook => sheet_id,
19166 };
19167 let sheet_name = self.graph.sheet_name(context_sheet);
19168 let cell_ref = self
19169 .graph
19170 .get_cell_ref(vertex_id)
19171 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19172 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
19173 match interpreter.evaluate_ast(ast) {
19174 Ok(cv) => {
19175 let value = cv.into_literal();
19176 match value {
19177 LiteralValue::Array(_) => {
19178 let err = ExcelError::new(ExcelErrorKind::NImpl)
19179 .with_message("Array result in scalar named range".to_string());
19180 let err_val = LiteralValue::Error(err.clone());
19181 self.graph.update_vertex_value(vertex_id, err_val.clone());
19182 Ok(err_val)
19183 }
19184 other => {
19185 self.graph.update_vertex_value(vertex_id, other.clone());
19186 Ok(other)
19187 }
19188 }
19189 }
19190 Err(err) => {
19191 let err_val = LiteralValue::Error(err.clone());
19192 self.graph.update_vertex_value(vertex_id, err_val.clone());
19193 Ok(err_val)
19194 }
19195 }
19196 }
19197 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
19198 .with_message("Range-valued name evaluated as scalar".to_string())),
19199 }
19200 }
19201
19202 fn evaluate_named_array(
19203 &mut self,
19204 vertex_id: VertexId,
19205 sheet_id: SheetId,
19206 ) -> Result<LiteralValue, ExcelError> {
19207 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19208 ExcelError::new(ExcelErrorKind::Name)
19209 .with_message("Named range metadata missing".to_string())
19210 })?;
19211
19212 let out = match &named_range.definition {
19213 NamedDefinition::Range(range_ref) => {
19214 if range_ref.start.sheet_id != range_ref.end.sheet_id {
19215 return Err(ExcelError::new(ExcelErrorKind::Ref)
19216 .with_message("Named range cannot span sheets".to_string()));
19217 }
19218
19219 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19220 let sr0 = range_ref.start.coord.row();
19221 let sc0 = range_ref.start.coord.col();
19222 let er0 = range_ref.end.coord.row();
19223 let ec0 = range_ref.end.coord.col();
19224 if sr0 > er0 || sc0 > ec0 {
19225 return Err(ExcelError::new(ExcelErrorKind::Ref)
19226 .with_message("Invalid named range bounds".to_string()));
19227 }
19228
19229 let h = (er0 - sr0 + 1) as usize;
19230 let w = (ec0 - sc0 + 1) as usize;
19231 let cell_count = (h as u64).saturating_mul(w as u64);
19232 if cell_count > self.config.spill.max_spill_cells as u64 {
19233 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19234 "Named range too large to materialize as an array".to_string(),
19235 ));
19236 }
19237
19238 let mut rows = Vec::with_capacity(h);
19239 for r0 in sr0..=er0 {
19240 let mut row = Vec::with_capacity(w);
19241 for c0 in sc0..=ec0 {
19242 let v = self
19243 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
19244 .unwrap_or(LiteralValue::Empty);
19245 row.push(v);
19246 }
19247 rows.push(row);
19248 }
19249 LiteralValue::Array(rows)
19250 }
19251 NamedDefinition::Cell(cell_ref) => {
19252 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19253 let row = cell_ref.coord.row() + 1;
19254 let col = cell_ref.coord.col() + 1;
19255 let v = self
19256 .get_cell_value(sheet_name, row, col)
19257 .unwrap_or(LiteralValue::Empty);
19258 LiteralValue::Array(vec![vec![v]])
19259 }
19260 NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
19261 NamedDefinition::Formula { ast, .. } => {
19262 let context_sheet = match named_range.scope {
19263 NameScope::Sheet(id) => id,
19264 NameScope::Workbook => sheet_id,
19265 };
19266 let sheet_name = self.graph.sheet_name(context_sheet);
19267 let cell_ref = self
19268 .graph
19269 .get_cell_ref(vertex_id)
19270 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19271 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
19272 match interpreter.evaluate_ast(ast) {
19273 Ok(cv) => {
19274 let v = cv.into_literal();
19275 match v {
19276 LiteralValue::Array(_) => v,
19277 other => LiteralValue::Array(vec![vec![other]]),
19278 }
19279 }
19280 Err(err) => LiteralValue::Error(err),
19281 }
19282 }
19283 };
19284
19285 self.graph.update_vertex_value(vertex_id, out.clone());
19286 Ok(out)
19287 }
19288
19289 fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
19290 crate::engine::ResourceLedgerError::Exhausted(
19291 formualizer_common::ResourceExhaustionDetail {
19292 reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
19293 limit: limit as u64,
19294 observed: limit.saturating_add(1) as u64,
19295 request_id: self
19296 .active_evaluation_resource_request
19297 .as_ref()
19298 .map(|stats| stats.request_id),
19299 },
19300 )
19301 .into_excel_error()
19302 .with_message(format!("{context} did not converge after {limit} replans"))
19303 }
19304
19305 fn transient_target_preparation_stale(error: &ExcelError) -> bool {
19306 matches!(
19307 &error.extra,
19308 formualizer_common::ExcelErrorExtra::PreparationStale {
19309 reason: formualizer_common::PreparationStaleReason::Semantic
19310 | formualizer_common::PreparationStaleReason::Provider
19311 }
19312 )
19313 }
19314
19315 fn prepare_graph_for_routed_evaluation(
19316 &mut self,
19317 targets: &[crate::engine::EvaluationTarget],
19318 options: &crate::engine::TargetEvalOptions<'_>,
19319 ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
19320 const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
19321 let mut retries = 0usize;
19322 loop {
19323 match self.prepare_graph_for_targets_unobserved(targets, options) {
19324 Err(error)
19325 if Self::transient_target_preparation_stale(&error)
19326 && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
19327 {
19328 retries = retries.saturating_add(1);
19329 }
19330 result => return result,
19331 }
19332 }
19333 }
19334
19335 fn evaluate_mixed_targets(
19336 &mut self,
19337 targets: &[crate::engine::EvaluationTarget],
19338 delta: Option<&mut DeltaCollector>,
19339 ) -> Result<EvalResult, ExcelError> {
19340 let _source_cache = self.source_cache_session();
19341 let cancel = self.active_cancel_flag.clone();
19342 let options = crate::engine::TargetEvalOptions {
19343 request_id: self
19344 .active_evaluation_resource_request
19345 .as_ref()
19346 .map(|stats| stats.request_id),
19347 cancel,
19348 deadline: None,
19349 budgets: None,
19350 opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
19351 };
19352 self.prepare_and_execute_target_recipe(targets, &options, delta)
19353 }
19354
19355 fn prepare_and_execute_target_recipe(
19356 &mut self,
19357 targets: &[crate::engine::EvaluationTarget],
19358 options: &crate::engine::TargetEvalOptions<'_>,
19359 delta: Option<&mut DeltaCollector>,
19360 ) -> Result<EvalResult, ExcelError> {
19361 let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
19362 self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
19363 }
19364
19365 fn execute_prepared_target_recipe(
19366 &mut self,
19367 targets: &[crate::engine::EvaluationTarget],
19368 scope: &crate::engine::PrepareScope,
19369 delta: Option<&mut DeltaCollector>,
19370 ) -> Result<EvalResult, ExcelError> {
19371 self.transition_off_mode_spans_to_legacy()?;
19372 if matches!(scope, crate::engine::PrepareScope::Workbook)
19373 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
19374 {
19375 stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
19376 }
19377 let mut roots = self.resolve_target_producers(targets)?;
19378 if let crate::engine::PrepareScope::Sheets(sheets) = scope {
19379 let request_id = self
19380 .active_evaluation_resource_request
19381 .as_ref()
19382 .map(|request| request.request_id);
19383 let root_count = roots.len();
19384 let mut widened_roots =
19385 OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
19386 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
19387 let sheet_ids = sheets
19388 .iter()
19389 .filter_map(|sheet| self.graph.sheet_id(sheet))
19390 .collect::<FxHashSet<_>>();
19391 for vertex in self.graph.formula_vertices() {
19392 if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
19393 widened_roots
19394 .push(crate::engine::target_preparation::TargetProducer::Legacy(
19395 vertex,
19396 ))
19397 .map_err(|_| {
19398 target_root_allocation_error(widened_roots.len() + 1, request_id)
19399 })?;
19400 }
19401 }
19402 let authority = self.graph.formula_authority();
19403 for span_ref in authority.active_span_refs() {
19404 let Some(span) = authority.plane.spans.get(span_ref) else {
19405 continue;
19406 };
19407 if sheet_ids.contains(&span.sheet_id) {
19408 widened_roots
19409 .push(crate::engine::target_preparation::TargetProducer::Span {
19410 span_ref,
19411 demanded: Region::from_domain(span.result_region.domain()),
19412 })
19413 .map_err(|_| {
19414 target_root_allocation_error(widened_roots.len() + 1, request_id)
19415 })?;
19416 }
19417 }
19418 roots = widened_roots.into_vec();
19419 }
19420 self.begin_evaluation_request();
19421 self.graph.flush_pending_edge_deltas();
19422 let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
19423 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
19424 && self.graph.formula_authority().active_span_count() > 0
19425 {
19426 if workbook_scope {
19427 self.evaluate_authoritative_formula_plane(None, delta)
19428 } else {
19429 self.evaluate_authoritative_formula_plane_targets(&roots, delta)
19430 }
19431 } else {
19432 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
19433 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
19434 }
19435 if workbook_scope {
19436 if let Some(delta) = delta {
19437 self.evaluate_all_with_delta_collector(delta)
19438 } else {
19439 self.evaluate_all_legacy_impl()
19440 }
19441 } else {
19442 self.evaluate_legacy_target_roots(&roots, delta)
19443 }
19444 }
19445 }
19446
19447 fn legacy_coordinate_targets(
19448 &mut self,
19449 targets: &[(&str, u32, u32)],
19450 ) -> Vec<crate::engine::EvaluationTarget> {
19451 targets
19452 .iter()
19453 .map(|(sheet, row, col)| {
19454 self.graph.sheet_id_mut(sheet);
19457 crate::engine::EvaluationTarget::Cell {
19458 sheet: (*sheet).to_string(),
19459 row: *row,
19460 col: *col,
19461 }
19462 })
19463 .collect()
19464 }
19465
19466 pub fn evaluate_targets(
19468 &mut self,
19469 targets: &[crate::engine::EvaluationTarget],
19470 ) -> Result<EvalResult, ExcelError> {
19471 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19472 engine.observe_function_semantic_epoch()?;
19473 engine.validate_deterministic_mode()?;
19474 engine.evaluate_mixed_targets(targets, None)
19475 })
19476 }
19477
19478 pub fn evaluate_targets_with_options(
19480 &mut self,
19481 targets: &[crate::engine::EvaluationTarget],
19482 options: crate::engine::TargetEvalOptions<'_>,
19483 ) -> Result<EvalResult, ExcelError> {
19484 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19485 engine.active_cancel_flag = options.cancel.clone();
19486 engine.active_evaluation_deadline = options.deadline;
19487 let result = (|| {
19488 engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
19489 engine.observe_function_semantic_epoch()?;
19490 engine.validate_deterministic_mode()?;
19491 let _source_cache = engine.source_cache_session();
19492 engine.prepare_and_execute_target_recipe(targets, &options, None)
19493 })();
19494 engine.active_cancel_flag = None;
19495 engine.active_evaluation_deadline = None;
19496 result
19497 })
19498 }
19499
19500 pub fn evaluate_targets_with_delta(
19502 &mut self,
19503 targets: &[crate::engine::EvaluationTarget],
19504 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
19505 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
19506 engine.observe_function_semantic_epoch()?;
19507 engine.validate_deterministic_mode()?;
19508 let mut collector = DeltaCollector::new(DeltaMode::Cells);
19509 let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
19510 Ok((result, collector.finish_target()))
19511 })
19512 }
19513
19514 pub fn evaluate_until(
19516 &mut self,
19517 targets: &[(&str, u32, u32)],
19518 ) -> Result<EvalResult, ExcelError> {
19519 self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19520 engine.evaluate_until_unobserved(targets)
19521 })
19522 }
19523
19524 fn evaluate_until_unobserved(
19525 &mut self,
19526 targets: &[(&str, u32, u32)],
19527 ) -> Result<EvalResult, ExcelError> {
19528 self.observe_function_semantic_epoch()?;
19529 let targets = self.legacy_coordinate_targets(targets);
19530 self.evaluate_mixed_targets(&targets, None)
19531 }
19532
19533 fn evaluate_until_with_delta_collector(
19534 &mut self,
19535 targets: &[(&str, u32, u32)],
19536 delta: &mut DeltaCollector,
19537 ) -> Result<EvalResult, ExcelError> {
19538 let targets = self.legacy_coordinate_targets(targets);
19539 self.evaluate_mixed_targets(&targets, Some(delta))
19540 }
19541
19542 fn evaluate_legacy_target_roots(
19543 &mut self,
19544 roots: &[crate::engine::target_preparation::TargetProducer],
19545 mut delta: Option<&mut DeltaCollector>,
19546 ) -> Result<EvalResult, ExcelError> {
19547 use crate::engine::target_preparation::TargetProducer;
19548 #[cfg(any(test, feature = "benchmark_internal"))]
19549 {
19550 self.recalc_reuse_probe
19551 .get_mut()
19552 .unwrap()
19553 .legacy_target_requests += 1;
19554 }
19555 let start = crate::instant::FzInstant::now();
19556 let root_vertices = roots
19557 .iter()
19558 .filter_map(|root| match root {
19559 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
19560 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
19561 })
19562 .collect::<Vec<_>>();
19563 let mut computed_vertices = 0usize;
19564 let mut cycle_errors = 0usize;
19565 let mut replans = 0usize;
19566 const MAX_REPLAN: usize = 5;
19567 loop {
19568 let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
19569 if precedents_to_eval.is_empty() {
19570 break;
19571 }
19572 #[cfg(any(test, feature = "benchmark_internal"))]
19573 {
19574 self.recalc_reuse_probe
19575 .get_mut()
19576 .unwrap()
19577 .target_schedule_builds += 1;
19578 }
19579 let scheduler = Scheduler::new(&self.graph);
19580 let schedule =
19581 scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
19582 for &unit in &schedule.units {
19583 self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
19584 match unit {
19585 ScheduleUnit::Cycle(index) => {
19586 if self.handle_cycle_unit(
19587 schedule.unit_cycle(index),
19588 delta.as_deref_mut(),
19589 None,
19590 None,
19591 )? > 0
19592 {
19593 cycle_errors = cycle_errors.saturating_add(1);
19594 }
19595 }
19596 ScheduleUnit::Layer(index) => {
19597 let layer = schedule.unit_layer(index);
19598 let evaluated = if let Some(delta) = delta.as_deref_mut() {
19599 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
19600 self.evaluate_layer_parallel_with_delta(layer, delta)?
19601 } else {
19602 self.evaluate_layer_sequential_with_delta(layer, delta)?
19603 }
19604 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
19605 self.evaluate_layer_parallel(layer)?
19606 } else {
19607 self.evaluate_layer_sequential(layer)?
19608 };
19609 computed_vertices = computed_vertices.saturating_add(evaluated);
19610 }
19611 }
19612 }
19613 let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
19614 self.resource_checkpoint(0)?;
19615 self.graph.clear_dirty_flags(&precedents_to_eval);
19616 for vertex in &changed {
19617 self.graph.set_dirty(*vertex, true);
19618 }
19619 if changed.is_empty() {
19620 break;
19621 }
19622 if replans >= MAX_REPLAN {
19623 return Err(self.replan_exhausted_error(
19624 MAX_REPLAN,
19625 "targeted legacy dynamic dependency evaluation",
19626 ));
19627 }
19628 replans = replans.saturating_add(1);
19629 }
19630 self.redirty_for_next_recalc();
19631 Ok(EvalResult {
19632 computed_vertices,
19633 cycle_errors,
19634 elapsed: start.elapsed(),
19635 })
19636 }
19637
19638 pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
19640 if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
19641 return Err(
19642 Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
19643 "compatibility recalculation plans require all staged formulas to be prepared",
19644 ),
19645 );
19646 }
19647 let key = self.recalc_plan_key();
19648 let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
19649 vertices.sort_unstable();
19650 let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
19651 let schedule = if vertices.is_empty() {
19652 crate::engine::Schedule {
19653 units: Vec::new(),
19654 layers: Vec::new(),
19655 cycles: Vec::new(),
19656 }
19657 } else {
19658 self.create_evaluation_schedule_uncached(&vertices)?.0
19659 };
19660 self.validate_recalc_plan_key(&key)?;
19661 Ok(RecalcPlan {
19662 key,
19663 kind: RecalcPlanKind::CompatibilityFull {
19664 schedule,
19665 has_dynamic_refs,
19666 },
19667 })
19668 }
19669
19670 pub fn build_recalc_plan_for_targets(
19672 &mut self,
19673 targets: &[crate::engine::EvaluationTarget],
19674 ) -> Result<RecalcPlan, ExcelError> {
19675 self.build_recalc_plan_for_targets_with_options(
19676 targets,
19677 crate::engine::TargetEvalOptions::default(),
19678 )
19679 }
19680
19681 pub fn build_recalc_plan_for_targets_with_options(
19682 &mut self,
19683 targets: &[crate::engine::EvaluationTarget],
19684 options: crate::engine::TargetEvalOptions<'_>,
19685 ) -> Result<RecalcPlan, ExcelError> {
19686 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19687 engine.observe_function_semantic_epoch()?;
19688 engine.validate_deterministic_mode()?;
19689 let _source_cache = engine.source_cache_session();
19690 let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
19691 engine.graph.flush_pending_edge_deltas();
19692 let topology = if matches!(
19693 preparation.widened_scope,
19694 crate::engine::PrepareScope::Workbook
19695 ) {
19696 RecalcTopology::Workbook
19697 } else {
19698 RecalcTopology::RunLocalRecipe
19699 };
19700 Ok(RecalcPlan {
19701 key: engine.recalc_plan_key(),
19702 kind: RecalcPlanKind::Target {
19703 targets: targets.to_vec(),
19704 scope: preparation.widened_scope,
19705 topology,
19706 dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
19707 },
19708 })
19709 })
19710 }
19711
19712 pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
19714 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19715 engine.evaluate_recalc_plan_unobserved(plan)
19716 })
19717 }
19718
19719 pub fn evaluate_recalc_plan_with_controls(
19720 &mut self,
19721 plan: &RecalcPlan,
19722 cancel: Option<crate::engine::CancelToken>,
19723 deadline: Option<Instant>,
19724 ) -> Result<EvalResult, ExcelError> {
19725 self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19726 engine.active_cancel_flag = cancel.clone();
19727 engine.active_evaluation_deadline = deadline;
19728 let result = engine.evaluate_recalc_plan_unobserved(plan);
19729 engine.active_cancel_flag = None;
19730 engine.active_evaluation_deadline = None;
19731 result
19732 })
19733 }
19734
19735 fn evaluate_recalc_plan_unobserved(
19736 &mut self,
19737 plan: &RecalcPlan,
19738 ) -> Result<EvalResult, ExcelError> {
19739 self.graph.flush_pending_edge_deltas();
19740 self.validate_recalc_plan_key(&plan.key)?;
19741 self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
19742 self.validate_deterministic_mode()?;
19743
19744 match &plan.kind {
19745 RecalcPlanKind::Target {
19746 targets,
19747 scope,
19748 topology,
19749 dynamic_policy,
19750 } => {
19751 debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
19752 debug_assert_eq!(
19753 matches!(topology, RecalcTopology::Workbook),
19754 matches!(scope, crate::engine::PrepareScope::Workbook)
19755 );
19756 let _source_cache = self.source_cache_session();
19757 self.execute_prepared_target_recipe(targets, scope, None)
19758 }
19759 RecalcPlanKind::CompatibilityFull {
19760 schedule,
19761 has_dynamic_refs,
19762 } => {
19763 let _source_cache = self.source_cache_session();
19764 let transitioned = self.transition_off_mode_spans_to_legacy()?;
19765 self.begin_evaluation_request();
19766 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
19767 && self.graph.formula_authority().active_span_count() > 0
19768 {
19769 return self.evaluate_authoritative_formula_plane_all();
19770 }
19771 if *has_dynamic_refs {
19772 self.virtual_dep_fallback_activations =
19773 self.virtual_dep_fallback_activations.saturating_add(1);
19774 if !transitioned {
19775 return self.evaluate_all_coordinator();
19776 }
19777 }
19778 if transitioned {
19779 return self.evaluate_all_legacy_impl();
19780 }
19781
19782 let start = crate::instant::FzInstant::now();
19783 let dirty_vertices = self.graph.get_evaluation_vertices();
19784 if dirty_vertices.is_empty() {
19785 return Ok(EvalResult {
19786 computed_vertices: 0,
19787 cycle_errors: 0,
19788 elapsed: start.elapsed(),
19789 });
19790 }
19791
19792 let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
19793 let mut computed_vertices = 0;
19794 let mut cycle_errors = 0;
19795 for &unit in &schedule.units {
19796 self.cancellation_checkpoint(
19797 "Evaluation cancelled before recalculation plan schedule unit",
19798 )?;
19799 match unit {
19800 ScheduleUnit::Cycle(i) => {
19801 let stamped = self.handle_cycle_unit(
19802 schedule.unit_cycle(i),
19803 None,
19804 Some(&dirty_set),
19805 None,
19806 )?;
19807 if stamped > 0 {
19808 cycle_errors += 1;
19809 }
19810 }
19811 ScheduleUnit::Layer(i) => {
19812 let work: Vec<VertexId> = schedule
19813 .unit_layer(i)
19814 .vertices
19815 .iter()
19816 .copied()
19817 .filter(|v| dirty_set.contains(v))
19818 .collect();
19819 if work.is_empty() {
19820 continue;
19821 }
19822 let temp_layer = crate::engine::scheduler::Layer { vertices: work };
19823 if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
19824 computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
19825 } else {
19826 computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
19827 }
19828 }
19829 }
19830 }
19831
19832 self.resource_checkpoint(0)?;
19833 self.graph.clear_dirty_flags(&dirty_vertices);
19834 self.redirty_for_next_recalc();
19835 Ok(EvalResult {
19836 computed_vertices,
19837 cycle_errors,
19838 elapsed: start.elapsed(),
19839 })
19840 }
19841 }
19842 }
19843 fn evaluate_all_legacy_and_ack_dirty(
19844 &mut self,
19845 formula_dirty: FormulaDirtyLease,
19846 ) -> Result<EvalResult, ExcelError> {
19847 let result = self.evaluate_all_legacy_impl()?;
19848 self.checked_ack_formula_dirty_observed(formula_dirty)?;
19849 Ok(result)
19850 }
19851
19852 fn evaluate_formula_plane_capacity_fallback(
19853 &mut self,
19854 mut formula_dirty: FormulaDirtyLease,
19855 owned_dirty_events: &[usize],
19856 selected_span_refs: &[FormulaSpanRef],
19857 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19858 delta: Option<&mut DeltaCollector>,
19859 ) -> Result<EvalResult, ExcelError> {
19860 let leased_len = formula_dirty.len();
19861 if !selected_span_refs.is_empty() {
19862 let materialization_started = crate::instant::FzInstant::now();
19863 let prepared = self
19864 .prepare_formula_span_demotion(selected_span_refs)
19865 .map_err(|error| {
19866 if let FormulaSpanDemotionError::Resource(error) = error {
19867 error
19868 } else {
19869 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
19870 "FormulaPlane capacity fallback demotion preparation failed: {error}"
19871 ))
19872 }
19873 })?;
19874 let report = self
19875 .commit_prepared_formula_span_demotion(prepared)
19876 .map_err(|error| {
19877 ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
19878 "FormulaPlane capacity fallback demotion commit failed: {error}"
19879 ))
19880 })?;
19881 self.observe_materialization(
19882 report.placements_materialized,
19883 false,
19884 materialization_started.elapsed(),
19885 );
19886 }
19887 #[cfg(test)]
19888 {
19889 self.last_formula_plane_span_eval_report = None;
19890 }
19891 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
19892 formula_dirty = extended;
19893 }
19894 let mut acknowledged = owned_dirty_events.to_vec();
19895 acknowledged.extend(leased_len..formula_dirty.len());
19896 let result = if let Some(target_roots) = target_roots {
19897 let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
19898 self.evaluate_legacy_target_roots(&refreshed, delta)?
19899 } else if let Some(delta) = delta {
19900 self.evaluate_all_with_delta_collector(delta)?
19901 } else {
19902 self.evaluate_all_legacy_impl()?
19903 };
19904 self.formula_plane_capacity_bailouts =
19905 self.formula_plane_capacity_bailouts.saturating_add(1);
19906 if target_roots.is_some() {
19907 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
19908 } else {
19909 self.checked_ack_formula_dirty_observed(formula_dirty)?;
19910 }
19911 Ok(result)
19912 }
19913
19914 fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
19915 self.evaluate_authoritative_formula_plane(None, None)
19916 }
19917
19918 fn evaluate_authoritative_formula_plane_targets(
19919 &mut self,
19920 roots: &[crate::engine::target_preparation::TargetProducer],
19921 delta: Option<&mut DeltaCollector>,
19922 ) -> Result<EvalResult, ExcelError> {
19923 self.evaluate_authoritative_formula_plane(Some(roots), delta)
19924 }
19925
19926 fn evaluate_authoritative_formula_plane(
19927 &mut self,
19928 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19929 mut delta: Option<&mut DeltaCollector>,
19930 ) -> Result<EvalResult, ExcelError> {
19931 let mut formula_dirty = self.graph.lease_formula_dirty();
19935 self.observe_dirty_lease_acquired(formula_dirty.is_empty());
19936
19937 if self.graph.formula_authority().active_span_count() == 0 {
19940 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
19941 #[cfg(test)]
19942 {
19943 self.last_formula_plane_span_eval_report = None;
19944 }
19945 if let Some(target_roots) = target_roots {
19946 let result =
19947 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
19948 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
19949 return Ok(result);
19950 }
19951 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
19952 }
19953
19954 if formula_dirty.is_empty() {
19957 self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
19958 #[cfg(test)]
19959 {
19960 self.last_formula_plane_span_eval_report = None;
19961 }
19962 if let Some(target_roots) = target_roots {
19963 let result =
19964 self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
19965 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
19966 return Ok(result);
19967 }
19968 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
19969 }
19970
19971 let start = crate::instant::FzInstant::now();
19972 let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
19973 let mut workbook_exact_attempted = target_roots.is_none();
19974 let mut prepass_cycle_errors = 0usize;
19977 const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
19978 let mut cycle_demote_iters = 0usize;
19979 let mut include_dirty_regions = true;
19980 let mut retry_whole_spans = Vec::new();
19981 let mut computed_vertices = 0usize;
19982 let mut runtime_replan_iterations = 0usize;
19983 const MAX_RUNTIME_REPLAN: usize = 5;
19984 let mut array_result_demotions = 0usize;
19985 const MAX_ARRAY_RESULT_DEMOTIONS: usize = 64;
19986 let mut virtual_telemetry = self
19987 .config
19988 .enable_virtual_dep_telemetry
19989 .then(|| self.start_virtual_dep_telemetry());
19990 'virtual_replan: loop {
19991 self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
19992 let (
19993 schedule,
19994 span_refs_by_id,
19995 plane_epoch,
19996 legacy_vertices,
19997 owned_dirty_events,
19998 island_plan,
19999 ) = loop {
20000 let (
20001 schedule,
20002 span_refs_by_id,
20003 plane_epoch,
20004 legacy_vertices,
20005 owned_dirty_events,
20006 island_plan,
20007 ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
20008 self.build_formula_plane_target_schedule(
20009 &formula_dirty,
20010 &retry_whole_spans,
20011 include_dirty_regions,
20012 target_roots,
20013 )?
20014 } else {
20015 self.build_formula_plane_mixed_schedule(
20016 &formula_dirty,
20017 &retry_whole_spans,
20018 include_dirty_regions,
20019 )?
20020 };
20021 #[cfg(test)]
20022 let mut schedule = schedule;
20023 #[cfg(test)]
20024 if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
20025 let producer = schedule
20026 .layers
20027 .iter()
20028 .flat_map(|layer| layer.work.iter())
20029 .map(|work| work.producer)
20030 .next()
20031 .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
20032 schedule.fallbacks.push(
20033 crate::formula_plane::scheduler::MixedScheduleFallback {
20034 producer,
20035 reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
20036 },
20037 );
20038 }
20039
20040 if schedule.is_authoritative_safe() {
20041 break (
20042 schedule,
20043 span_refs_by_id,
20044 plane_epoch,
20045 legacy_vertices,
20046 owned_dirty_events,
20047 island_plan,
20048 );
20049 }
20050
20051 let has_cycle_fallback = schedule.stats.cycle_count > 0
20056 || schedule
20057 .fallbacks
20058 .iter()
20059 .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
20060 if !has_cycle_fallback {
20061 let selected_span_refs = schedule
20062 .layers
20063 .iter()
20064 .flat_map(|layer| layer.work.iter())
20065 .filter_map(|work| match work.producer {
20066 FormulaProducerId::Span(span_id) => {
20067 span_refs_by_id.get(&span_id).copied()
20068 }
20069 FormulaProducerId::Legacy(_) => None,
20070 })
20071 .collect::<Vec<_>>();
20072
20073 return self.evaluate_formula_plane_capacity_fallback(
20074 formula_dirty,
20075 &owned_dirty_events,
20076 &selected_span_refs,
20077 runtime_target_roots.as_deref(),
20078 delta.as_deref_mut(),
20079 );
20080 }
20081
20082 let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
20092 if !cyclic_spans.is_empty() {
20093 self.demote_cyclic_spans(&cyclic_spans)?;
20094 }
20095
20096 if self.graph.formula_authority().active_span_count() == 0 {
20097 if let Some(target_roots) = runtime_target_roots.as_deref() {
20100 let refreshed =
20101 self.resolve_target_producers_from_existing_roots(target_roots)?;
20102 let result =
20103 self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
20104 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
20105 return Ok(result);
20106 }
20107 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
20108 }
20109
20110 prepass_cycle_errors =
20114 prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
20115
20116 retry_whole_spans = if runtime_target_roots.is_some() {
20120 let active = self
20121 .graph
20122 .formula_authority()
20123 .active_span_refs()
20124 .into_iter()
20125 .map(|span_ref| (span_ref.id, span_ref))
20126 .collect::<BTreeMap<_, _>>();
20127 let mut refs = schedule
20128 .layers
20129 .iter()
20130 .flat_map(|layer| layer.work.iter())
20131 .filter_map(|work| match work.producer {
20132 FormulaProducerId::Span(id) => active.get(&id).copied(),
20133 FormulaProducerId::Legacy(_) => None,
20134 })
20135 .collect::<Vec<_>>();
20136 refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
20137 match fallback.producer {
20138 FormulaProducerId::Span(id) => active.get(&id).copied(),
20139 FormulaProducerId::Legacy(_) => None,
20140 }
20141 }));
20142 refs.sort_by_key(|span_ref| span_ref.id);
20143 refs.dedup();
20144 refs
20145 } else {
20146 self.graph.formula_authority().active_span_refs()
20147 };
20148 self.graph.mark_formula_spans_dirty(
20149 retry_whole_spans.iter().copied(),
20150 WholeSpanDirtyReason::CycleRetry,
20151 );
20152 if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
20157 {
20158 formula_dirty = extended;
20159 #[cfg(test)]
20160 if std::mem::take(
20161 &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
20162 ) {
20163 self.graph.mark_formula_spans_dirty(
20164 retry_whole_spans.iter().copied().take(1),
20165 WholeSpanDirtyReason::GlobalInvalidation,
20166 );
20167 }
20168 }
20169 include_dirty_regions = false;
20170
20171 cycle_demote_iters += 1;
20172 if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
20173 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
20174 "FormulaPlane cyclic demotion did not converge".to_string(),
20175 ));
20176 }
20177 };
20178 self.observe_formula_plane_route(
20179 &island_plan,
20180 FormulaPlaneRoutePhase::Planned,
20181 FormulaPlaneRouteTransitionReason::ProvenIsolated,
20182 cycle_demote_iters,
20183 runtime_replan_iterations,
20184 );
20185 if !island_plan.dirty_vertices.is_empty() {
20186 self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
20187 let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
20188 &island_plan.dirty_vertices,
20189 delta.as_deref_mut(),
20190 )?;
20191 computed_vertices = computed_vertices.saturating_add(island_computed);
20192 prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
20193 self.observe_formula_plane_route(
20194 &island_plan,
20195 FormulaPlaneRoutePhase::Executed,
20196 FormulaPlaneRouteTransitionReason::ProvenIsolated,
20197 cycle_demote_iters,
20198 runtime_replan_iterations,
20199 );
20200 self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
20201 }
20202 let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
20203 let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
20204
20205 #[cfg(test)]
20206 {
20207 self.last_formula_plane_span_eval_report = None;
20208 }
20209 let mut array_result_span: Option<FormulaSpanRef> = None;
20213 'layers: for layer in schedule.layers {
20214 self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
20215 let mut buffer = ComputedWriteBuffer::default();
20216 let mut sink = SpanComputedWriteSink::new(&mut buffer);
20217 let work_items = layer.work;
20218 let mut work_index = 0usize;
20219 while work_index < work_items.len() {
20220 match work_items[work_index].producer {
20221 FormulaProducerId::Span(span_id) => {
20222 let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
20223 ExcelError::new(ExcelErrorKind::NImpl)
20224 .with_message("FormulaPlane schedule referenced a stale span")
20225 })?;
20226 let sheet_id = {
20227 let authority = self.graph.formula_authority();
20228 let span =
20229 authority.plane.spans.get(span_ref).ok_or_else(|| {
20230 ExcelError::new(ExcelErrorKind::NImpl).with_message(
20231 "FormulaPlane schedule referenced a stale span",
20232 )
20233 })?;
20234 span.sheet_id
20235 };
20236 let current_sheet = self.graph.sheet_name(sheet_id);
20237 let authority = self.graph.formula_authority();
20238 let evaluator = SpanEvaluator::new_with_cancel(
20239 &authority.plane,
20240 self,
20241 current_sheet,
20242 self.graph.data_store(),
20243 self.graph.sheet_reg(),
20244 self.active_cancel_flag.as_ref(),
20245 );
20246 #[cfg(test)]
20247 let mut last_group_report = None;
20248 let mut selected_group_work = 0_u64;
20249 while work_index < work_items.len() {
20250 let FormulaProducerId::Span(group_span_id) =
20251 work_items[work_index].producer
20252 else {
20253 break;
20254 };
20255 let group_span_ref =
20256 *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
20257 ExcelError::new(ExcelErrorKind::NImpl).with_message(
20258 "FormulaPlane schedule referenced a stale span",
20259 )
20260 })?;
20261 let group_sheet_id = {
20262 let authority = self.graph.formula_authority();
20263 let span =
20264 authority.plane.spans.get(group_span_ref).ok_or_else(
20265 || {
20266 ExcelError::new(ExcelErrorKind::NImpl).with_message(
20267 "FormulaPlane schedule referenced a stale span",
20268 )
20269 },
20270 )?;
20271 span.sheet_id
20272 };
20273 if group_sheet_id != sheet_id {
20274 break;
20275 }
20276
20277 let dirty = producer_dirty_to_span_dirty(
20278 work_items[work_index].dirty.clone(),
20279 group_span_ref,
20280 );
20281 let task = SpanEvalTask {
20282 span: group_span_ref,
20283 dirty,
20284 plane_epoch,
20285 };
20286 self.cancellation_checkpoint(
20287 "Evaluation cancelled during FormulaPlane span",
20288 )?;
20289 let report = match evaluator.evaluate_task(&task, &mut sink) {
20290 Ok(report) => report,
20291 Err(
20296 crate::formula_plane::span_eval::SpanEvalError::ArrayResultRequiresSpill,
20297 ) => {
20298 array_result_span = Some(group_span_ref);
20299 break 'layers;
20300 }
20301 Err(
20302 crate::formula_plane::span_eval::SpanEvalError::Cancelled,
20303 ) => {
20304 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
20305 .with_message(
20306 "Evaluation cancelled during FormulaPlane span",
20307 ));
20308 }
20309 Err(err) => {
20310 return Err(ExcelError::new(ExcelErrorKind::NImpl)
20311 .with_message(format!(
20312 "FormulaPlane span evaluation failed: {err:?}"
20313 )));
20314 }
20315 };
20316 #[cfg(test)]
20317 {
20318 last_group_report = Some(report.clone());
20319 }
20320 selected_group_work = selected_group_work
20321 .saturating_add(report.span_eval_placement_count);
20322 computed_vertices = computed_vertices
20323 .saturating_add(report.span_eval_placement_count as usize);
20324 work_index = work_index.saturating_add(1);
20325 }
20326 self.charge_bounded_work(selected_group_work)?;
20329 #[cfg(test)]
20330 {
20331 if let Some(report) = last_group_report {
20332 self.last_formula_plane_span_eval_report = Some(report);
20333 }
20334 }
20335 }
20336 FormulaProducerId::Legacy(_) => {
20337 let mut vertices = Vec::new();
20346 while work_index < work_items.len() {
20347 let FormulaProducerId::Legacy(vertex_id) =
20348 work_items[work_index].producer
20349 else {
20350 break;
20351 };
20352 vertices.push(vertex_id);
20353 work_index = work_index.saturating_add(1);
20354 }
20355 let legacy_layer = crate::engine::scheduler::Layer { vertices };
20356 let evaluated = if let Some(delta) = delta.as_deref_mut() {
20357 if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
20358 self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
20359 } else {
20360 self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
20361 }
20362 } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
20363 {
20364 self.evaluate_layer_parallel(&legacy_layer)?
20365 } else {
20366 self.evaluate_layer_sequential(&legacy_layer)?
20367 };
20368 computed_vertices = computed_vertices.saturating_add(evaluated);
20369 }
20370 }
20371 }
20372 self.resource_checkpoint(0)?;
20373 self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
20374 if let Some(delta) = delta.as_deref_mut() {
20375 self.record_computed_write_buffer_delta(&buffer, delta);
20376 }
20377 self.flush_authoritative_computed_write_buffer(&mut buffer)?;
20378 }
20379
20380 if let Some(span_ref) = array_result_span {
20381 array_result_demotions = array_result_demotions.saturating_add(1);
20387 if array_result_demotions >= MAX_ARRAY_RESULT_DEMOTIONS {
20388 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
20389 "FormulaPlane array-result span demotion did not converge".to_string(),
20390 ));
20391 }
20392 self.demote_array_result_span(span_ref)?;
20393 if self.graph.formula_authority().active_span_count() == 0 {
20394 if let Some(target_roots) = runtime_target_roots.as_deref() {
20395 let refreshed =
20396 self.resolve_target_producers_from_existing_roots(target_roots)?;
20397 let result =
20398 self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
20399 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
20400 return Ok(result);
20401 }
20402 return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
20403 }
20404 include_dirty_regions = true;
20405 continue 'virtual_replan;
20406 }
20407
20408 let mut changed_virtual =
20409 self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
20410 let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
20411 let mut dynamic_candidates = old_dynamic_regions
20412 .keys()
20413 .chain(new_dynamic_regions.keys())
20414 .copied()
20415 .collect::<FxHashSet<_>>();
20416 for vertex in dynamic_candidates.drain() {
20417 if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
20418 && !changed_virtual.contains(&vertex)
20419 {
20420 changed_virtual.push(vertex);
20421 }
20422 }
20423 if let Some(telemetry) = virtual_telemetry.as_mut() {
20424 telemetry.changed_vdeps_total = telemetry
20425 .changed_vdeps_total
20426 .saturating_add(changed_virtual.len());
20427 }
20428 self.resource_checkpoint(0)?;
20429 self.graph.clear_dirty_flags(&legacy_vertices);
20430 if !changed_virtual.is_empty() {
20431 for vertex in &changed_virtual {
20432 self.graph.set_dirty(*vertex, true);
20433 }
20434 let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
20435 let can_attempt_workbook =
20436 !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
20437 if can_replan || can_attempt_workbook {
20438 if can_replan {
20439 if let Some(roots) = runtime_target_roots.as_mut() {
20440 self.extend_target_roots_with_dynamic_regions(
20441 roots,
20442 old_dynamic_regions
20443 .values()
20444 .flatten()
20445 .chain(new_dynamic_regions.values().flatten()),
20446 )?;
20447 }
20448 runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
20449 } else {
20450 runtime_target_roots = None;
20451 workbook_exact_attempted = true;
20452 }
20453 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20454 if can_replan {
20455 stats.runtime_replan_rounds =
20456 stats.runtime_replan_rounds.saturating_add(1);
20457 }
20458 stats.runtime_widening_rounds =
20459 stats.runtime_widening_rounds.saturating_add(1);
20460 if can_attempt_workbook {
20461 stats.workbook_exact_attempts = 1;
20462 }
20463 }
20464 self.cached_mixed_topology = None;
20465 if let Some(ledger) = self.active_resource_ledger.as_mut() {
20466 ledger
20467 .account_mixed_cache(0)
20468 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20469 }
20470 include_dirty_regions = true;
20471 continue 'virtual_replan;
20472 }
20473 if let Some(telemetry) = virtual_telemetry.as_mut() {
20474 telemetry.replan_iterations = runtime_replan_iterations;
20475 telemetry.bailout_reason = Some("max_replan");
20476 self.last_virtual_dep_telemetry = telemetry.clone();
20477 }
20478 return Err(self.replan_exhausted_error(
20479 MAX_RUNTIME_REPLAN,
20480 "mixed dynamic dependency evaluation",
20481 ));
20482 }
20483 if let Some(mut telemetry) = virtual_telemetry {
20484 telemetry.replan_iterations = runtime_replan_iterations;
20485 telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
20486 "converged"
20487 } else {
20488 "max_replan"
20489 });
20490 self.last_virtual_dep_telemetry = telemetry;
20491 }
20492 self.redirty_for_next_recalc();
20496 self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
20497 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20498 return Ok(EvalResult {
20499 computed_vertices,
20500 cycle_errors: prepass_cycle_errors,
20501 elapsed: start.elapsed(),
20502 });
20503 }
20504 }
20505
20506 fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
20507 let budgets = &self.evaluation_resource_budgets;
20508 let candidate_limit = budgets
20509 .optimization
20510 .mixed_cache_candidates
20511 .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
20512 limit.min(self.config.max_formula_plane_cache_candidates)
20513 });
20514 let edge_limit = budgets
20515 .optimization
20516 .mixed_cache_edges
20517 .map_or(self.config.max_formula_plane_cache_edges, |limit| {
20518 limit.min(self.config.max_formula_plane_cache_edges)
20519 });
20520 let mut byte_limit =
20521 u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
20522 if let Some(limit) = budgets.retained.total_bytes {
20523 byte_limit = byte_limit.min(limit);
20524 }
20525 if let Some(limit) = budgets.retained.mixed_cache_bytes {
20526 byte_limit = byte_limit.min(limit);
20527 }
20528 (
20529 candidate_limit,
20530 edge_limit,
20531 usize::try_from(byte_limit).unwrap_or(usize::MAX),
20532 )
20533 }
20534
20535 fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
20536 let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
20537 MixedTopologyCacheKey {
20538 engine_topology_epoch: self.topology_epoch,
20539 graph_topology_revision: self.graph.topology_revision(),
20540 authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
20541 legacy_island_revision: self.graph.topology_revision(),
20542 boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
20543 function_semantic_epoch: self.function_semantic_epoch_seen,
20544 function_provider_revision: self.function_provider_revision_seen,
20545 max_candidates,
20546 max_edges,
20547 max_memory_bytes,
20548 }
20549 }
20550
20551 fn mixed_topology_index_preflight_bytes(&self) -> u64 {
20552 use crate::formula_plane::region_index::AxisRange;
20553
20554 const FIXED_BYTES: u64 = 8 * 1024;
20555 const ENTRY_BYTES: u64 = 512;
20556 const INDEX_UNIT_BYTES: u64 = 1024;
20557 const BINDING_BYTES: u64 = 128;
20558 let index_units = |region: Region| -> u64 {
20559 match region.normalized().axis_ranges() {
20560 (AxisRange::Point(_), AxisRange::Point(_)) => 2,
20561 (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
20562 let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
20563 let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
20564 row_buckets.saturating_mul(col_buckets).saturating_mul(2)
20565 }
20566 _ => 1,
20567 }
20568 };
20569
20570 let authority = self.graph.formula_authority();
20571 let span_refs = authority.active_span_refs();
20572 let legacy_vertices = self.graph.formula_vertices();
20573 let mut entry_count = 0_u64;
20574 let mut units = 0_u64;
20575 for span_ref in &span_refs {
20576 if let Some(span) = authority.plane.spans.get(*span_ref) {
20577 let result_region = Region::from_domain(span.result_region.domain());
20578 entry_count = entry_count.saturating_add(1);
20579 units = units.saturating_add(index_units(result_region));
20580 if let Some(summary_id) = span.read_summary_id
20581 && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
20582 {
20583 for dependency in &summary.dependencies {
20584 entry_count = entry_count.saturating_add(1);
20585 units = units.saturating_add(index_units(dependency.read_region));
20586 }
20587 }
20588 }
20589 }
20590 for &vertex in &legacy_vertices {
20591 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
20592 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
20593 entry_count = entry_count.saturating_add(1);
20594 units = units.saturating_add(index_units(point));
20595 }
20596 for dependency in self.graph.get_dependencies(vertex) {
20597 if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
20598 let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
20599 entry_count = entry_count.saturating_add(1);
20600 units = units.saturating_add(index_units(point));
20601 }
20602 }
20603 let vertex_sheet = self.graph.get_vertex_sheet_id(vertex);
20604 if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
20605 for range in ranges {
20606 if let Ok(Some(region)) =
20607 self.shared_range_to_region_pattern(range, vertex_sheet)
20608 {
20609 entry_count = entry_count.saturating_add(1);
20610 units = units.saturating_add(index_units(region));
20611 }
20612 }
20613 }
20614 }
20615 FIXED_BYTES
20616 .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
20617 .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
20618 .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
20619 }
20620
20621 fn prove_sheet_disjoint_legacy_island(
20622 &self,
20623 span_results: &FormulaProducerResultIndex,
20624 span_reads: &FormulaConsumerReadIndex,
20625 legacy_vertices: &[VertexId],
20626 ) -> Option<LegacyIslandPlan> {
20627 if !self.legacy_island_structural_summaries_trusted
20628 || self.graph.named_ranges_iter().next().is_some()
20629 || self.graph.sheet_named_ranges_iter().next().is_some()
20630 || self.graph.spill_registry_counts() != (0, 0)
20631 || legacy_vertices.iter().copied().any(|vertex| {
20632 self.graph.is_dynamic(vertex)
20633 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
20634 })
20635 {
20636 return None;
20637 }
20638 let span_result_sheets = span_results
20639 .entries()
20640 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
20641 .map(|entry| entry.result_region.sheet_id())
20642 .collect::<FxHashSet<_>>();
20643 let span_read_sheets = span_reads
20644 .entries()
20645 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
20646 .map(|entry| entry.read_region.sheet_id())
20647 .collect::<FxHashSet<_>>();
20648
20649 let span_boundary_sheets = span_result_sheets
20650 .union(&span_read_sheets)
20651 .copied()
20652 .collect::<FxHashSet<_>>();
20653 for vertex in legacy_vertices {
20654 let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
20655 if span_boundary_sheets.contains(&cell.sheet_id) {
20656 return None;
20657 }
20658 for dependency in self.graph.get_dependencies(*vertex) {
20659 let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
20660 if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
20661 return None;
20662 }
20663 }
20664 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
20665 for range in ranges {
20666 let region = self
20667 .shared_range_to_region_pattern(range, cell.sheet_id)
20668 .ok()??;
20669 if span_boundary_sheets.contains(®ion.sheet_id()) {
20670 return None;
20671 }
20672 }
20673 }
20674 }
20675
20676 let mut membership = legacy_vertices.to_vec();
20677 membership.sort_unstable();
20678 if membership.is_empty() {
20679 return None;
20680 }
20681 let mut sheet_ids = membership
20682 .iter()
20683 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
20684 .map(|cell| cell.sheet_id)
20685 .collect::<Vec<_>>();
20686 sheet_ids.sort_unstable();
20687 sheet_ids.dedup();
20688 let island_id = membership
20689 .iter()
20690 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
20691 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
20692 });
20693 let retained_bytes = membership
20694 .capacity()
20695 .saturating_mul(std::mem::size_of::<VertexId>())
20696 .saturating_add(
20697 sheet_ids
20698 .capacity()
20699 .saturating_mul(std::mem::size_of::<SheetId>()),
20700 );
20701 Some(LegacyIslandPlan {
20702 membership,
20703 dirty_vertices: Vec::new(),
20704 sheet_ids,
20705 island_id,
20706 retained_bytes,
20707 boundary_relationships: 0,
20708 omitted_relationships: legacy_vertices.len(),
20709 omitted_direct_relationships: 0,
20710 })
20711 }
20712
20713 fn contract_legacy_islands(
20714 &self,
20715 full_results: &FormulaProducerResultIndex,
20716 full_reads: &FormulaConsumerReadIndex,
20717 legacy_reads_complete: bool,
20718 candidate_cap: usize,
20719 ) -> Option<(
20720 FormulaProducerResultIndex,
20721 FormulaConsumerReadIndex,
20722 LegacyIslandPlan,
20723 )> {
20724 use crate::formula_plane::producer::ProjectionResult;
20725 use crate::formula_plane::region_index::BoundedRegionQueryResult;
20726
20727 if !self.legacy_island_structural_summaries_trusted
20728 || !legacy_reads_complete
20729 || self.graph.named_ranges_iter().next().is_some()
20730 || self.graph.sheet_named_ranges_iter().next().is_some()
20731 || self.graph.spill_registry_counts() != (0, 0)
20732 {
20733 return None;
20734 }
20735 let legacy_vertices = self.graph.formula_vertices();
20736 if legacy_vertices.iter().copied().any(|vertex| {
20737 self.graph.is_dynamic(vertex)
20738 || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
20739 }) {
20740 return None;
20741 }
20742
20743 let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
20744 for entry in full_reads.entries() {
20745 if let FormulaProducerId::Legacy(vertex) = entry.consumer {
20746 reads_by_consumer
20747 .entry(vertex)
20748 .or_default()
20749 .push(entry.read_region);
20750 }
20751 }
20752
20753 let mut connected = FxHashSet::default();
20754 let mut queue = VecDeque::new();
20755 let mut observed_candidates = 0usize;
20756 let mut boundary_relationships = 0usize;
20757 let add_connected = |vertex: VertexId,
20758 connected: &mut FxHashSet<VertexId>,
20759 queue: &mut VecDeque<VertexId>| {
20760 if connected.insert(vertex) {
20761 queue.push_back(vertex);
20762 }
20763 };
20764
20765 for result in full_results.entries() {
20767 if !matches!(result.producer, FormulaProducerId::Span(_)) {
20768 continue;
20769 }
20770 let remaining = candidate_cap.saturating_sub(observed_candidates);
20771 let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
20772 let BoundedRegionQueryResult::Complete(query) = query else {
20773 return None;
20774 };
20775 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
20776 for matched in query.matches {
20777 if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
20778 match matched.value.dirty {
20779 ProjectionResult::NoIntersection => {}
20780 ProjectionResult::Unsupported(_) => return None,
20781 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
20782 boundary_relationships = boundary_relationships.saturating_add(1);
20783 add_connected(vertex, &mut connected, &mut queue);
20784 }
20785 }
20786 }
20787 }
20788 }
20789
20790 for read in full_reads.entries() {
20792 if !matches!(read.consumer, FormulaProducerId::Span(_)) {
20793 continue;
20794 }
20795 let remaining = candidate_cap.saturating_sub(observed_candidates);
20796 let query = full_results.query_bounded(read.read_region, remaining);
20797 let BoundedRegionQueryResult::Complete(query) = query else {
20798 return None;
20799 };
20800 observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
20801 for matched in query.matches {
20802 if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
20803 boundary_relationships = boundary_relationships.saturating_add(1);
20804 add_connected(vertex, &mut connected, &mut queue);
20805 }
20806 }
20807 }
20808
20809 while let Some(vertex) = queue.pop_front() {
20813 let producer = FormulaProducerId::Legacy(vertex);
20814 if let Some(result_region) = full_results.producer_result_region(producer) {
20815 let remaining = candidate_cap.saturating_sub(observed_candidates);
20816 let query = full_reads.query_changed_region_bounded(result_region, remaining);
20817 let BoundedRegionQueryResult::Complete(query) = query else {
20818 return None;
20819 };
20820 observed_candidates =
20821 observed_candidates.saturating_add(query.stats.candidate_count);
20822 for matched in query.matches {
20823 if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
20824 match matched.value.dirty {
20825 ProjectionResult::NoIntersection => {}
20826 ProjectionResult::Unsupported(_) => return None,
20827 ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
20828 add_connected(consumer, &mut connected, &mut queue);
20829 }
20830 }
20831 }
20832 }
20833 }
20834 for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
20835 let remaining = candidate_cap.saturating_sub(observed_candidates);
20836 let query = full_results.query_bounded(*read_region, remaining);
20837 let BoundedRegionQueryResult::Complete(query) = query else {
20838 return None;
20839 };
20840 observed_candidates =
20841 observed_candidates.saturating_add(query.stats.candidate_count);
20842 for matched in query.matches {
20843 if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
20844 add_connected(precedent, &mut connected, &mut queue);
20845 }
20846 }
20847 }
20848 }
20849
20850 let mut membership = legacy_vertices
20851 .iter()
20852 .copied()
20853 .filter(|vertex| !connected.contains(vertex))
20854 .collect::<Vec<_>>();
20855 membership.sort_unstable();
20856 if membership.is_empty() {
20857 return None;
20858 }
20859
20860 let mut producer_results = FormulaProducerResultIndex::default();
20861 let mut consumer_reads = FormulaConsumerReadIndex::default();
20862 let retained_span_results = full_results
20863 .entries()
20864 .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
20865 .count();
20866 producer_results
20867 .try_reserve(retained_span_results.saturating_add(connected.len()))
20868 .ok()?;
20869 for entry in full_results.entries() {
20870 let retain = match entry.producer {
20871 FormulaProducerId::Span(_) => true,
20872 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
20873 };
20874 if retain {
20875 producer_results.insert_producer(entry.producer, entry.result_region);
20876 }
20877 }
20878 for entry in full_reads.entries() {
20879 let retain = match entry.consumer {
20880 FormulaProducerId::Span(_) => true,
20881 FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
20882 };
20883 if retain {
20884 consumer_reads.try_reserve(1).ok()?;
20885 consumer_reads.insert_read(
20886 entry.consumer,
20887 entry.read_region,
20888 entry.consumer_result_region,
20889 entry.projection,
20890 );
20891 }
20892 }
20893
20894 let mut sheet_ids = membership
20895 .iter()
20896 .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
20897 .map(|cell| cell.sheet_id)
20898 .collect::<Vec<_>>();
20899 sheet_ids.sort_unstable();
20900 sheet_ids.dedup();
20901 let island_id = membership
20902 .iter()
20903 .fold(0xcbf29ce484222325_u64, |hash, vertex| {
20904 (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
20905 });
20906 let retained_bytes = membership
20907 .capacity()
20908 .saturating_mul(std::mem::size_of::<VertexId>())
20909 .saturating_add(
20910 sheet_ids
20911 .capacity()
20912 .saturating_mul(std::mem::size_of::<SheetId>()),
20913 );
20914 let omitted_relationships = full_reads
20915 .entries()
20916 .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
20917 .count();
20918 let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
20919 let omitted_direct_relationships = membership
20920 .iter()
20921 .map(|vertex| {
20922 self.graph
20923 .get_dependencies(*vertex)
20924 .into_iter()
20925 .filter(|dependency| membership_set.contains(dependency))
20926 .count()
20927 })
20928 .sum();
20929 Some((
20930 producer_results,
20931 consumer_reads,
20932 LegacyIslandPlan {
20933 membership,
20934 dirty_vertices: Vec::new(),
20935 sheet_ids,
20936 island_id,
20937 retained_bytes,
20938 boundary_relationships,
20939 omitted_relationships,
20940 omitted_direct_relationships,
20941 },
20942 ))
20943 }
20944
20945 fn compile_formula_plane_mixed_topology(
20946 &mut self,
20947 key: MixedTopologyCacheKey,
20948 ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
20949 #[cfg(test)]
20950 {
20951 self.mixed_topology_index_builds_for_test =
20952 self.mixed_topology_index_builds_for_test.saturating_add(1);
20953 }
20954 let authority = self.graph.formula_authority();
20955 let mut producer_results = FormulaProducerResultIndex::default();
20956 let mut consumer_reads = FormulaConsumerReadIndex::default();
20957 let mut legacy_reads_complete = true;
20958 let span_refs = authority.active_span_refs();
20959 let legacy_vertices = self.graph.formula_vertices();
20960 let _producer_count = span_refs
20961 .len()
20962 .checked_add(legacy_vertices.len())
20963 .ok_or_else(|| {
20964 ExcelError::new(ExcelErrorKind::NImpl)
20965 .with_message("FormulaPlane cache producer count overflow")
20966 })?;
20967 producer_results.try_reserve(_producer_count).map_err(|_| {
20968 ExcelError::new(ExcelErrorKind::NImpl)
20969 .with_message("FormulaPlane cache producer index reservation failed")
20970 })?;
20971 let span_refs_by_id = span_refs
20972 .iter()
20973 .copied()
20974 .map(|span_ref| (span_ref.id, span_ref))
20975 .collect::<BTreeMap<_, _>>();
20976 for span_ref in &span_refs {
20977 let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
20978 ExcelError::new(ExcelErrorKind::NImpl)
20979 .with_message("FormulaPlane active span ref is stale")
20980 })?;
20981 let result_region = Region::from_domain(span.result_region.domain());
20982 producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
20983 let Some(read_summary_id) = span.read_summary_id else {
20984 return Err(ExcelError::new(ExcelErrorKind::NImpl)
20985 .with_message("FormulaPlane active span is missing read summary"));
20986 };
20987 let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
20988 else {
20989 return Err(ExcelError::new(ExcelErrorKind::NImpl)
20990 .with_message("FormulaPlane active span has stale read summary"));
20991 };
20992 if read_summary.result_region != result_region {
20993 return Err(ExcelError::new(ExcelErrorKind::NImpl)
20994 .with_message("FormulaPlane active span read summary is stale"));
20995 }
20996 for dependency in &read_summary.dependencies {
20997 consumer_reads.try_reserve(1).map_err(|_| {
20998 ExcelError::new(ExcelErrorKind::NImpl)
20999 .with_message("FormulaPlane cache read-index reservation failed")
21000 })?;
21001 consumer_reads.insert_read(
21002 FormulaProducerId::Span(span.id),
21003 dependency.read_region,
21004 read_summary.result_region,
21005 dependency.projection,
21006 );
21007 }
21008 }
21009
21010 let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
21011 &producer_results,
21012 &consumer_reads,
21013 &legacy_vertices,
21014 );
21015 if sheet_disjoint_island.is_none() {
21016 for vertex in &legacy_vertices {
21017 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
21018 continue;
21019 };
21020 producer_results.insert_producer(
21021 FormulaProducerId::Legacy(*vertex),
21022 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
21023 );
21024 }
21025 for vertex in &legacy_vertices {
21026 let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
21027 continue;
21028 };
21029 let result_region =
21030 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21031 let mut seen = rustc_hash::FxHashSet::default();
21032 for dep in self.graph.get_dependencies(*vertex) {
21033 let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
21034 legacy_reads_complete = false;
21037 continue;
21038 };
21039 let read_region = Region::point(
21040 dep_cell.sheet_id,
21041 dep_cell.coord.row(),
21042 dep_cell.coord.col(),
21043 );
21044 if seen.insert(read_region) {
21045 consumer_reads.try_reserve(1).map_err(|_| {
21046 ExcelError::new(ExcelErrorKind::NImpl)
21047 .with_message("FormulaPlane cache read-index reservation failed")
21048 })?;
21049 consumer_reads.insert_read(
21050 FormulaProducerId::Legacy(*vertex),
21051 read_region,
21052 result_region,
21053 DirtyProjectionRule::WholeResult,
21054 );
21055 }
21056 }
21057 if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
21058 for range in ranges {
21059 let Some(read_region) =
21060 self.shared_range_to_region_pattern(range, cell.sheet_id)?
21061 else {
21062 legacy_reads_complete = false;
21063 continue;
21064 };
21065 if seen.insert(read_region) {
21066 consumer_reads.try_reserve(1).map_err(|_| {
21067 ExcelError::new(ExcelErrorKind::NImpl).with_message(
21068 "FormulaPlane cache read-index reservation failed",
21069 )
21070 })?;
21071 consumer_reads.insert_read(
21072 FormulaProducerId::Legacy(*vertex),
21073 read_region,
21074 result_region,
21075 DirtyProjectionRule::WholeResult,
21076 );
21077 }
21078 }
21079 }
21080 let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
21081 for dependency in virtual_dependencies
21082 .get(vertex)
21083 .into_iter()
21084 .flatten()
21085 .copied()
21086 {
21087 let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
21088 continue;
21089 };
21090 let read_region =
21091 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21092 if seen.insert(read_region) {
21093 consumer_reads.try_reserve(1).map_err(|_| {
21094 ExcelError::new(ExcelErrorKind::NImpl)
21095 .with_message("FormulaPlane cache virtual read reservation failed")
21096 })?;
21097 consumer_reads.insert_read(
21098 FormulaProducerId::Legacy(*vertex),
21099 read_region,
21100 result_region,
21101 DirtyProjectionRule::WholeResult,
21102 );
21103 }
21104 }
21105 for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
21106 {
21107 if seen.insert(read_region) {
21108 consumer_reads.try_reserve(1).map_err(|_| {
21109 ExcelError::new(ExcelErrorKind::NImpl)
21110 .with_message("FormulaPlane dynamic-region reservation failed")
21111 })?;
21112 consumer_reads.insert_read(
21113 FormulaProducerId::Legacy(*vertex),
21114 read_region,
21115 result_region,
21116 DirtyProjectionRule::WholeResult,
21117 );
21118 }
21119 }
21120 }
21121 }
21122
21123 let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
21127 {
21128 (producer_results, consumer_reads, island)
21129 } else {
21130 self.contract_legacy_islands(
21131 &producer_results,
21132 &consumer_reads,
21133 legacy_reads_complete,
21134 key.max_candidates,
21135 )
21136 .unwrap_or((
21137 producer_results,
21138 consumer_reads,
21139 LegacyIslandPlan::default(),
21140 ))
21141 };
21142
21143 let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
21144 .checked_add(
21145 producer_results
21146 .estimated_memory_bytes()
21147 .unwrap_or(usize::MAX),
21148 )
21149 .and_then(|bytes| {
21150 bytes.checked_add(
21151 consumer_reads
21152 .estimated_memory_bytes()
21153 .unwrap_or(usize::MAX),
21154 )
21155 })
21156 .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
21157 .and_then(|bytes| bytes.checked_add(island.retained_bytes))
21158 .unwrap_or(usize::MAX);
21159 let config = MixedTopologyConfig {
21160 max_candidates: key.max_candidates,
21161 max_edges: key.max_edges,
21162 max_memory_bytes: key.max_memory_bytes,
21163 retained_memory_bytes,
21164 };
21165 let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
21166 match &mut compiled {
21170 MixedTopologyCompileResult::Cached(topology) => {
21171 topology.stats.producers = topology
21172 .stats
21173 .producers
21174 .saturating_add(island.membership.len());
21175 topology.stats.candidates = topology
21176 .stats
21177 .candidates
21178 .saturating_add(island.omitted_direct_relationships);
21179 topology.stats.relationships = topology
21180 .stats
21181 .relationships
21182 .saturating_add(island.omitted_direct_relationships);
21183 }
21184 MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
21185 observed.producers = observed.producers.saturating_add(island.membership.len());
21186 observed.candidates = observed
21187 .candidates
21188 .saturating_add(island.omitted_direct_relationships);
21189 observed.relationships = observed
21190 .relationships
21191 .saturating_add(island.omitted_direct_relationships);
21192 }
21193 }
21194 let plane_epoch = authority.plane.epoch().0;
21195 Ok(match compiled {
21196 MixedTopologyCompileResult::Cached(topology) => {
21197 FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
21198 key,
21199 topology,
21200 producer_results,
21201 consumer_reads,
21202 span_refs_by_id,
21203 plane_epoch,
21204 island,
21205 })
21206 }
21207 MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
21208 FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
21209 reason,
21210 observed,
21211 producer_results,
21212 consumer_reads,
21213 span_refs_by_id,
21214 plane_epoch,
21215 island,
21216 })
21217 }
21218 })
21219 }
21220
21221 fn include_non_grid_legacy_work(
21222 &self,
21223 schedule: MixedSchedule,
21224 vertices: &[VertexId],
21225 initial_fallbacks: &[MixedScheduleFallback],
21226 producer_results: &FormulaProducerResultIndex,
21227 consumer_reads: &FormulaConsumerReadIndex,
21228 max_candidates: usize,
21229 max_edges: usize,
21230 ) -> MixedSchedule {
21231 if vertices.is_empty() && initial_fallbacks.is_empty() {
21232 return schedule;
21233 }
21234
21235 let mut candidate_count = 0usize;
21238 let mut preflight_fallbacks = initial_fallbacks.to_vec();
21239 for &vertex in vertices {
21240 let producer = FormulaProducerId::Legacy(vertex);
21241 let named_formula_is_dynamic =
21242 self.graph
21243 .named_range_by_vertex(vertex)
21244 .is_some_and(|named| match &named.definition {
21245 crate::engine::named_range::NamedDefinition::Formula { ast, .. } => {
21246 self.graph.is_ast_dynamic(ast)
21247 }
21248 _ => false,
21249 });
21250 if self.graph.is_dynamic(vertex) || named_formula_is_dynamic {
21251 preflight_fallbacks.push(MixedScheduleFallback {
21252 producer,
21253 reason: MixedScheduleFallbackReason::UnsupportedProjection,
21254 });
21255 continue;
21256 }
21257
21258 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
21259 for range in self
21260 .graph
21261 .get_range_dependencies(vertex)
21262 .into_iter()
21263 .flatten()
21264 {
21265 let Ok(Some(region)) = self.shared_range_to_region_pattern(range, context_sheet)
21266 else {
21267 preflight_fallbacks.push(MixedScheduleFallback {
21268 producer,
21269 reason: MixedScheduleFallbackReason::UnsupportedProjection,
21270 });
21271 break;
21272 };
21273
21274 let (rows, cols) = region.axis_ranges();
21275 let (row_start, row_end) = rows.query_bounds();
21276 let (col_start, col_end) = cols.query_bounds();
21277 if !self
21278 .graph
21279 .spill_anchors_in_region(
21280 region.sheet_id(),
21281 row_start,
21282 col_start,
21283 row_end,
21284 col_end,
21285 )
21286 .is_empty()
21287 {
21288 preflight_fallbacks.push(MixedScheduleFallback {
21289 producer,
21290 reason: MixedScheduleFallbackReason::UnsupportedProjection,
21291 });
21292 break;
21293 }
21294
21295 let remaining = max_candidates.saturating_sub(candidate_count);
21296 match producer_results.query_bounded(region, remaining) {
21297 BoundedRegionQueryResult::Incomplete { .. } => {
21298 preflight_fallbacks.push(MixedScheduleFallback {
21299 producer,
21300 reason: MixedScheduleFallbackReason::MaxCandidatesExceeded,
21301 });
21302 break;
21303 }
21304 BoundedRegionQueryResult::Complete(query) => {
21305 candidate_count = candidate_count.saturating_add(query.matches.len());
21306 if query
21307 .matches
21308 .iter()
21309 .any(|matched| matched.value.result_region.as_point().is_none())
21310 {
21311 preflight_fallbacks.push(MixedScheduleFallback {
21312 producer,
21313 reason: MixedScheduleFallbackReason::UnsupportedProjection,
21314 });
21315 break;
21316 }
21317 }
21318 }
21319 }
21320 }
21321
21322 if !preflight_fallbacks.is_empty() {
21323 let mut schedule = schedule;
21324 schedule.stats.input_work_items = schedule
21325 .stats
21326 .input_work_items
21327 .saturating_add(vertices.len())
21328 .saturating_add(initial_fallbacks.len());
21329 schedule.stats.consumer_candidate_count = schedule
21330 .stats
21331 .consumer_candidate_count
21332 .saturating_add(candidate_count);
21333 schedule.stats.max_candidates_exceeded_count =
21334 schedule.stats.max_candidates_exceeded_count.saturating_add(
21335 preflight_fallbacks
21336 .iter()
21337 .filter(|fallback| {
21338 fallback.reason == MixedScheduleFallbackReason::MaxCandidatesExceeded
21339 })
21340 .count(),
21341 );
21342 schedule.fallbacks.extend(preflight_fallbacks);
21343 return schedule;
21344 }
21345
21346 let mut work_by_producer = BTreeMap::new();
21347 for item in schedule
21348 .layers
21349 .iter()
21350 .flat_map(|layer| layer.work.iter())
21351 .cloned()
21352 {
21353 work_by_producer.insert(item.producer, item);
21354 }
21355 for &vertex in vertices {
21356 let producer = FormulaProducerId::Legacy(vertex);
21357 work_by_producer
21358 .entry(producer)
21359 .or_insert(FormulaProducerWork {
21360 producer,
21361 dirty: ProducerDirtyDomain::Whole,
21362 });
21363 }
21364
21365 if work_by_producer.len() > max_candidates {
21366 let mut schedule = schedule;
21367 schedule.stats.input_work_items = schedule
21368 .stats
21369 .input_work_items
21370 .saturating_add(vertices.len());
21371 schedule.stats.max_candidates_exceeded_count = schedule
21372 .stats
21373 .max_candidates_exceeded_count
21374 .saturating_add(1);
21375 schedule.fallbacks.push(MixedScheduleFallback {
21376 producer: FormulaProducerId::Legacy(vertices[0]),
21377 reason: MixedScheduleFallbackReason::MaxCandidatesExceeded,
21378 });
21379 return schedule;
21380 }
21381
21382 let producer_set = work_by_producer.keys().copied().collect::<FxHashSet<_>>();
21383 let mut outgoing = BTreeMap::<FormulaProducerId, BTreeSet<FormulaProducerId>>::new();
21384 let mut indegree = work_by_producer
21385 .keys()
21386 .copied()
21387 .map(|producer| (producer, 0usize))
21388 .collect::<BTreeMap<_, _>>();
21389 let mut derived_edges = 0usize;
21390 let mut duplicate_edges = 0usize;
21391 let try_add_edge = |source: FormulaProducerId,
21392 target: FormulaProducerId,
21393 outgoing: &mut BTreeMap<_, BTreeSet<_>>,
21394 indegree: &mut BTreeMap<_, usize>,
21395 derived_edges: &mut usize,
21396 duplicate_edges: &mut usize|
21397 -> bool {
21398 if source == target
21399 || !producer_set.contains(&source)
21400 || !producer_set.contains(&target)
21401 {
21402 return true;
21403 }
21404 if outgoing
21405 .get(&source)
21406 .is_some_and(|targets| targets.contains(&target))
21407 {
21408 *duplicate_edges = duplicate_edges.saturating_add(1);
21409 return true;
21410 }
21411 if *derived_edges >= max_edges {
21412 return false;
21413 }
21414 outgoing.entry(source).or_default().insert(target);
21415 *indegree.entry(target).or_default() += 1;
21416 *derived_edges = derived_edges.saturating_add(1);
21417 true
21418 };
21419
21420 let mut overflow_reason = None;
21421 'regional_edges: for &source in &producer_set {
21422 let Some(result_region) = producer_results.producer_result_region(source) else {
21423 continue;
21424 };
21425 let remaining = max_candidates.saturating_sub(candidate_count);
21426 let query = match consumer_reads.query_changed_region_bounded(result_region, remaining)
21427 {
21428 BoundedRegionQueryResult::Incomplete { .. } => {
21429 overflow_reason = Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21430 break 'regional_edges;
21431 }
21432 BoundedRegionQueryResult::Complete(query) => query,
21433 };
21434 candidate_count = candidate_count.saturating_add(query.matches.len());
21435 for matched in query.matches {
21436 if !matches!(matched.value.dirty, ProjectionResult::NoIntersection)
21437 && !try_add_edge(
21438 source,
21439 matched.value.consumer,
21440 &mut outgoing,
21441 &mut indegree,
21442 &mut derived_edges,
21443 &mut duplicate_edges,
21444 )
21445 {
21446 overflow_reason = Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21447 break 'regional_edges;
21448 }
21449 }
21450 }
21451
21452 if overflow_reason.is_none() {
21453 'legacy_edges: for &consumer in &producer_set {
21454 let FormulaProducerId::Legacy(vertex) = consumer else {
21455 continue;
21456 };
21457 for dependency in self.graph.get_dependencies(vertex) {
21458 let direct = FormulaProducerId::Legacy(dependency);
21459 if !try_add_edge(
21460 direct,
21461 consumer,
21462 &mut outgoing,
21463 &mut indegree,
21464 &mut derived_edges,
21465 &mut duplicate_edges,
21466 ) {
21467 overflow_reason = Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21468 break 'legacy_edges;
21469 }
21470
21471 if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
21472 let point =
21473 Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21474 let remaining = max_candidates.saturating_sub(candidate_count);
21475 let query = match producer_results.query_bounded(point, remaining) {
21476 BoundedRegionQueryResult::Incomplete { .. } => {
21477 overflow_reason =
21478 Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21479 break 'legacy_edges;
21480 }
21481 BoundedRegionQueryResult::Complete(query) => query,
21482 };
21483 candidate_count = candidate_count.saturating_add(query.matches.len());
21484 for matched in query.matches {
21485 if !try_add_edge(
21486 matched.value.producer,
21487 consumer,
21488 &mut outgoing,
21489 &mut indegree,
21490 &mut derived_edges,
21491 &mut duplicate_edges,
21492 ) {
21493 overflow_reason =
21494 Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21495 break 'legacy_edges;
21496 }
21497 }
21498 }
21499 }
21500 let context_sheet = self.graph.get_vertex_sheet_id(vertex);
21501 for range in self
21502 .graph
21503 .get_range_dependencies(vertex)
21504 .into_iter()
21505 .flatten()
21506 {
21507 if let Ok(Some(region)) =
21508 self.shared_range_to_region_pattern(range, context_sheet)
21509 {
21510 let remaining = max_candidates.saturating_sub(candidate_count);
21511 let query = match producer_results.query_bounded(region, remaining) {
21512 BoundedRegionQueryResult::Incomplete { .. } => {
21513 overflow_reason =
21514 Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21515 break 'legacy_edges;
21516 }
21517 BoundedRegionQueryResult::Complete(query) => query,
21518 };
21519 candidate_count = candidate_count.saturating_add(query.matches.len());
21520 for matched in query.matches {
21521 if !try_add_edge(
21522 matched.value.producer,
21523 consumer,
21524 &mut outgoing,
21525 &mut indegree,
21526 &mut derived_edges,
21527 &mut duplicate_edges,
21528 ) {
21529 overflow_reason =
21530 Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21531 break 'legacy_edges;
21532 }
21533 }
21534 }
21535 }
21536 }
21537 }
21538
21539 if let Some(reason) = overflow_reason {
21540 let mut schedule = schedule;
21541 schedule.stats.input_work_items = schedule
21542 .stats
21543 .input_work_items
21544 .saturating_add(vertices.len());
21545 schedule.stats.consumer_candidate_count = schedule
21546 .stats
21547 .consumer_candidate_count
21548 .saturating_add(candidate_count);
21549 schedule.stats.duplicate_edges_skipped = schedule
21550 .stats
21551 .duplicate_edges_skipped
21552 .saturating_add(duplicate_edges);
21553 match reason {
21554 MixedScheduleFallbackReason::MaxEdgesExceeded => {
21555 schedule.stats.max_edges_exceeded_count =
21556 schedule.stats.max_edges_exceeded_count.saturating_add(1);
21557 }
21558 MixedScheduleFallbackReason::MaxCandidatesExceeded => {
21559 schedule.stats.max_candidates_exceeded_count = schedule
21560 .stats
21561 .max_candidates_exceeded_count
21562 .saturating_add(1);
21563 }
21564 _ => unreachable!("non-grid reconstruction overflow reason is capped"),
21565 }
21566 schedule.fallbacks.push(MixedScheduleFallback {
21567 producer: FormulaProducerId::Legacy(vertices[0]),
21568 reason,
21569 });
21570 return schedule;
21571 }
21572
21573 let mut ready = indegree
21574 .iter()
21575 .filter_map(|(&producer, °ree)| (degree == 0).then_some(producer))
21576 .collect::<BTreeSet<_>>();
21577 let mut layers = Vec::new();
21578 let mut scheduled_count = 0usize;
21579 while !ready.is_empty() {
21580 let current = std::mem::take(&mut ready);
21581 let mut next = BTreeSet::new();
21582 let mut work = Vec::with_capacity(current.len());
21583 for producer in current {
21584 scheduled_count = scheduled_count.saturating_add(1);
21585 work.push(
21586 work_by_producer
21587 .get(&producer)
21588 .expect("ready producer has work")
21589 .clone(),
21590 );
21591 for &target in outgoing.get(&producer).into_iter().flatten() {
21592 let degree = indegree.get_mut(&target).expect("edge target has indegree");
21593 *degree -= 1;
21594 if *degree == 0 {
21595 next.insert(target);
21596 }
21597 }
21598 }
21599 layers.push(MixedLayer { work });
21600 ready = next;
21601 }
21602
21603 let mut fallbacks = schedule.fallbacks;
21604 let mut stats = schedule.stats;
21605 if scheduled_count != work_by_producer.len() {
21606 for (&producer, °ree) in &indegree {
21607 if degree != 0 {
21608 fallbacks.push(MixedScheduleFallback {
21609 producer,
21610 reason: MixedScheduleFallbackReason::CycleDetected,
21611 });
21612 }
21613 }
21614 stats.cycle_count = stats.cycle_count.saturating_add(1);
21615 }
21616 stats.input_work_items = stats.input_work_items.saturating_add(vertices.len());
21617 stats.merged_work_items = work_by_producer.len();
21618 stats.unique_producers = work_by_producer.len();
21619 stats.consumer_candidate_count = stats
21620 .consumer_candidate_count
21621 .saturating_add(candidate_count);
21622 stats.edges_added = stats.edges_added.saturating_add(derived_edges);
21623 stats.duplicate_edges_skipped = stats
21624 .duplicate_edges_skipped
21625 .saturating_add(duplicate_edges);
21626 stats.layers = layers.len();
21627 MixedSchedule {
21628 layers,
21629 stats,
21630 fallbacks,
21631 }
21632 }
21633
21634 fn build_formula_plane_mixed_schedule(
21635 &mut self,
21636 formula_dirty: &FormulaDirtyLease,
21637 retry_whole_spans: &[FormulaSpanRef],
21638 include_dirty_regions: bool,
21639 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21640 self.build_formula_plane_mixed_schedule_inner(
21641 formula_dirty,
21642 retry_whole_spans,
21643 include_dirty_regions,
21644 None,
21645 )
21646 }
21647
21648 fn build_formula_plane_target_schedule(
21649 &mut self,
21650 formula_dirty: &FormulaDirtyLease,
21651 retry_whole_spans: &[FormulaSpanRef],
21652 include_dirty_regions: bool,
21653 target_roots: &[crate::engine::target_preparation::TargetProducer],
21654 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21655 self.build_formula_plane_mixed_schedule_inner(
21656 formula_dirty,
21657 retry_whole_spans,
21658 include_dirty_regions,
21659 Some(target_roots),
21660 )
21661 }
21662
21663 fn build_formula_plane_mixed_schedule_inner(
21664 &mut self,
21665 formula_dirty: &FormulaDirtyLease,
21666 retry_whole_spans: &[FormulaSpanRef],
21667 include_dirty_regions: bool,
21668 target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
21669 ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21670 let baseline = self.graph.baseline_stats();
21671 self.charge_bounded_work(
21672 baseline
21673 .graph_vertex_count
21674 .saturating_add(baseline.graph_edge_count) as u64,
21675 )?;
21676 let key = self.mixed_topology_cache_key();
21677 let active_refs = self
21678 .graph
21679 .formula_authority()
21680 .active_span_refs()
21681 .into_iter()
21682 .map(|span_ref| (span_ref.id, span_ref))
21683 .collect::<BTreeMap<_, _>>();
21684 let has_dynamic_legacy = self
21685 .graph
21686 .formula_vertices()
21687 .into_iter()
21688 .any(|vertex| self.graph.is_dynamic(vertex));
21689 let cache_hit = !has_dynamic_legacy
21690 && self
21691 .cached_mixed_topology
21692 .as_ref()
21693 .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
21694 let mut skipped_topology = None;
21695 let mut index_scratch_reserved = 0_u64;
21696 let mut demand_scratch_reserved = 0_u64;
21697 let topology_started = crate::instant::FzInstant::now();
21698 let (cache_outcome, strategy, compile_stats) = if cache_hit {
21699 self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
21700 self.mixed_topology_cache_skip_streak = 0;
21701 let cached = self
21702 .cached_mixed_topology
21703 .as_ref()
21704 .expect("cache hit has topology");
21705 let stats = cached.topology.stats.clone();
21706 let strategy = if cached.topology.is_complete() {
21707 FormulaPlaneTopologyStrategy::Cached
21708 } else {
21709 FormulaPlaneTopologyStrategy::ExactPagedIndexed
21710 };
21711 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21712 ledger
21713 .account_mixed_cache(stats.estimated_memory_bytes as u64)
21714 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21715 }
21716 (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
21717 } else {
21718 self.cached_mixed_topology = None;
21721 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21722 ledger
21723 .account_mixed_cache(0)
21724 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21725 }
21726
21727 let preflight = self.mixed_topology_index_preflight_bytes();
21728 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21729 ledger
21730 .reserve_schedule_discovery(preflight)
21731 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21732 }
21733 index_scratch_reserved = preflight;
21734 let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
21735 Ok(compiled) => compiled,
21736 Err(error) => {
21737 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21738 let _ = ledger.release_scratch(index_scratch_reserved);
21739 }
21740 return Err(error);
21741 }
21742 };
21743 let actual_index_bytes = match &compiled {
21744 FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
21745 .producer_results
21746 .estimated_memory_bytes()
21747 .and_then(|bytes| {
21748 bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
21749 })
21750 .and_then(|bytes| {
21751 bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
21752 })
21753 .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
21754 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
21755 .producer_results
21756 .estimated_memory_bytes()
21757 .and_then(|bytes| {
21758 bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
21759 })
21760 .and_then(|bytes| {
21761 bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
21762 })
21763 .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
21764 }
21765 .unwrap_or(usize::MAX) as u64;
21766 debug_assert!(
21767 actual_index_bytes <= index_scratch_reserved,
21768 "mixed topology index preflight underestimated actual allocation"
21769 );
21770 if actual_index_bytes > index_scratch_reserved {
21771 let additional = actual_index_bytes - index_scratch_reserved;
21772 let reserve_result = self
21773 .active_resource_ledger
21774 .as_mut()
21775 .map_or(Ok(()), |ledger| {
21776 ledger.reserve_schedule_discovery(additional)
21777 });
21778 if let Err(error) = reserve_result {
21779 drop(compiled);
21780 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21781 let _ = ledger.release_scratch(index_scratch_reserved);
21782 }
21783 return Err(error.into_excel_error());
21784 }
21785 } else if actual_index_bytes < index_scratch_reserved
21786 && let Some(ledger) = self.active_resource_ledger.as_mut()
21787 {
21788 ledger
21789 .release_scratch(index_scratch_reserved - actual_index_bytes)
21790 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21791 }
21792 index_scratch_reserved = actual_index_bytes;
21793
21794 self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
21795 match compiled {
21796 FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
21797 let stats = compiled.topology.stats.clone();
21798 self.mixed_topology_cache_skip_streak =
21799 self.mixed_topology_cache_skip_streak.saturating_add(1);
21800 skipped_topology = Some(SkippedMixedTopology {
21801 reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
21802 observed: stats.clone(),
21803 producer_results: compiled.producer_results,
21804 consumer_reads: compiled.consumer_reads,
21805 span_refs_by_id: compiled.span_refs_by_id,
21806 plane_epoch: compiled.plane_epoch,
21807 island: compiled.island,
21808 });
21809 (
21810 FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
21811 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
21812 stats,
21813 )
21814 }
21815 FormulaPlaneTopologyCompileResult::Cached(compiled) => {
21816 let stats = compiled.topology.stats.clone();
21817 let strategy = if compiled.topology.is_complete() {
21818 FormulaPlaneTopologyStrategy::CompiledAndCached
21819 } else {
21820 FormulaPlaneTopologyStrategy::ExactPagedIndexed
21821 };
21822 let retained_result = self
21823 .active_resource_ledger
21824 .as_mut()
21825 .map_or(Ok(()), |ledger| {
21826 ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
21827 });
21828 if let Err(error) = retained_result {
21829 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21830 let _ = ledger.release_scratch(index_scratch_reserved);
21831 }
21832 return Err(error.into_excel_error());
21833 }
21834 self.mixed_topology_cache_skip_streak = 0;
21835 self.cached_mixed_topology = Some(compiled);
21836 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21837 ledger
21838 .release_scratch(index_scratch_reserved)
21839 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21840 }
21841 index_scratch_reserved = 0;
21842 (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
21843 }
21844 FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
21845 debug_assert!(self.cached_mixed_topology.is_none());
21846 self.mixed_topology_cache_overflows =
21847 self.mixed_topology_cache_overflows.saturating_add(1);
21848 self.mixed_topology_cache_skip_streak =
21849 self.mixed_topology_cache_skip_streak.saturating_add(1);
21850 let stats = skipped.observed.clone();
21851 skipped_topology = Some(skipped);
21852 (
21853 FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
21854 FormulaPlaneTopologyStrategy::ExactPagedIndexed,
21855 stats,
21856 )
21857 }
21858 }
21859 };
21860 self.observe_topology(
21861 cache_outcome,
21862 strategy,
21863 &compile_stats,
21864 topology_started.elapsed(),
21865 );
21866 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
21867 stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
21868 stats.topology.candidate_cap = Some(key.max_candidates as u64);
21869 stats.topology.edge_cap = Some(key.max_edges as u64);
21870 stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
21871 if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
21872 stats.topology.operator_guidance =
21873 Some("dynamic legacy dependencies require an exact request topology rebuild");
21874 } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
21875 stats.topology.operator_guidance =
21876 Some(if compile_stats.candidate_overflow_count > 0 {
21877 "raise the mixed-cache candidate limit to avoid exact request rebuilds"
21878 } else if compile_stats.edge_overflow_count > 0 {
21879 "raise the mixed-cache edge limit to avoid exact request rebuilds"
21880 } else {
21881 "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
21882 });
21883 }
21884 }
21885
21886 let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
21887 let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21888 let (
21889 producer_results,
21890 consumer_reads,
21891 span_refs_by_id,
21892 plane_epoch,
21893 cached_topology,
21894 partial_topology,
21895 island,
21896 ) = if let Some(skipped) = skipped_topology.as_ref() {
21897 (
21898 &skipped.producer_results,
21899 &skipped.consumer_reads,
21900 &skipped.span_refs_by_id,
21901 skipped.plane_epoch,
21902 None,
21903 None,
21904 &skipped.island,
21905 )
21906 } else {
21907 let cached = self
21908 .cached_mixed_topology
21909 .as_ref()
21910 .expect("mixed topology available for request");
21911 let (complete, partial) = if cached.topology.is_complete() {
21912 (Some(&cached.topology), None)
21913 } else {
21914 (None, Some(&cached.topology))
21915 };
21916 (
21917 &cached.producer_results,
21918 &cached.consumer_reads,
21919 &cached.span_refs_by_id,
21920 cached.plane_epoch,
21921 complete,
21922 partial,
21923 &cached.island,
21924 )
21925 };
21926 let demand = if let Some(target_roots) = target_roots {
21927 use crate::engine::target_preparation::TargetProducer;
21928 let mut roots = Vec::new();
21929 let mut graph_roots = Vec::new();
21930 for root in target_roots {
21931 match *root {
21932 TargetProducer::Span { span_ref, demanded } => {
21933 roots.push((FormulaProducerId::Span(span_ref.id), demanded))
21934 }
21935 TargetProducer::Legacy(vertex) => {
21936 graph_roots.push(vertex);
21937 if let Some(region) = producer_results
21938 .producer_result_region(FormulaProducerId::Legacy(vertex))
21939 {
21940 roots.push((FormulaProducerId::Legacy(vertex), region));
21941 }
21942 }
21943 TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
21944 TargetProducer::ValueOnly(_) => {}
21945 }
21946 }
21947 if !graph_roots.is_empty() {
21948 let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
21949 for vertex in legacy_demand {
21950 if let Some(region) = producer_results
21951 .producer_result_region(FormulaProducerId::Legacy(vertex))
21952 {
21953 roots.push((FormulaProducerId::Legacy(vertex), region));
21954 }
21955 }
21956 }
21957 let roots = roots;
21958 let closure_base = (producer_results.len() as u64)
21959 .saturating_mul(128)
21960 .saturating_add(4096);
21961 let scratch_estimates =
21962 exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
21963 let native_allowed = native_exact_demand_allowed(
21964 self.active_resource_ledger
21965 .as_ref()
21966 .and_then(ResourceLedger::disk_scratch_policy),
21967 !cfg!(target_arch = "wasm32"),
21968 );
21969 let (selected_strategy, reservation) = if cached_topology.is_some() {
21970 (None, closure_base)
21971 } else {
21972 let (strategy, reservation) =
21973 select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
21974 self.can_reserve_topology_scratch(bytes)
21975 });
21976 (Some(strategy), reservation)
21977 };
21978 if let Some(ledger) = self.active_resource_ledger.as_mut() {
21979 ledger
21980 .reserve_schedule_discovery(reservation)
21981 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21982 }
21983 demand_scratch_reserved = reservation;
21984 #[allow(unused_mut)]
21985 let mut native_disk_bytes = 0_u64;
21986 let (demand, actual_strategy, passes) = match selected_strategy {
21987 None => (
21988 build_demand_closure_cached(
21989 roots,
21990 producer_results,
21991 cached_topology.expect("cached demand strategy has topology"),
21992 ),
21993 None,
21994 0,
21995 ),
21996 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
21997 let (demand, passes) = build_demand_closure_paged(
21998 roots,
21999 producer_results,
22000 consumer_reads,
22001 |units| {
22002 Self::resource_loop_checkpoint(
22003 &mut self.active_resource_ledger,
22004 units,
22005 )
22006 },
22007 )?;
22008 (demand, selected_strategy, passes)
22009 }
22010 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
22011 let (demand, passes) = build_demand_closure_in_memory_runs(
22012 roots,
22013 producer_results,
22014 consumer_reads,
22015 |units| {
22016 Self::resource_loop_checkpoint(
22017 &mut self.active_resource_ledger,
22018 units,
22019 )
22020 },
22021 )?;
22022 (demand, selected_strategy, passes)
22023 }
22024 #[cfg(not(target_arch = "wasm32"))]
22025 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
22026 let request_id = self
22027 .active_evaluation_resource_request
22028 .as_ref()
22029 .map_or(0, |stats| stats.request_id);
22030 match NativeTopologyScratch::create(request_id) {
22031 Ok(mut native) => match build_demand_closure_native(
22032 roots.clone(),
22033 producer_results,
22034 consumer_reads,
22035 native.file.as_mut().expect("native scratch file"),
22036 native
22037 .auxiliary_file
22038 .as_mut()
22039 .expect("native auxiliary scratch file"),
22040 |units| {
22041 Self::resource_loop_checkpoint(
22042 &mut self.active_resource_ledger,
22043 units,
22044 )
22045 },
22046 ) {
22047 Ok((demand, passes, bytes)) => {
22048 native_disk_bytes = bytes;
22049 (demand, selected_strategy, passes)
22050 }
22051 Err(NativeExactDemandError::Work(error)) => return Err(error),
22052 Err(NativeExactDemandError::Io(_)) => {
22053 let (demand, passes) = build_demand_closure_repeated_passes(
22054 roots,
22055 producer_results,
22056 consumer_reads,
22057 |units| {
22058 Self::resource_loop_checkpoint(
22059 &mut self.active_resource_ledger,
22060 units,
22061 )
22062 },
22063 )?;
22064 (
22065 demand,
22066 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22067 passes,
22068 )
22069 }
22070 },
22071 Err(_) => {
22072 let (demand, passes) = build_demand_closure_repeated_passes(
22073 roots,
22074 producer_results,
22075 consumer_reads,
22076 |units| {
22077 Self::resource_loop_checkpoint(
22078 &mut self.active_resource_ledger,
22079 units,
22080 )
22081 },
22082 )?;
22083 (
22084 demand,
22085 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22086 passes,
22087 )
22088 }
22089 }
22090 }
22091 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
22092 let (demand, passes) = build_demand_closure_repeated_passes(
22093 roots,
22094 producer_results,
22095 consumer_reads,
22096 |units| {
22097 Self::resource_loop_checkpoint(
22098 &mut self.active_resource_ledger,
22099 units,
22100 )
22101 },
22102 )?;
22103 (demand, selected_strategy, passes)
22104 }
22105 _ => unreachable!("exact demand strategy selection is exhaustive"),
22106 };
22107 let strategy_memory = match actual_strategy {
22108 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
22109 scratch_estimates.paged.saturating_sub(closure_base)
22110 }
22111 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
22112 scratch_estimates.runs.saturating_sub(closure_base)
22113 }
22114 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
22115 scratch_estimates.native.saturating_sub(closure_base)
22116 }
22117 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
22118 scratch_estimates.repeated.saturating_sub(closure_base)
22119 }
22120 None => 0,
22121 _ => 0,
22122 };
22123 let actual_scratch = demand
22124 .estimated_memory_bytes()
22125 .saturating_add(strategy_memory);
22126 if actual_scratch > demand_scratch_reserved {
22127 if let Some(ledger) = self.active_resource_ledger.as_mut() {
22128 ledger
22129 .reserve_schedule_discovery(
22130 actual_scratch.saturating_sub(demand_scratch_reserved),
22131 )
22132 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22133 }
22134 } else if actual_scratch < demand_scratch_reserved
22135 && let Some(ledger) = self.active_resource_ledger.as_mut()
22136 {
22137 ledger
22138 .release_scratch(demand_scratch_reserved - actual_scratch)
22139 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22140 }
22141 demand_scratch_reserved = actual_scratch;
22142 if let Some(strategy) = actual_strategy
22143 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
22144 {
22145 if strategy.severity() > stats.topology.strategy.severity() {
22146 stats.topology.strategy = strategy;
22147 }
22148 stats.topology.exact_pass_count = stats
22149 .topology
22150 .exact_pass_count
22151 .saturating_add(passes.max(1));
22152 stats.topology.native_topology_disk_bytes = stats
22153 .topology
22154 .native_topology_disk_bytes
22155 .saturating_add(native_disk_bytes);
22156 }
22157 Some(demand)
22158 } else {
22159 None
22160 };
22161
22162 let dirty_legacy = self.graph.get_evaluation_vertices();
22163 let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
22164 let island_target_demand = target_roots.map(|roots| {
22165 use crate::engine::target_preparation::TargetProducer;
22166 let graph_roots = roots
22167 .iter()
22168 .filter_map(|root| match *root {
22169 TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
22170 Some(vertex)
22171 }
22172 TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
22173 })
22174 .collect::<Vec<_>>();
22175 self.build_demand_subgraph(&graph_roots)
22176 .0
22177 .into_iter()
22178 .collect::<FxHashSet<_>>()
22179 });
22180 let mut island_plan = island.clone();
22181 island_plan.dirty_vertices = island_plan
22182 .membership
22183 .iter()
22184 .copied()
22185 .filter(|vertex| dirty_legacy_set.contains(vertex))
22186 .filter(|vertex| {
22187 island_target_demand
22188 .as_ref()
22189 .is_none_or(|demand| demand.contains(vertex))
22190 })
22191 .collect();
22192 let mut work = Vec::new();
22193 let mut scheduled_legacy_vertices = Vec::new();
22194 let mut non_grid_legacy_vertices = Vec::new();
22195 let mut non_grid_legacy_fallbacks = Vec::new();
22196 let demanded_dirty = |producer: FormulaProducerId,
22197 dirty: ProducerDirtyDomain|
22198 -> Option<ProducerDirtyDomain> {
22199 let Some(demand) = demand.as_ref() else {
22200 return Some(dirty);
22201 };
22202 let result = producer_results.producer_result_region(producer)?;
22203 dirty.intersect_demanded(result, demand.regions(producer))
22204 };
22205 for (span_ref, region) in formula_dirty.span_regions() {
22206 let producer = FormulaProducerId::Span(span_ref.id);
22207 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
22208 && let Some(dirty) =
22209 demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
22210 {
22211 work.push(FormulaProducerWork { producer, dirty });
22212 }
22213 }
22214 for span_ref in formula_dirty
22215 .whole_spans()
22216 .chain(retry_whole_spans.iter().copied())
22217 {
22218 let producer = FormulaProducerId::Span(span_ref.id);
22219 if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
22220 && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
22221 {
22222 work.push(FormulaProducerWork { producer, dirty });
22223 }
22224 }
22225 for vertex in dirty_legacy {
22226 let producer = FormulaProducerId::Legacy(vertex);
22227 if producer_results.producer_result_region(producer).is_some() {
22228 if let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole) {
22229 scheduled_legacy_vertices.push(vertex);
22230 work.push(FormulaProducerWork { producer, dirty });
22231 }
22232 } else if matches!(
22233 self.graph.get_vertex_kind(vertex),
22234 VertexKind::NamedScalar | VertexKind::NamedArray
22235 ) {
22236 if island_target_demand
22237 .as_ref()
22238 .is_none_or(|demand| demand.contains(&vertex))
22239 {
22240 non_grid_legacy_vertices.push(vertex);
22245 } else {
22246 non_grid_legacy_fallbacks.push(MixedScheduleFallback {
22247 producer,
22248 reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
22249 });
22250 }
22251 }
22252 }
22253
22254 let pending_changed_regions = formula_dirty
22255 .regions()
22256 .chain(formula_dirty.span_regions().map(|(_, region)| region))
22257 .collect::<Vec<_>>();
22258 if include_dirty_regions && !pending_changed_regions.is_empty() {
22259 use crate::formula_plane::producer::compute_dirty_closure_checked;
22260 let closure = compute_dirty_closure_checked(
22261 consumer_reads,
22262 pending_changed_regions.iter().copied(),
22263 |producer| producer_results.producer_result_region(producer),
22264 |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
22265 )?;
22266 if closure.incomplete {
22267 work.clear();
22270 scheduled_legacy_vertices.clear();
22271 if let Some(demand) = demand.as_ref() {
22272 for producer in demand.producers() {
22273 if let Some(dirty) =
22274 demanded_dirty(producer, ProducerDirtyDomain::Whole)
22275 {
22276 if let FormulaProducerId::Legacy(vertex) = producer {
22277 scheduled_legacy_vertices.push(vertex);
22278 }
22279 work.push(FormulaProducerWork { producer, dirty });
22280 }
22281 }
22282 } else {
22283 work.extend(span_refs_by_id.keys().copied().map(|id| {
22284 FormulaProducerWork {
22285 producer: FormulaProducerId::Span(id),
22286 dirty: ProducerDirtyDomain::Whole,
22287 }
22288 }));
22289 for vertex in self.graph.formula_vertices() {
22290 let producer = FormulaProducerId::Legacy(vertex);
22291 if producer_results.producer_result_region(producer).is_some() {
22292 scheduled_legacy_vertices.push(vertex);
22293 work.push(FormulaProducerWork {
22294 producer,
22295 dirty: ProducerDirtyDomain::Whole,
22296 });
22297 }
22298 }
22299 }
22300 if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
22301 stats.topology.incomplete_reason =
22302 Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
22303 }
22304 } else if let Some(demand) = demand.as_ref() {
22305 for item in closure.work {
22306 if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
22307 work.push(FormulaProducerWork {
22308 producer: item.producer,
22309 dirty,
22310 });
22311 }
22312 }
22313 } else {
22314 work.extend(closure.work);
22315 }
22316 if !closure.incomplete && !closure.fallbacks.is_empty() {
22317 let mut already_whole = work
22318 .iter()
22319 .filter_map(|item| match (item.producer, &item.dirty) {
22320 (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
22321 _ => None,
22322 })
22323 .collect::<rustc_hash::FxHashSet<_>>();
22324 for fallback in closure.fallbacks {
22325 if let FormulaProducerId::Span(id) = fallback.consumer
22326 && already_whole.insert(id)
22327 {
22328 work.push(FormulaProducerWork {
22329 producer: FormulaProducerId::Span(id),
22330 dirty: ProducerDirtyDomain::Whole,
22331 });
22332 }
22333 }
22334 }
22335 }
22336
22337 scheduled_legacy_vertices.extend(non_grid_legacy_vertices.iter().copied());
22338
22339 let owned_dirty_events = if let Some(demand) = demand.as_ref() {
22340 use crate::formula_plane::producer::compute_dirty_closure_checked;
22341 let mut owned = Vec::new();
22342 for (index, event) in formula_dirty.events() {
22343 let (seed_regions, initial) = match event {
22344 FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
22345 FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
22346 vec![region],
22347 Some(FormulaProducerWork {
22348 producer: FormulaProducerId::Span(span_ref.id),
22349 dirty: ProducerDirtyDomain::Regions(vec![region]),
22350 }),
22351 ),
22352 FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
22353 let producer = FormulaProducerId::Span(span_ref.id);
22354 let Some(result) = producer_results.producer_result_region(producer)
22355 else {
22356 continue;
22357 };
22358 (
22359 vec![result],
22360 Some(FormulaProducerWork {
22361 producer,
22362 dirty: ProducerDirtyDomain::Whole,
22363 }),
22364 )
22365 }
22366 };
22367 let closure = compute_dirty_closure_checked(
22368 consumer_reads,
22369 seed_regions,
22370 |producer| producer_results.producer_result_region(producer),
22371 |units| {
22372 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
22373 },
22374 )?;
22375 if closure.incomplete || !closure.fallbacks.is_empty() {
22376 continue;
22377 }
22378 let fully_demanded =
22379 initial
22380 .into_iter()
22381 .chain(closure.work.into_iter())
22382 .all(|item| {
22383 let Some(result) =
22384 producer_results.producer_result_region(item.producer)
22385 else {
22386 return false;
22387 };
22388 item.dirty
22389 .result_regions(result)
22390 .into_iter()
22391 .all(|dirty_region| {
22392 demand
22393 .regions(item.producer)
22394 .iter()
22395 .any(|demanded| demanded.contains_region(dirty_region))
22396 })
22397 });
22398 if fully_demanded {
22399 owned.push(index);
22400 }
22401 }
22402 owned
22403 } else {
22404 (0..formula_dirty.len()).collect()
22405 };
22406
22407 Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
22408 let producer_count = producer_results.len();
22409 let dirty_units = work.iter().fold(0usize, |total, item| {
22410 total.saturating_add(match &item.dirty {
22411 ProducerDirtyDomain::Whole => 0,
22412 ProducerDirtyDomain::Cells(cells) => cells.len(),
22413 ProducerDirtyDomain::Regions(regions) => regions.len(),
22414 })
22415 });
22416 let work_bytes = work
22417 .len()
22418 .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
22419 .saturating_add(
22420 dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
22421 std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
22422 )),
22423 ) as u64;
22424 let binding_bytes =
22425 estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
22426 let schedule_bytes = work_bytes
22427 .saturating_mul(4)
22428 .saturating_add((producer_count as u64).saturating_mul(192))
22429 .saturating_add(binding_bytes)
22430 .saturating_add(1024);
22431 let maximum_edges =
22432 producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
22433 let paged_bytes = schedule_bytes
22437 .saturating_add(maximum_edges.saturating_mul(160))
22438 .saturating_add(16 * 1024);
22439 let runs_bytes = schedule_bytes
22440 .saturating_add(maximum_edges.saturating_mul(48))
22441 .saturating_add(8192);
22442 let native_bytes = schedule_bytes
22443 .saturating_add((producer_count as u64).saturating_mul(128))
22444 .saturating_add(2048);
22445 let repeated_bytes = schedule_bytes
22446 .saturating_add((producer_count as u64).saturating_mul(96))
22447 .saturating_add(1024);
22448 let native_allowed = self
22449 .active_resource_ledger
22450 .as_ref()
22451 .and_then(ResourceLedger::disk_scratch_policy)
22452 == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
22453 && !cfg!(target_arch = "wasm32");
22454
22455 let cached_schedule_bytes = schedule_bytes
22456 .saturating_add(
22457 cached_topology
22458 .map_or(0, |topology| topology.stats.relationships as u64)
22459 .saturating_mul(160),
22460 )
22461 .saturating_add(1024);
22462 let use_cached_schedule = cached_topology.is_some()
22463 && self.can_reserve_topology_scratch(cached_schedule_bytes);
22464 let (selected_strategy, scratch) = if use_cached_schedule {
22465 (None, cached_schedule_bytes)
22466 } else if self.can_reserve_topology_scratch(paged_bytes) {
22467 (
22468 Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
22469 paged_bytes,
22470 )
22471 } else if self.can_reserve_topology_scratch(runs_bytes) {
22472 (
22473 Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
22474 runs_bytes,
22475 )
22476 } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
22477 (
22478 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
22479 native_bytes,
22480 )
22481 } else if self.can_reserve_topology_scratch(repeated_bytes) {
22482 (
22483 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22484 repeated_bytes,
22485 )
22486 } else {
22487 if let Some(ledger) = self.active_resource_ledger.as_mut() {
22488 ledger
22489 .reserve_schedule_discovery(repeated_bytes)
22490 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22491 }
22492 unreachable!("failed scratch reservation must return a typed error")
22493 };
22494 if let Some(ledger) = self.active_resource_ledger.as_mut() {
22495 ledger
22496 .reserve_schedule_discovery(scratch)
22497 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22498 }
22499 let request_id = self
22500 .active_evaluation_resource_request
22501 .as_ref()
22502 .map_or(0, |stats| stats.request_id);
22503 let build_result = (|| -> Result<_, ExcelError> {
22504 let (schedule, actual_strategy, passes, native_disk_bytes) =
22505 match (cached_topology, selected_strategy) {
22506 (Some(topology), None) => (
22507 schedule_dirty_work(work, producer_results, topology, 256),
22508 None,
22509 0,
22510 0,
22511 ),
22512 (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
22513 let (schedule, passes) = schedule_dirty_work_paged_hybrid(
22514 work,
22515 producer_results,
22516 consumer_reads,
22517 partial_topology.filter(|_| reuse_partial_topology),
22518 256,
22519 |units| {
22520 Self::resource_loop_checkpoint(
22521 &mut self.active_resource_ledger,
22522 units,
22523 )
22524 },
22525 )?;
22526 (schedule, selected_strategy, passes.max(1), 0)
22527 }
22528 (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
22529 let (schedule, passes) = schedule_dirty_work_in_memory_runs(
22530 work,
22531 producer_results,
22532 consumer_reads,
22533 256,
22534 |units| {
22535 Self::resource_loop_checkpoint(
22536 &mut self.active_resource_ledger,
22537 units,
22538 )
22539 },
22540 )?;
22541 (schedule, selected_strategy, passes.max(1), 0)
22542 }
22543 #[cfg(not(target_arch = "wasm32"))]
22544 (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
22545 let native = NativeTopologyScratch::create(request_id);
22546 if let Ok(mut native) = native {
22547 match schedule_dirty_work_native(
22548 work.clone(),
22549 producer_results,
22550 consumer_reads,
22551 256,
22552 native.file.as_mut().expect("native scratch file"),
22553 |units| {
22554 Self::resource_loop_checkpoint(
22555 &mut self.active_resource_ledger,
22556 units,
22557 )
22558 },
22559 ) {
22560 Ok((schedule, passes, topology_bytes)) => (
22561 schedule,
22562 Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
22563 passes.max(1),
22564 topology_bytes,
22565 ),
22566 Err(NativeExactScheduleError::Work(error)) => {
22567 return Err(error);
22568 }
22569 Err(NativeExactScheduleError::Io(_)) => {
22570 let (schedule, passes) =
22571 schedule_dirty_work_repeated_passes(
22572 work,
22573 producer_results,
22574 consumer_reads,
22575 256,
22576 |units| {
22577 Self::resource_loop_checkpoint(
22578 &mut self.active_resource_ledger,
22579 units,
22580 )
22581 },
22582 )?;
22583 (
22584 schedule,
22585 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22586 passes.max(1),
22587 0,
22588 )
22589 }
22590 }
22591 } else {
22592 let (schedule, passes) = schedule_dirty_work_repeated_passes(
22593 work,
22594 producer_results,
22595 consumer_reads,
22596 256,
22597 |units| {
22598 Self::resource_loop_checkpoint(
22599 &mut self.active_resource_ledger,
22600 units,
22601 )
22602 },
22603 )?;
22604 (
22605 schedule,
22606 Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22607 passes.max(1),
22608 0,
22609 )
22610 }
22611 }
22612 (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
22613 let (schedule, passes) = schedule_dirty_work_repeated_passes(
22614 work,
22615 producer_results,
22616 consumer_reads,
22617 256,
22618 |units| {
22619 Self::resource_loop_checkpoint(
22620 &mut self.active_resource_ledger,
22621 units,
22622 )
22623 },
22624 )?;
22625 (schedule, selected_strategy, passes.max(1), 0)
22626 }
22627 _ => unreachable!(
22628 "topology cache and exact strategy selection are exhaustive"
22629 ),
22630 };
22631 Ok((
22632 schedule,
22633 actual_strategy,
22634 passes,
22635 native_disk_bytes,
22636 span_refs_by_id.clone(),
22637 ))
22638 })();
22639 let (mut schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
22640 match build_result {
22641 Ok(output) => output,
22642 Err(error) => {
22643 if let Some(ledger) = self.active_resource_ledger.as_mut() {
22644 let _ = ledger.release_scratch(scratch);
22645 }
22646 return Err(error);
22647 }
22648 };
22649 schedule = self.include_non_grid_legacy_work(
22652 schedule,
22653 &non_grid_legacy_vertices,
22654 &non_grid_legacy_fallbacks,
22655 producer_results,
22656 consumer_reads,
22657 key.max_candidates,
22658 key.max_edges,
22659 );
22660 self.active_resource_ledger
22661 .as_mut()
22662 .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
22663 .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22664 if let Some(strategy) = exact_strategy
22665 && let Some(stats) = self.active_evaluation_resource_request.as_mut()
22666 {
22667 if strategy.severity() > stats.topology.strategy.severity() {
22668 stats.topology.strategy = strategy;
22669 }
22670 stats.topology.exact_pass_count = pass_count;
22671 stats.topology.native_topology_disk_bytes = stats
22672 .topology
22673 .native_topology_disk_bytes
22674 .saturating_add(native_disk_bytes);
22675 }
22676 Ok((
22677 schedule,
22678 result_span_refs,
22679 plane_epoch,
22680 scheduled_legacy_vertices,
22681 owned_dirty_events,
22682 island_plan,
22683 ))
22684 })();
22685 let scratch_release_result = self
22686 .active_resource_ledger
22687 .as_mut()
22688 .map_or(Ok(()), |ledger| {
22689 ledger
22690 .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
22691 })
22692 .map_err(crate::engine::ResourceLedgerError::into_excel_error);
22693 match schedule_result {
22694 Ok(output) => {
22695 scratch_release_result?;
22696 Ok(output)
22697 }
22698 Err(error) => {
22699 let _ = scratch_release_result;
22700 Err(error)
22701 }
22702 }
22703 }
22704}
22705
22706impl<R> Engine<R>
22707where
22708 R: EvaluationContext,
22709{
22710 fn shared_range_to_region_pattern(
22719 &self,
22720 range: &crate::reference::SharedRangeRef<'static>,
22721 context_sheet: SheetId,
22722 ) -> Result<Option<Region>, ExcelError> {
22723 assert!(
22724 matches!(range.sheet, crate::reference::SharedSheetLocator::Id(_)),
22725 "PROBE: non-Id locator reached shared_range_to_region_pattern: {:?}",
22726 range.sheet
22727 );
22728 let Ok(sheet_id) = self.resolve_sheet_locator(&range.sheet, context_sheet) else {
22729 return Ok(None);
22731 };
22732 match (
22733 range.start_row,
22734 range.end_row,
22735 range.start_col,
22736 range.end_col,
22737 ) {
22738 (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
22739 sheet_id, sr.index, er.index, sc.index, ec.index,
22740 ))),
22741 (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
22742 Ok(Some(Region::whole_col(sheet_id, sc.index)))
22743 }
22744 (Some(sr), Some(er), None, None) if sr.index == er.index => {
22745 Ok(Some(Region::whole_row(sheet_id, sr.index)))
22746 }
22747 _ => Ok(None),
22748 }
22749 }
22750
22751 pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
22753 self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
22754 engine.evaluate_all_unobserved()
22755 })
22756 }
22757
22758 fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
22759 debug_assert!(
22760 !self.graph.deferred_dirty_active(),
22761 "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
22762 was not balanced by end_deferred_dirty"
22763 );
22764 self.observe_function_semantic_epoch()?;
22765 self.lookup_index_cache.reset_counters();
22766 let _source_cache = self.source_cache_session();
22767 self.validate_deterministic_mode()?;
22768 if self.config.defer_graph_building {
22769 self.build_graph_all()?;
22771 }
22772 self.evaluate_all_coordinator()
22773 }
22774
22775 fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
22780 self.transition_off_mode_spans_to_legacy()?;
22781 self.begin_evaluation_request();
22782 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
22783 return self.evaluate_authoritative_formula_plane_all();
22784 }
22785 self.evaluate_all_legacy_impl()
22786 }
22787
22788 fn legacy_pass_run_units(
22794 &mut self,
22795 schedule: &crate::engine::scheduler::Schedule,
22796 ) -> Result<(usize, usize), ExcelError> {
22797 let mut computed_vertices = 0;
22798 let mut cycle_count = 0;
22799 for &unit in &schedule.units {
22800 match unit {
22801 ScheduleUnit::Cycle(i) => {
22802 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
22803 cycle_count += 1;
22804 }
22805 }
22806 ScheduleUnit::Layer(i) => {
22807 let layer = schedule.unit_layer(i);
22808 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22809 computed_vertices += self.evaluate_layer_parallel(layer)?;
22810 } else {
22811 computed_vertices += self.evaluate_layer_sequential(layer)?;
22812 }
22813 }
22814 }
22815 }
22816 Ok((computed_vertices, cycle_count))
22817 }
22818
22819 fn evaluate_legacy_island_non_finalizing(
22824 &mut self,
22825 vertices: &[VertexId],
22826 mut delta: Option<&mut DeltaCollector>,
22827 ) -> Result<(usize, usize), ExcelError> {
22828 if vertices.is_empty() {
22829 return Ok((0, 0));
22830 }
22831 let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
22832 let mut computed = 0usize;
22833 let mut cycles = 0usize;
22834 for &unit in &schedule.units {
22835 self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
22836 match unit {
22837 ScheduleUnit::Cycle(index) => {
22838 if self.handle_cycle_unit(
22839 schedule.unit_cycle(index),
22840 delta.as_deref_mut(),
22841 None,
22842 None,
22843 )? > 0
22844 {
22845 cycles = cycles.saturating_add(1);
22846 }
22847 }
22848 ScheduleUnit::Layer(index) => {
22849 let layer = schedule.unit_layer(index);
22850 let evaluated = if let Some(delta) = delta.as_deref_mut() {
22851 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22852 self.evaluate_layer_parallel_with_delta(layer, delta)?
22853 } else {
22854 self.evaluate_layer_sequential_with_delta(layer, delta)?
22855 }
22856 } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22857 self.evaluate_layer_parallel(layer)?
22858 } else {
22859 self.evaluate_layer_sequential(layer)?
22860 };
22861 computed = computed.saturating_add(evaluated);
22862 }
22863 }
22864 }
22865 self.graph.clear_dirty_flags(vertices);
22866 Ok((computed, cycles))
22867 }
22868
22869 fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
22881 self.reset_virtual_dep_telemetry_if_disabled();
22882 #[cfg(feature = "tracing")]
22883 let _span_eval = tracing::info_span!("evaluate_all").entered();
22884 let start = crate::instant::FzInstant::now();
22885 let mut computed_vertices = 0;
22886 let mut cycle_errors = 0;
22887 let mut replan_iterations = 0;
22888 const MAX_REPLAN: usize = 5;
22889 let mut telemetry = self
22890 .config
22891 .enable_virtual_dep_telemetry
22892 .then(|| self.start_virtual_dep_telemetry());
22893
22894 loop {
22895 let to_evaluate = self.graph.get_evaluation_vertices();
22896 if to_evaluate.is_empty() {
22897 if let Some(t) = telemetry.as_mut()
22898 && t.bailout_reason.is_none()
22899 {
22900 t.bailout_reason = Some("no_work");
22901 }
22902 break;
22903 }
22904
22905 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
22906 if let Some(t) = telemetry.as_mut() {
22907 Self::accumulate_schedule_meta(t, &meta);
22908 }
22909
22910 let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
22911 computed_vertices += pass_computed;
22912 cycle_errors += pass_cycles;
22913
22914 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22916 if let Some(t) = telemetry.as_mut() {
22917 t.changed_vdeps_total += changed_vertices.len();
22918 }
22919
22920 self.resource_checkpoint(0)?;
22921 self.graph.clear_dirty_flags(&to_evaluate);
22922 for v in &changed_vertices {
22923 self.graph.set_dirty(*v, true);
22924 }
22925
22926 if changed_vertices.is_empty() {
22927 if let Some(t) = telemetry.as_mut() {
22928 t.bailout_reason = Some("converged");
22929 }
22930 break;
22931 }
22932 if replan_iterations >= MAX_REPLAN {
22933 if let Some(mut t) = telemetry.take() {
22934 t.bailout_reason = Some("max_replan");
22935 t.replan_iterations = replan_iterations;
22936 self.last_virtual_dep_telemetry = t;
22937 }
22938 return Err(
22939 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22940 );
22941 }
22942
22943 replan_iterations += 1;
22944 }
22945
22946 if let Some(mut t) = telemetry {
22947 t.replan_iterations = replan_iterations;
22948 self.last_virtual_dep_telemetry = t;
22949 }
22950
22951 self.redirty_for_next_recalc();
22953
22954 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22956
22957 Ok(EvalResult {
22958 computed_vertices,
22959 cycle_errors,
22960 elapsed: start.elapsed(),
22961 })
22962 }
22963
22964 pub fn evaluate_all_with_target_delta(
22965 &mut self,
22966 ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
22967 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
22968 engine.observe_function_semantic_epoch()?;
22969 let mut collector = DeltaCollector::new(DeltaMode::Cells);
22970 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
22971 Ok((result, collector.finish_target()))
22972 })
22973 }
22974
22975 pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
22976 self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
22977 }
22978
22979 pub fn evaluate_all_with_delta_policy(
22980 &mut self,
22981 policy: EvalDeltaCompatibilityPolicy,
22982 ) -> Result<(EvalResult, EvalDelta), ExcelError> {
22983 self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
22984 engine.observe_function_semantic_epoch()?;
22985 let mut collector = DeltaCollector::new(DeltaMode::Cells);
22986 let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
22987 Ok((result, collector.finish_with_policy(policy)?))
22988 })
22989 }
22990
22991 fn evaluate_all_with_delta_collector(
22992 &mut self,
22993 delta: &mut DeltaCollector,
22994 ) -> Result<EvalResult, ExcelError> {
22995 let _source_cache = self.source_cache_session();
22996 if self.config.defer_graph_building {
22997 self.build_graph_all()?;
22998 }
22999 self.transition_off_mode_spans_to_legacy()?;
23000 self.begin_evaluation_request();
23001 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
23002 && self.graph.formula_authority().active_span_count() > 0
23003 {
23004 return self.evaluate_authoritative_formula_plane(None, Some(delta));
23005 }
23006 self.reset_virtual_dep_telemetry_if_disabled();
23007 #[cfg(feature = "tracing")]
23008 let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
23009 let start = crate::instant::FzInstant::now();
23010 let mut computed_vertices = 0;
23011 let mut cycle_errors = 0;
23012
23013 let mut replan_iterations = 0;
23014 const MAX_REPLAN: usize = 5;
23015 let mut telemetry = self
23016 .config
23017 .enable_virtual_dep_telemetry
23018 .then(|| self.start_virtual_dep_telemetry());
23019
23020 loop {
23021 let to_evaluate = self.graph.get_evaluation_vertices();
23022 if to_evaluate.is_empty() {
23023 if let Some(t) = telemetry.as_mut()
23024 && t.bailout_reason.is_none()
23025 {
23026 t.bailout_reason = Some("no_work");
23027 }
23028 break;
23029 }
23030
23031 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
23032 if let Some(t) = telemetry.as_mut() {
23033 Self::accumulate_schedule_meta(t, &meta);
23034 }
23035
23036 for &unit in &schedule.units {
23037 match unit {
23038 ScheduleUnit::Cycle(i) => {
23039 if self.handle_cycle_unit(
23040 schedule.unit_cycle(i),
23041 Some(delta),
23042 None,
23043 None,
23044 )? > 0
23045 {
23046 cycle_errors += 1;
23047 }
23048 }
23049 ScheduleUnit::Layer(i) => {
23050 let layer = schedule.unit_layer(i);
23051 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
23052 computed_vertices +=
23053 self.evaluate_layer_parallel_with_delta(layer, delta)?;
23054 } else {
23055 computed_vertices +=
23056 self.evaluate_layer_sequential_with_delta(layer, delta)?;
23057 }
23058 }
23059 }
23060 }
23061
23062 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
23063 if let Some(t) = telemetry.as_mut() {
23064 t.changed_vdeps_total += changed_vertices.len();
23065 }
23066 self.resource_checkpoint(0)?;
23067 self.graph.clear_dirty_flags(&to_evaluate);
23068 for v in &changed_vertices {
23069 self.graph.set_dirty(*v, true);
23070 }
23071
23072 if changed_vertices.is_empty() {
23073 if let Some(t) = telemetry.as_mut() {
23074 t.bailout_reason = Some("converged");
23075 }
23076 break;
23077 }
23078 if replan_iterations >= MAX_REPLAN {
23079 if let Some(mut t) = telemetry.take() {
23080 t.bailout_reason = Some("max_replan");
23081 t.replan_iterations = replan_iterations;
23082 self.last_virtual_dep_telemetry = t;
23083 }
23084 return Err(
23085 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
23086 );
23087 }
23088 replan_iterations += 1;
23089 }
23090
23091 if let Some(mut t) = telemetry {
23092 t.replan_iterations = replan_iterations;
23093 self.last_virtual_dep_telemetry = t;
23094 }
23095
23096 self.redirty_for_next_recalc();
23097 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
23098
23099 Ok(EvalResult {
23100 computed_vertices,
23101 cycle_errors,
23102 elapsed: start.elapsed(),
23103 })
23104 }
23105
23106 pub fn evaluate_cell(
23116 &mut self,
23117 sheet: &str,
23118 row: u32,
23119 col: u32,
23120 ) -> Result<Option<LiteralValue>, ExcelError> {
23121 self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
23122 engine.evaluate_cell_unobserved(sheet, row, col)
23123 })
23124 }
23125
23126 fn evaluate_cell_unobserved(
23127 &mut self,
23128 sheet: &str,
23129 row: u32,
23130 col: u32,
23131 ) -> Result<Option<LiteralValue>, ExcelError> {
23132 if row == 0 || col == 0 {
23133 return Err(ExcelError::new(ExcelErrorKind::Ref)
23134 .with_message("Row and column must be >= 1".to_string()));
23135 }
23136
23137 let result = self.evaluate_cells(&[(sheet, row, col)])?;
23138
23139 match result.len() {
23140 0 => Ok(None),
23141 1 => {
23142 let v = result.into_iter().next().unwrap();
23143 Ok(v)
23144 }
23145 _ => unreachable!("evaluate_cells returned unexpected length"),
23146 }
23147 }
23148
23149 pub fn evaluate_cells(
23156 &mut self,
23157 targets: &[(&str, u32, u32)],
23158 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23159 self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
23160 engine.evaluate_cells_unobserved(targets)
23161 })
23162 }
23163
23164 fn evaluate_cells_unobserved(
23165 &mut self,
23166 targets: &[(&str, u32, u32)],
23167 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23168 self.observe_function_semantic_epoch()?;
23169 debug_assert!(
23170 !self.graph.deferred_dirty_active(),
23171 "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
23172 was not balanced by end_deferred_dirty"
23173 );
23174 self.validate_deterministic_mode()?;
23175 if targets.is_empty() {
23176 return Ok(Vec::new());
23177 }
23178 let typed_targets = self.legacy_coordinate_targets(targets);
23179 self.evaluate_mixed_targets(&typed_targets, None)?;
23180 Ok(targets
23181 .iter()
23182 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
23183 .collect())
23184 }
23185
23186 pub fn evaluate_cells_cancellable(
23187 &mut self,
23188 targets: &[(&str, u32, u32)],
23189 cancel: crate::engine::CancelToken,
23190 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23191 self.observe_evaluation_resource_request(
23192 EvaluationRequestKind::CellsCancellable,
23193 |engine| {
23194 engine.observe_function_semantic_epoch()?;
23195 engine.active_cancel_flag = Some(cancel.clone());
23196 let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
23197 engine.active_cancel_flag = None;
23198 res
23199 },
23200 )
23201 }
23202
23203 fn evaluate_cells_cancellable_impl(
23204 &mut self,
23205 targets: &[(&str, u32, u32)],
23206 cancel_flag: &AtomicBool,
23207 ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23208 self.validate_deterministic_mode()?;
23209 if targets.is_empty() {
23210 return Ok(Vec::new());
23211 }
23212 if cancel_flag.load(Ordering::Relaxed) {
23213 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23214 .with_message("Evaluation cancelled before target preparation"));
23215 }
23216 let typed_targets = self.legacy_coordinate_targets(targets);
23217 self.evaluate_mixed_targets(&typed_targets, None)?;
23218 Ok(targets
23219 .iter()
23220 .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
23221 .collect())
23222 }
23223
23224 pub fn evaluate_cells_with_target_delta(
23225 &mut self,
23226 targets: &[(&str, u32, u32)],
23227 ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
23228 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
23229 engine.observe_function_semantic_epoch()?;
23230 engine.validate_deterministic_mode()?;
23231 if targets.is_empty() {
23232 return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
23233 }
23234 let mut collector = DeltaCollector::new(DeltaMode::Cells);
23235 engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
23236 let values = targets
23237 .iter()
23238 .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
23239 .collect();
23240 Ok((values, collector.finish_target()))
23241 })
23242 }
23243
23244 pub fn evaluate_cells_with_delta(
23245 &mut self,
23246 targets: &[(&str, u32, u32)],
23247 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23248 self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
23249 }
23250
23251 pub fn evaluate_cells_with_delta_policy(
23252 &mut self,
23253 targets: &[(&str, u32, u32)],
23254 policy: EvalDeltaCompatibilityPolicy,
23255 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23256 self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
23257 engine.evaluate_cells_with_delta_unobserved(targets, policy)
23258 })
23259 }
23260
23261 fn evaluate_cells_with_delta_unobserved(
23262 &mut self,
23263 targets: &[(&str, u32, u32)],
23264 policy: EvalDeltaCompatibilityPolicy,
23265 ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23266 self.observe_function_semantic_epoch()?;
23267 self.validate_deterministic_mode()?;
23268 if targets.is_empty() {
23269 return Ok((Vec::new(), EvalDelta::default()));
23270 }
23271 let mut collector = DeltaCollector::new(DeltaMode::Cells);
23272 self.evaluate_until_with_delta_collector(targets, &mut collector)?;
23273 let values = targets
23274 .iter()
23275 .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
23276 .collect();
23277 Ok((values, collector.finish_with_policy(policy)?))
23278 }
23279
23280 pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
23282 if targets.is_empty() {
23283 return Ok(EvalPlan {
23284 total_vertices_to_evaluate: 0,
23285 layers: Vec::new(),
23286 cycles_detected: 0,
23287 dirty_count: 0,
23288 volatile_count: 0,
23289 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23290 estimated_parallel_layers: 0,
23291 target_cells: Vec::new(),
23292 });
23293 }
23294 if self.config.defer_graph_building && self.has_staged_formulas() {
23295 return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
23296 "Evaluation plan requested with deferred graph; build first or call evaluate_*",
23297 ));
23298 }
23299
23300 let addresses: Vec<String> = targets
23302 .iter()
23303 .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
23304 .collect();
23305
23306 let mut target_addrs = Vec::new();
23308 for (sheet, row, col) in targets {
23309 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
23310 let coord = Coord::from_excel(*row, *col, true, true);
23311 target_addrs.push(CellRef::new(sheet_id, coord));
23312 }
23313 }
23314
23315 let mut target_vertex_ids = Vec::new();
23317 for addr in &target_addrs {
23318 if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
23319 target_vertex_ids.push(*vertex_id);
23320 }
23321 }
23322
23323 if target_vertex_ids.is_empty() {
23324 return Ok(EvalPlan {
23325 total_vertices_to_evaluate: 0,
23326 layers: Vec::new(),
23327 cycles_detected: 0,
23328 dirty_count: 0,
23329 volatile_count: 0,
23330 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23331 estimated_parallel_layers: 0,
23332 target_cells: addresses,
23333 });
23334 }
23335
23336 let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
23338
23339 if precedents_to_eval.is_empty() {
23340 return Ok(EvalPlan {
23341 total_vertices_to_evaluate: 0,
23342 layers: Vec::new(),
23343 cycles_detected: 0,
23344 dirty_count: 0,
23345 volatile_count: 0,
23346 parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23347 estimated_parallel_layers: 0,
23348 target_cells: addresses,
23349 });
23350 }
23351
23352 let mut dirty_count = 0;
23354 let mut volatile_count = 0;
23355 for &vertex_id in &precedents_to_eval {
23356 if self.graph.is_dirty(vertex_id) {
23357 dirty_count += 1;
23358 }
23359 if self.graph.is_volatile(vertex_id) {
23360 volatile_count += 1;
23361 }
23362 }
23363
23364 let scheduler = Scheduler::new(&self.graph);
23366 let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
23367
23368 let mut layers = Vec::new();
23370 let mut estimated_parallel_layers = 0;
23371 let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
23372
23373 for layer in &schedule.layers {
23374 let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
23375 if parallel_eligible {
23376 estimated_parallel_layers += 1;
23377 }
23378
23379 let sample_cells: Vec<String> = layer
23381 .vertices
23382 .iter()
23383 .take(5)
23384 .filter_map(|&vertex_id| {
23385 self.graph
23386 .get_cell_ref_for_vertex(vertex_id)
23387 .map(|cell_ref| {
23388 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23389 format!(
23390 "{}!{}{}",
23391 sheet_name,
23392 Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
23393 cell_ref.coord.row() + 1
23394 )
23395 })
23396 })
23397 .collect();
23398
23399 layers.push(LayerInfo {
23400 vertex_count: layer.vertices.len(),
23401 parallel_eligible,
23402 sample_cells,
23403 });
23404 }
23405
23406 Ok(EvalPlan {
23407 total_vertices_to_evaluate: precedents_to_eval.len(),
23408 layers,
23409 cycles_detected: schedule.cycles.len(),
23410 dirty_count,
23411 volatile_count,
23412 parallel_enabled,
23413 estimated_parallel_layers,
23414 target_cells: addresses,
23415 })
23416 }
23417 fn create_evaluation_schedule(
23419 &mut self,
23420 to_evaluate: &[VertexId],
23421 ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
23422 #[cfg(any(test, feature = "benchmark_internal"))]
23423 {
23424 self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
23425 }
23426 self.graph.flush_pending_edge_deltas();
23429 if self.can_use_static_schedule_cache(to_evaluate) {
23430 if let Some(cached) = self.cached_static_schedule.as_ref()
23431 && cached.topology_epoch == self.topology_epoch
23432 && cached.candidate_vertices.as_slice() == to_evaluate
23433 {
23434 let meta = ScheduleBuildMeta {
23435 candidate_vertices: to_evaluate.len(),
23436 vdeps_vertices: 0,
23437 vdeps_edges: 0,
23438 builder_elapsed_ms: 0,
23439 used_virtual_schedule: false,
23440 schedule_cache_hit: true,
23441 schedule_cache_eligible: true,
23442 };
23443 #[cfg(any(test, feature = "benchmark_internal"))]
23444 {
23445 let mut probe = self.recalc_reuse_probe.lock().unwrap();
23446 probe.schedule_cache_hits += 1;
23447 probe.schedule_shared_handles += 1;
23448 }
23449 return Ok((
23450 EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
23451 FxHashMap::default(),
23452 meta,
23453 ));
23454 }
23455
23456 let (schedule, vdeps, mut meta) =
23457 self.create_evaluation_schedule_uncached(to_evaluate)?;
23458 meta.schedule_cache_hit = false;
23459 meta.schedule_cache_eligible = true;
23460 #[cfg(any(test, feature = "benchmark_internal"))]
23461 {
23462 self.recalc_reuse_probe
23463 .get_mut()
23464 .unwrap()
23465 .schedule_cache_misses += 1;
23466 }
23467 let schedule = if vdeps.is_empty() {
23468 let mut schedule = schedule;
23471 schedule.units.shrink_to_fit();
23472 for layer in &mut schedule.layers {
23473 layer.vertices.shrink_to_fit();
23474 }
23475 schedule.layers.shrink_to_fit();
23476 for cycle in &mut schedule.cycles {
23477 cycle.shrink_to_fit();
23478 }
23479 schedule.cycles.shrink_to_fit();
23480 let schedule = Arc::new(schedule);
23481 #[cfg(any(test, feature = "benchmark_internal"))]
23482 {
23483 self.recalc_reuse_probe
23484 .get_mut()
23485 .unwrap()
23486 .schedule_shared_handles += 1;
23487 }
23488 self.cached_static_schedule = Some(CachedScheduleEntry {
23489 topology_epoch: self.topology_epoch,
23490 candidate_vertices: to_evaluate.to_vec(),
23491 schedule: Arc::clone(&schedule),
23492 });
23493 EvaluationSchedule::Shared(schedule)
23494 } else {
23495 EvaluationSchedule::Owned(schedule)
23496 };
23497 return Ok((schedule, vdeps, meta));
23498 }
23499
23500 let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
23501 meta.schedule_cache_hit = false;
23502 meta.schedule_cache_eligible = false;
23503 #[cfg(any(test, feature = "benchmark_internal"))]
23504 {
23505 self.recalc_reuse_probe
23506 .get_mut()
23507 .unwrap()
23508 .schedule_cache_ineligible += 1;
23509 }
23510 Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
23511 }
23512
23513 fn create_evaluation_schedule_uncached(
23514 &self,
23515 to_evaluate: &[VertexId],
23516 ) -> Result<ScheduleBuildOutput, ExcelError> {
23517 #[cfg(any(test, feature = "benchmark_internal"))]
23518 {
23519 self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
23520 }
23521 let builder = VirtualDepBuilder::new(self);
23522 let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
23523 if self.config.enable_virtual_dep_telemetry {
23524 let build_started = crate::instant::FzInstant::now();
23525 let (vdeps, augmented) = builder.build(to_evaluate);
23526 let builder_elapsed_ms = build_started.elapsed().as_millis();
23527 let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
23528 (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
23529 } else {
23530 let (vdeps, augmented) = builder.build(to_evaluate);
23531 (vdeps, augmented, 0, 0)
23532 };
23533
23534 let mut final_evaluate = to_evaluate.to_vec();
23535 if !augmented.is_empty() {
23536 final_evaluate.extend(augmented);
23537 final_evaluate.sort_unstable();
23538 final_evaluate.dedup();
23539 }
23540
23541 let use_virtual = !vdeps.is_empty();
23542
23543 let scheduler = Scheduler::new(&self.graph);
23544 let schedule = if use_virtual {
23545 scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
23546 } else {
23547 scheduler.create_schedule(&final_evaluate)?
23548 };
23549
23550 let meta = ScheduleBuildMeta {
23551 candidate_vertices: to_evaluate.len(),
23552 vdeps_vertices: vdeps.len(),
23553 vdeps_edges,
23554 builder_elapsed_ms,
23555 used_virtual_schedule: use_virtual,
23556 schedule_cache_hit: false,
23557 schedule_cache_eligible: false,
23558 };
23559
23560 Ok((schedule, vdeps, meta))
23561 }
23562
23563 fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
23564 !to_evaluate.is_empty()
23565 && to_evaluate.iter().copied().all(|v| {
23566 !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
23567 })
23568 }
23569
23570 fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
23571 VirtualDepTelemetry {
23572 fallback_mode_activations: self.virtual_dep_fallback_activations,
23573 ..VirtualDepTelemetry::default()
23574 }
23575 }
23576
23577 fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
23578 telemetry.candidate_vertices_total += meta.candidate_vertices;
23579 telemetry.vdeps_vertices_total += meta.vdeps_vertices;
23580 telemetry.vdeps_edges_total += meta.vdeps_edges;
23581 telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
23582 if meta.schedule_cache_eligible {
23583 if meta.schedule_cache_hit {
23584 telemetry.schedule_cache_hits += 1;
23585 telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
23586 } else {
23587 telemetry.schedule_cache_misses += 1;
23588 }
23589 }
23590 if meta.used_virtual_schedule {
23591 telemetry.schedule_virtual_passes += 1;
23592 } else {
23593 telemetry.schedule_static_passes += 1;
23594 }
23595 }
23596
23597 fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
23598 vertices
23599 .iter()
23600 .copied()
23601 .filter(|vertex| self.graph.is_dynamic(*vertex))
23602 .filter_map(|vertex| {
23603 let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
23604 (!regions.is_empty()).then_some((vertex, regions))
23605 })
23606 .collect()
23607 }
23608
23609 fn extend_target_roots_with_dynamic_regions<'a>(
23610 &self,
23611 roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
23612 regions: impl Iterator<Item = &'a Region>,
23613 ) -> Result<(), ExcelError> {
23614 use crate::engine::target_preparation::TargetProducer;
23615 let request_id = self
23616 .active_evaluation_resource_request
23617 .as_ref()
23618 .map(|request| request.request_id);
23619 let root_count = roots.len();
23620 let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
23621 .map_err(|_| target_root_allocation_error(root_count, request_id))?;
23622 for region in regions {
23623 for vertex in self.graph.formula_vertices() {
23624 let Some(position) = self.graph.vertex_grid_addr(vertex) else {
23625 continue;
23626 };
23627 let key = crate::formula_plane::region_index::RegionKey {
23628 sheet_id: self.graph.get_vertex_sheet_id(vertex),
23629 row: position.row(),
23630 col: position.col(),
23631 };
23632 if region.contains_key(key) {
23633 extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
23634 target_root_allocation_error(extended.len() + 1, request_id)
23635 })?;
23636 }
23637 }
23638 let authority = self.graph.formula_authority();
23639 for span_ref in authority.active_span_refs() {
23640 let Some(span) = authority.plane.spans.get(span_ref) else {
23641 continue;
23642 };
23643 let Some(demanded) =
23644 Region::from_domain(span.result_region.domain()).intersection(*region)
23645 else {
23646 continue;
23647 };
23648 extended
23649 .push(TargetProducer::Span { span_ref, demanded })
23650 .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
23651 }
23652 }
23653 *roots = extended.into_vec();
23654 Ok(())
23655 }
23656
23657 fn changed_virtual_dep_vertices(
23658 &mut self,
23659 to_evaluate: &[VertexId],
23660 old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
23661 ) -> Vec<VertexId> {
23662 #[cfg(test)]
23663 if self.force_virtual_dep_changes_remaining_for_test > 0
23664 && let Some(vertex) = to_evaluate.first().copied()
23665 {
23666 self.force_virtual_dep_changes_remaining_for_test -= 1;
23667 return vec![vertex];
23668 }
23669 if !to_evaluate
23670 .iter()
23671 .copied()
23672 .any(|v| self.graph.is_dynamic(v))
23673 {
23674 return Vec::new();
23675 }
23676
23677 let builder = VirtualDepBuilder::new(self);
23678 let (new_vdeps, _) = builder.build(to_evaluate);
23679
23680 let mut candidates = FxHashSet::default();
23681 candidates.extend(old_vdeps.keys().copied());
23682 candidates.extend(new_vdeps.keys().copied());
23683
23684 let mut changed = Vec::new();
23685 for v in candidates {
23686 if old_vdeps.get(&v) != new_vdeps.get(&v) {
23687 changed.push(v);
23688 }
23689 }
23690 changed
23691 }
23692
23693 fn build_demand_subgraph(
23696 &self,
23697 target_vertices: &[VertexId],
23698 ) -> (
23699 Vec<VertexId>,
23700 rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
23701 ) {
23702 #[cfg(feature = "tracing")]
23703 let _span =
23704 tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
23705 use rustc_hash::{FxHashMap, FxHashSet};
23706
23707 let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
23708 let mut visited: FxHashSet<VertexId> = FxHashSet::default();
23709 let mut stack: Vec<VertexId> = Vec::new();
23710 let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); #[cfg(any(test, feature = "benchmark_internal"))]
23712 let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges) = (0, 0, 0);
23713
23714 for &t in target_vertices {
23715 stack.push(t);
23716 }
23717
23718 while let Some(v) = stack.pop() {
23719 if !visited.insert(v) {
23720 continue;
23721 }
23722 if !self.graph.vertex_exists(v) {
23723 continue;
23724 }
23725 #[cfg(any(test, feature = "benchmark_internal"))]
23726 {
23727 probe_vertices += 1;
23728 if matches!(
23729 self.graph.get_vertex_kind(v),
23730 VertexKind::FormulaScalar | VertexKind::FormulaArray
23731 ) && !self.graph.is_dirty(v)
23732 && !self.graph.is_volatile(v)
23733 {
23734 probe_clean_formulas += 1;
23735 }
23736 }
23737 match self.graph.get_vertex_kind(v) {
23745 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
23746 if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
23747 to_evaluate.insert(v);
23748 }
23749 }
23750 VertexKind::NamedScalar
23751 | VertexKind::NamedArray
23752 | VertexKind::Range
23753 | VertexKind::InfiniteRange => {
23754 to_evaluate.insert(v);
23755 }
23756 _ => {}
23757 }
23758
23759 if let Some(dependencies) = self.graph.dependencies_slice(v) {
23770 for &dep in dependencies {
23771 #[cfg(any(test, feature = "benchmark_internal"))]
23772 {
23773 probe_edges += 1;
23774 }
23775 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
23776 stack.push(dep);
23777 }
23778 }
23779 } else {
23780 for dep in self.graph.get_dependencies(v) {
23781 #[cfg(any(test, feature = "benchmark_internal"))]
23782 {
23783 probe_edges += 1;
23784 }
23785 if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
23786 stack.push(dep);
23787 }
23788 }
23789 } let builder = VirtualDepBuilder::new(self);
23791 let (vdeps_map, _) = builder.build(&[v]);
23792 if let Some(deps) = vdeps_map.get(&v) {
23793 for &u in deps {
23794 vdeps.entry(v).or_default().push(u);
23795 if !visited.contains(&u) {
23796 stack.push(u);
23797 }
23798 }
23799 }
23800 }
23801
23802 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
23803 result.sort_unstable();
23804 for deps in vdeps.values_mut() {
23806 deps.sort_unstable();
23807 deps.dedup();
23808 }
23809 #[cfg(any(test, feature = "benchmark_internal"))]
23810 {
23811 let mut probe = self.recalc_reuse_probe.lock().unwrap();
23812 probe.demand_builds += 1;
23813 probe.demand_vertices += probe_vertices;
23814 probe.demand_clean_formulas += probe_clean_formulas;
23815 probe.demand_explicit_edges += probe_edges;
23816 probe.demand_virtual_builder_calls += probe_vertices;
23817 }
23818 (result, vdeps)
23819 }
23820
23821 fn col_to_letters(col: u32) -> String {
23823 col_letters_from_1based(col).expect("column index must be >= 1")
23824 }
23825
23826 pub fn evaluate_all_cancellable(
23828 &mut self,
23829 cancel: crate::engine::CancelToken,
23830 ) -> Result<EvalResult, ExcelError> {
23831 self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
23832 engine.observe_function_semantic_epoch()?;
23833 engine.active_cancel_flag = Some(cancel.clone());
23834 let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
23835 engine.active_cancel_flag = None;
23836 res
23837 })
23838 }
23839
23840 fn evaluate_all_cancellable_impl(
23841 &mut self,
23842 cancel_flag: &AtomicBool,
23843 ) -> Result<EvalResult, ExcelError> {
23844 let _source_cache = self.source_cache_session();
23845 self.validate_deterministic_mode()?;
23846 if self.config.defer_graph_building {
23847 self.build_graph_all()?;
23848 }
23849 if cancel_flag.load(Ordering::Relaxed) {
23850 let message = if self.config.formula_plane_mode
23851 == FormulaPlaneMode::AuthoritativeExperimental
23852 && self.graph.formula_authority().active_span_count() > 0
23853 {
23854 "Evaluation cancelled before FormulaPlane scheduling"
23855 } else {
23856 "Evaluation cancelled before scheduling"
23857 };
23858 return Err(
23859 ExcelError::new(ExcelErrorKind::Cancelled).with_message(message.to_string())
23860 );
23861 }
23862 self.transition_off_mode_spans_to_legacy()?;
23863 self.begin_evaluation_request();
23864 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
23865 && self.graph.formula_authority().active_span_count() > 0
23866 {
23867 if cancel_flag.load(Ordering::Relaxed) {
23868 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
23869 "Evaluation cancelled before FormulaPlane scheduling".to_string(),
23870 ));
23871 }
23872 return self.evaluate_authoritative_formula_plane_all();
23873 }
23874 self.reset_virtual_dep_telemetry_if_disabled();
23875 let start = crate::instant::FzInstant::now();
23876 let mut computed_vertices = 0;
23877 let mut cycle_errors = 0;
23878
23879 let mut replan_iterations = 0;
23880 const MAX_REPLAN: usize = 5;
23881 let mut telemetry = self
23882 .config
23883 .enable_virtual_dep_telemetry
23884 .then(|| self.start_virtual_dep_telemetry());
23885
23886 loop {
23887 if cancel_flag.load(Ordering::Relaxed) {
23888 if let Some(mut t) = telemetry {
23889 t.bailout_reason = Some("cancelled");
23890 t.replan_iterations = replan_iterations;
23891 self.last_virtual_dep_telemetry = t;
23892 }
23893 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23894 .with_message("Evaluation cancelled before scheduling".to_string()));
23895 }
23896
23897 let to_evaluate = self.graph.get_evaluation_vertices();
23898 if to_evaluate.is_empty() {
23899 if let Some(t) = telemetry.as_mut()
23900 && t.bailout_reason.is_none()
23901 {
23902 t.bailout_reason = Some("no_work");
23903 }
23904 break;
23905 }
23906
23907 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
23908 if let Some(t) = telemetry.as_mut() {
23909 Self::accumulate_schedule_meta(t, &meta);
23910 }
23911
23912 for &unit in &schedule.units {
23915 match unit {
23916 ScheduleUnit::Cycle(i) => {
23917 if cancel_flag.load(Ordering::Relaxed) {
23919 if let Some(mut t) = telemetry {
23920 t.bailout_reason = Some("cancelled");
23921 t.replan_iterations = replan_iterations;
23922 self.last_virtual_dep_telemetry = t;
23923 }
23924 return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
23925 "Evaluation cancelled during cycle handling".to_string(),
23926 ));
23927 }
23928
23929 if self.handle_cycle_unit(
23930 schedule.unit_cycle(i),
23931 None,
23932 None,
23933 Some(cancel_flag),
23934 )? > 0
23935 {
23936 cycle_errors += 1;
23937 }
23938 }
23939 ScheduleUnit::Layer(i) => {
23940 let layer = schedule.unit_layer(i);
23941 if cancel_flag.load(Ordering::Relaxed) {
23943 if let Some(mut t) = telemetry {
23944 t.bailout_reason = Some("cancelled");
23945 t.replan_iterations = replan_iterations;
23946 self.last_virtual_dep_telemetry = t;
23947 }
23948 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23949 .with_message("Evaluation cancelled between layers".to_string()));
23950 }
23951
23952 if self.thread_pool.is_some() && layer.vertices.len() > 1 {
23954 computed_vertices +=
23955 self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
23956 } else {
23957 computed_vertices +=
23958 self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
23959 }
23960 }
23961 }
23962 }
23963
23964 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
23965 if let Some(t) = telemetry.as_mut() {
23966 t.changed_vdeps_total += changed_vertices.len();
23967 }
23968 self.resource_checkpoint(0)?;
23969 self.graph.clear_dirty_flags(&to_evaluate);
23970 for v in &changed_vertices {
23971 self.graph.set_dirty(*v, true);
23972 }
23973
23974 if changed_vertices.is_empty() {
23975 if let Some(t) = telemetry.as_mut() {
23976 t.bailout_reason = Some("converged");
23977 }
23978 break;
23979 }
23980 if replan_iterations >= MAX_REPLAN {
23981 if let Some(mut t) = telemetry.take() {
23982 t.bailout_reason = Some("max_replan");
23983 t.replan_iterations = replan_iterations;
23984 self.last_virtual_dep_telemetry = t;
23985 }
23986 return Err(
23987 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
23988 );
23989 }
23990 replan_iterations += 1;
23991 }
23992
23993 if let Some(mut t) = telemetry {
23994 t.replan_iterations = replan_iterations;
23995 self.last_virtual_dep_telemetry = t;
23996 }
23997
23998 self.redirty_for_next_recalc();
24000 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
24001
24002 Ok(EvalResult {
24003 computed_vertices,
24004 cycle_errors,
24005 elapsed: start.elapsed(),
24006 })
24007 }
24008
24009 pub fn evaluate_until_cancellable(
24011 &mut self,
24012 targets: &[&str],
24013 cancel: crate::engine::CancelToken,
24014 ) -> Result<EvalResult, ExcelError> {
24015 self.observe_evaluation_resource_request(
24016 EvaluationRequestKind::TargetedCancellable,
24017 |engine| {
24018 engine.observe_function_semantic_epoch()?;
24019 engine.active_cancel_flag = Some(cancel.clone());
24020 let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
24021 engine.active_cancel_flag = None;
24022 res
24023 },
24024 )
24025 }
24026
24027 fn evaluate_until_cancellable_impl(
24028 &mut self,
24029 targets: &[&str],
24030 cancel_flag: &AtomicBool,
24031 ) -> Result<EvalResult, ExcelError> {
24032 if cancel_flag.load(Ordering::Relaxed) {
24033 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24034 .with_message("Evaluation cancelled before target preparation"));
24035 }
24036 let mut typed_targets = Vec::with_capacity(targets.len());
24037 for target in targets {
24038 let (sheet, row, col) = self.parse_a1_notation(target)?;
24039 self.graph.sheet_id_mut(&sheet);
24040 typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
24041 }
24042 self.evaluate_mixed_targets(&typed_targets, None)
24043 }
24044
24045 fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
24046 let mut quoted = false;
24047 let mut separator = None;
24048 let bytes = address.as_bytes();
24049 let mut index = 0usize;
24050 while index < bytes.len() {
24051 match bytes[index] {
24052 b'\'' => {
24053 if quoted && bytes.get(index + 1) == Some(&b'\'') {
24054 index = index.saturating_add(1);
24055 } else {
24056 quoted = !quoted;
24057 }
24058 }
24059 b'!' if !quoted => separator = Some(index),
24060 _ => {}
24061 }
24062 index = index.saturating_add(1);
24063 }
24064 if quoted {
24065 return Err(ExcelError::new(ExcelErrorKind::Ref)
24066 .with_message(format!("Invalid quoted sheet reference `{address}`")));
24067 }
24068 let (sheet, cell_part) = match separator {
24069 Some(separator) => {
24070 let raw_sheet = &address[..separator];
24071 let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
24072 raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
24073 } else {
24074 raw_sheet.to_string()
24075 };
24076 (sheet, &address[separator + 1..])
24077 }
24078 None => (self.default_sheet_name().to_string(), address),
24079 };
24080
24081 let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
24082 ExcelError::new(ExcelErrorKind::Ref)
24083 .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
24084 })?;
24085
24086 Ok((sheet, row, col))
24087 }
24088
24089 fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
24091 use formualizer_parse::parser::ASTNodeType;
24092 match &ast.node_type {
24093 ASTNodeType::Function { name, args, .. } => {
24094 if let Some(func) = self
24095 .get_function("", name)
24096 .or_else(|| crate::function_registry::get("", name))
24097 && func.caps().contains(crate::function::FnCaps::VOLATILE)
24098 {
24099 return true;
24100 }
24101 args.iter()
24102 .any(|arg| self.is_ast_volatile_with_provider(arg))
24103 }
24104 ASTNodeType::BinaryOp { left, right, .. } => {
24105 self.is_ast_volatile_with_provider(left)
24106 || self.is_ast_volatile_with_provider(right)
24107 }
24108 ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
24109 ASTNodeType::Array(rows) => rows.iter().any(|row| {
24110 row.iter()
24111 .any(|cell| self.is_ast_volatile_with_provider(cell))
24112 }),
24113 _ => false,
24114 }
24115 }
24116
24117 fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
24119 let mut to_evaluate = FxHashSet::default();
24120 let mut visited = FxHashSet::default();
24121 let mut stack = Vec::new();
24122
24123 for &target in target_vertices {
24125 stack.push(target);
24126 }
24127
24128 while let Some(vertex_id) = stack.pop() {
24129 if !visited.insert(vertex_id) {
24130 continue; }
24132
24133 if self.graph.vertex_exists(vertex_id) {
24134 let kind = self.graph.get_vertex_kind(vertex_id);
24136 let needs_eval = match kind {
24137 super::vertex::VertexKind::FormulaScalar
24138 | super::vertex::VertexKind::FormulaArray => {
24139 self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
24140 }
24141 _ => false, };
24143
24144 if needs_eval {
24145 to_evaluate.insert(vertex_id);
24146 }
24147
24148 if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
24150 for &dep_id in dependencies {
24151 if !visited.contains(&dep_id) {
24152 stack.push(dep_id);
24153 }
24154 }
24155 } else {
24156 let dependencies = self.graph.get_dependencies(vertex_id);
24157 for dep_id in dependencies {
24158 if !visited.contains(&dep_id) {
24159 stack.push(dep_id);
24160 }
24161 }
24162 }
24163 }
24164 }
24165
24166 let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
24167 result.sort_unstable();
24168 result
24169 }
24170
24171 fn evaluate_layer_sequential(
24173 &mut self,
24174 layer: &super::scheduler::Layer,
24175 ) -> Result<usize, ExcelError> {
24176 self.resource_checkpoint(layer.vertices.len() as u64)?;
24177 self.evaluate_layer_sequential_effects(layer)
24178 }
24179
24180 fn update_vertex_value_with_delta(
24181 &mut self,
24182 vertex_id: VertexId,
24183 new_value: LiteralValue,
24184 delta: &mut DeltaCollector,
24185 ) {
24186 if delta.mode != DeltaMode::Off
24187 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24188 {
24189 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24190 let old = self
24191 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24192 .unwrap_or(LiteralValue::Empty);
24193 if old != new_value {
24194 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24195 }
24196 }
24197 self.graph.update_vertex_value(vertex_id, new_value.clone());
24198 self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
24199 }
24200
24201 fn evaluate_layer_sequential_with_delta(
24202 &mut self,
24203 layer: &super::scheduler::Layer,
24204 delta: &mut DeltaCollector,
24205 ) -> Result<usize, ExcelError> {
24206 self.resource_checkpoint(layer.vertices.len() as u64)?;
24207 self.evaluate_layer_sequential_with_delta_effects(layer, delta)
24208 }
24209
24210 fn evaluate_layer_sequential_cancellable(
24212 &mut self,
24213 layer: &super::scheduler::Layer,
24214 cancel_flag: &AtomicBool,
24215 ) -> Result<usize, ExcelError> {
24216 self.resource_checkpoint(layer.vertices.len() as u64)?;
24217 self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
24218 }
24219
24220 fn evaluate_layer_sequential_cancellable_demand_driven(
24222 &mut self,
24223 layer: &super::scheduler::Layer,
24224 cancel_flag: &AtomicBool,
24225 ) -> Result<usize, ExcelError> {
24226 self.resource_checkpoint(layer.vertices.len() as u64)?;
24227 self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
24228 }
24229
24230 fn evaluate_layer_parallel(
24232 &mut self,
24233 layer: &super::scheduler::Layer,
24234 ) -> Result<usize, ExcelError> {
24235 self.resource_checkpoint(layer.vertices.len() as u64)?;
24236 self.evaluate_layer_parallel_effects(layer)
24237 }
24238
24239 fn evaluate_layer_parallel_with_delta(
24240 &mut self,
24241 layer: &super::scheduler::Layer,
24242 delta: &mut DeltaCollector,
24243 ) -> Result<usize, ExcelError> {
24244 self.resource_checkpoint(layer.vertices.len() as u64)?;
24245 self.evaluate_layer_parallel_with_delta_effects(layer, delta)
24246 }
24247
24248 fn evaluate_layer_parallel_cancellable(
24250 &mut self,
24251 layer: &super::scheduler::Layer,
24252 cancel_flag: &AtomicBool,
24253 ) -> Result<usize, ExcelError> {
24254 self.resource_checkpoint(layer.vertices.len() as u64)?;
24255 self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
24256 }
24257
24258 fn apply_parallel_vertex_result(
24263 &mut self,
24264 vertex_id: VertexId,
24265 result: LiteralValue,
24266 mut delta: Option<&mut DeltaCollector>,
24267 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24268 ) -> Result<(), ExcelError> {
24269 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
24272 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
24273 && owner != vertex_id
24274 {
24275 return Ok(());
24276 }
24277
24278 let kind = self.graph.get_vertex_kind(vertex_id);
24279
24280 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
24282 if is_formula {
24283 let derived_format = self
24284 .derived_format_results
24285 .write()
24286 .unwrap()
24287 .remove(&vertex_id)
24288 .flatten();
24289 if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
24290 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24291 self.write_computed_overlay_format_0based(
24292 &sheet_name,
24293 cell.coord.row(),
24294 cell.coord.col(),
24295 derived_format,
24296 );
24297 }
24298 match result {
24299 LiteralValue::Array(rows) => {
24300 self.apply_array_result_from_parallel(
24301 vertex_id,
24302 rows,
24303 delta.as_deref_mut(),
24304 overwritable_formulas,
24305 )?;
24306 }
24307 other => {
24308 self.apply_non_array_result_from_parallel(
24309 vertex_id,
24310 other,
24311 delta.as_deref_mut(),
24312 );
24313 }
24314 }
24315 return Ok(());
24316 }
24317
24318 if let Some(d) = delta {
24320 self.update_vertex_value_with_delta(vertex_id, result, d);
24321 } else {
24322 self.graph.update_vertex_value(vertex_id, result.clone());
24323 self.mirror_vertex_value_to_overlay(vertex_id, &result);
24324 }
24325 Ok(())
24326 }
24327
24328 fn apply_non_array_result_from_parallel(
24329 &mut self,
24330 vertex_id: VertexId,
24331 value: LiteralValue,
24332 delta: Option<&mut DeltaCollector>,
24333 ) {
24334 let spill_cells = self
24337 .graph
24338 .spill_cells_for_anchor(vertex_id)
24339 .map(|cells| cells.to_vec())
24340 .unwrap_or_default();
24341
24342 if let Some(d) = delta
24343 && d.mode != DeltaMode::Off
24344 && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
24345 {
24346 if spill_cells.is_empty() {
24347 let old = self
24348 .read_cell_value(
24349 self.graph.sheet_name(anchor.sheet_id),
24350 anchor.coord.row() + 1,
24351 anchor.coord.col() + 1,
24352 )
24353 .unwrap_or(LiteralValue::Empty);
24354 if old != value {
24355 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24356 }
24357 } else {
24358 for cell in spill_cells.iter() {
24359 let sheet_name = self.graph.sheet_name(cell.sheet_id);
24360 let old = self
24361 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24362 .unwrap_or(LiteralValue::Empty);
24363 let new = if cell.sheet_id == anchor.sheet_id
24364 && cell.coord.row() == anchor.coord.row()
24365 && cell.coord.col() == anchor.coord.col()
24366 {
24367 value.clone()
24368 } else {
24369 LiteralValue::Empty
24370 };
24371 Self::record_cell_if_changed(d, cell, &old, &new);
24372 }
24373 }
24374 }
24375
24376 self.graph.clear_spill_region(vertex_id);
24377 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24378 self.record_formula_plane_structural_change(scope);
24379 }
24380
24381 if self.config.arrow_storage_enabled
24382 && self.config.delta_overlay_enabled
24383 && self.config.write_formula_overlay_enabled
24384 {
24385 let empty = LiteralValue::Empty;
24386 for cell in spill_cells.iter() {
24387 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24388 self.mirror_value_to_computed_overlay(
24389 &sheet_name,
24390 cell.coord.row() + 1,
24391 cell.coord.col() + 1,
24392 &empty,
24393 );
24394 }
24395 }
24396
24397 self.graph.update_vertex_value(vertex_id, value.clone());
24398 self.mirror_vertex_value_to_overlay(vertex_id, &value);
24399 }
24400
24401 fn apply_array_result_from_parallel(
24402 &mut self,
24403 vertex_id: VertexId,
24404 rows: Vec<Vec<LiteralValue>>,
24405 mut delta: Option<&mut DeltaCollector>,
24406 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24407 ) -> Result<(), ExcelError> {
24408 self.graph
24410 .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
24411
24412 let anchor = self
24413 .graph
24414 .get_cell_ref(vertex_id)
24415 .expect("cell ref for vertex");
24416 let sheet_id = anchor.sheet_id;
24417 let h = rows.len() as u32;
24418 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
24419
24420 let spill_cells = (h as u64).saturating_mul(w as u64);
24422 if spill_cells > self.config.spill.max_spill_cells as u64 {
24423 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24424 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24425 .with_message("SpillTooLarge")
24426 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24427 expected_rows: h,
24428 expected_cols: w,
24429 });
24430 let spill_val = LiteralValue::Error(spill_err.clone());
24431 if let Some(d) = delta.as_deref_mut()
24432 && d.mode != DeltaMode::Off
24433 {
24434 let old = self
24435 .read_cell_value(
24436 self.graph.sheet_name(anchor.sheet_id),
24437 anchor.coord.row() + 1,
24438 anchor.coord.col() + 1,
24439 )
24440 .unwrap_or(LiteralValue::Empty);
24441 if old != spill_val {
24442 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24443 }
24444 }
24445 self.graph.update_vertex_value(vertex_id, spill_val.clone());
24446 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24447 return Ok(());
24448 }
24449
24450 const PACKED_MAX_ROW: u32 = 1_048_575; const PACKED_MAX_COL: u32 = 16_383; let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
24454 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
24455 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
24456 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24457 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24458 .with_message("Spill exceeds sheet bounds")
24459 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24460 expected_rows: h,
24461 expected_cols: w,
24462 });
24463 let spill_val = LiteralValue::Error(spill_err.clone());
24464 if let Some(d) = delta.as_deref_mut()
24465 && d.mode != DeltaMode::Off
24466 {
24467 let old = self
24468 .read_cell_value(
24469 self.graph.sheet_name(anchor.sheet_id),
24470 anchor.coord.row() + 1,
24471 anchor.coord.col() + 1,
24472 )
24473 .unwrap_or(LiteralValue::Empty);
24474 if old != spill_val {
24475 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24476 }
24477 }
24478 self.graph.update_vertex_value(vertex_id, spill_val.clone());
24479 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24480 return Ok(());
24481 }
24482
24483 let mut targets = Vec::new();
24484 for r in 0..h {
24485 for c in 0..w {
24486 targets.push(self.graph.make_cell_ref_internal(
24487 sheet_id,
24488 anchor.coord.row() + r,
24489 anchor.coord.col() + c,
24490 ));
24491 }
24492 }
24493
24494 match self.spill_mgr.reserve(
24495 vertex_id,
24496 anchor,
24497 SpillShape { rows: h, cols: w },
24498 SpillMeta {
24499 epoch: self.recalc_epoch,
24500 config: self.config.spill,
24501 },
24502 ) {
24503 Ok(()) => {
24504 if let Err(e) = self.commit_spill_and_mirror(
24505 vertex_id,
24506 &targets,
24507 rows.clone(),
24508 delta.as_deref_mut(),
24509 overwritable_formulas,
24510 ) {
24511 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24512 let err_val = LiteralValue::Error(e.clone());
24513 if let Some(d) = delta.as_deref_mut()
24514 && d.mode != DeltaMode::Off
24515 {
24516 let old = self
24517 .read_cell_value(
24518 self.graph.sheet_name(anchor.sheet_id),
24519 anchor.coord.row() + 1,
24520 anchor.coord.col() + 1,
24521 )
24522 .unwrap_or(LiteralValue::Empty);
24523 if old != err_val {
24524 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24525 }
24526 }
24527 self.graph.update_vertex_value(vertex_id, err_val.clone());
24528 self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
24529 return Ok(());
24530 }
24531
24532 let top_left = rows
24534 .first()
24535 .and_then(|r| r.first())
24536 .cloned()
24537 .unwrap_or(LiteralValue::Empty);
24538 self.graph.update_vertex_value(vertex_id, top_left.clone());
24539 self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
24540 Ok(())
24541 }
24542 Err(e) => {
24543 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24544 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24545 .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
24546 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24547 expected_rows: h,
24548 expected_cols: w,
24549 });
24550 let spill_val = LiteralValue::Error(spill_err.clone());
24551 if let Some(d) = delta
24552 && d.mode != DeltaMode::Off
24553 {
24554 let old = self
24555 .read_cell_value(
24556 self.graph.sheet_name(anchor.sheet_id),
24557 anchor.coord.row() + 1,
24558 anchor.coord.col() + 1,
24559 )
24560 .unwrap_or(LiteralValue::Empty);
24561 if old != spill_val {
24562 d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24563 }
24564 }
24565 self.graph.update_vertex_value(vertex_id, spill_val.clone());
24566 self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24567 Ok(())
24568 }
24569 }
24570 }
24571
24572 fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
24574 if !self.graph.vertex_exists(vertex_id) {
24576 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24577 .with_message(format!("Vertex not found: {vertex_id:?}")));
24578 }
24579
24580 let kind = self.graph.get_vertex_kind(vertex_id);
24582 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24583
24584 let ast_id = match kind {
24585 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24586 if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
24587 ast_id
24588 } else {
24589 return Ok(LiteralValue::Number(0.0));
24590 }
24591 }
24592 VertexKind::Empty | VertexKind::Cell => {
24593 if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
24594 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24595 let row = cell_ref.coord.row() + 1;
24596 let col = cell_ref.coord.col() + 1;
24597 if let Some(v) = self.read_cell_value(sheet_name, row, col) {
24598 return Ok(v);
24599 }
24600 }
24601 return Ok(LiteralValue::Number(0.0));
24602 }
24603 VertexKind::NamedScalar => {
24604 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24605 ExcelError::new(ExcelErrorKind::Name)
24606 .with_message("Named range metadata missing".to_string())
24607 })?;
24608
24609 return match &named_range.definition {
24610 NamedDefinition::Cell(cell_ref) => {
24611 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24612 Ok(self
24613 .get_cell_value(
24614 sheet_name,
24615 cell_ref.coord.row() + 1,
24616 cell_ref.coord.col() + 1,
24617 )
24618 .unwrap_or(LiteralValue::Empty))
24619 }
24620 NamedDefinition::Literal(v) => Ok(v.clone()),
24621 NamedDefinition::Formula { ast, .. } => {
24622 let context_sheet = match named_range.scope {
24623 NameScope::Sheet(id) => id,
24624 NameScope::Workbook => sheet_id,
24625 };
24626 let sheet_name = self.graph.sheet_name(context_sheet);
24627 let cell_ref = self
24628 .graph
24629 .get_cell_ref(vertex_id)
24630 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24631 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24632 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24633 }
24634 NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
24635 .with_message("Range-valued name evaluated as scalar".to_string())),
24636 };
24637 }
24638 VertexKind::NamedArray => {
24639 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24640 ExcelError::new(ExcelErrorKind::Name)
24641 .with_message("Named range metadata missing".to_string())
24642 })?;
24643
24644 return match &named_range.definition {
24645 NamedDefinition::Range(range_ref) => {
24646 if range_ref.start.sheet_id != range_ref.end.sheet_id {
24647 return Err(ExcelError::new(ExcelErrorKind::Ref)
24648 .with_message("Named range cannot span sheets".to_string()));
24649 }
24650 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
24651 let sr0 = range_ref.start.coord.row();
24652 let sc0 = range_ref.start.coord.col();
24653 let er0 = range_ref.end.coord.row();
24654 let ec0 = range_ref.end.coord.col();
24655 if sr0 > er0 || sc0 > ec0 {
24656 return Err(ExcelError::new(ExcelErrorKind::Ref)
24657 .with_message("Invalid named range bounds".to_string()));
24658 }
24659
24660 let h = (er0 - sr0 + 1) as usize;
24661 let w = (ec0 - sc0 + 1) as usize;
24662 let cell_count = (h as u64).saturating_mul(w as u64);
24663 if cell_count > self.config.spill.max_spill_cells as u64 {
24664 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24665 "Named range too large to materialize as an array".to_string(),
24666 ));
24667 }
24668
24669 let mut rows = Vec::with_capacity(h);
24670 for r0 in sr0..=er0 {
24671 let mut row = Vec::with_capacity(w);
24672 for c0 in sc0..=ec0 {
24673 let v = self
24674 .get_cell_value(sheet_name, r0 + 1, c0 + 1)
24675 .unwrap_or(LiteralValue::Empty);
24676 row.push(v);
24677 }
24678 rows.push(row);
24679 }
24680 Ok(LiteralValue::Array(rows))
24681 }
24682 NamedDefinition::Cell(cell_ref) => {
24683 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24684 let row = cell_ref.coord.row() + 1;
24685 let col = cell_ref.coord.col() + 1;
24686 let v = self
24687 .get_cell_value(sheet_name, row, col)
24688 .unwrap_or(LiteralValue::Empty);
24689 Ok(LiteralValue::Array(vec![vec![v]]))
24690 }
24691 NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
24692 NamedDefinition::Formula { ast, .. } => {
24693 let context_sheet = match named_range.scope {
24694 NameScope::Sheet(id) => id,
24695 NameScope::Workbook => sheet_id,
24696 };
24697 let sheet_name = self.graph.sheet_name(context_sheet);
24698 let cell_ref = self
24699 .graph
24700 .get_cell_ref(vertex_id)
24701 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24702 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24703 match interpreter.evaluate_ast(ast) {
24704 Ok(cv) => {
24705 let v = cv.into_literal();
24706 match v {
24707 LiteralValue::Array(_) => Ok(v),
24708 other => Ok(LiteralValue::Array(vec![vec![other]])),
24709 }
24710 }
24711 Err(err) => Ok(LiteralValue::Error(err)),
24712 }
24713 }
24714 };
24715 }
24716 VertexKind::InfiniteRange
24717 | VertexKind::Range
24718 | VertexKind::External
24719 | VertexKind::Table => {
24720 return Ok(LiteralValue::Number(0.0));
24722 }
24723 };
24724
24725 let sheet_name = self.graph.sheet_name(sheet_id);
24727 let cell_ref = self
24728 .graph
24729 .get_cell_ref(vertex_id)
24730 .expect("cell ref for vertex");
24731 let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24732
24733 interpreter
24734 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24735 .map(|cv| {
24736 let format = cv.format_id();
24737 self.derived_format_results
24738 .write()
24739 .unwrap()
24740 .insert(vertex_id, format);
24741 self.record_derived_format(vertex_id, format);
24742 crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
24743 })
24744 }
24745
24746 pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
24748 self.thread_pool.as_ref()
24749 }
24750}
24751
24752#[derive(Default)]
24753struct RowBoundsCache {
24754 snapshot: u64,
24755 map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
24757}
24758
24759impl RowBoundsCache {
24760 fn new(snapshot: u64) -> Self {
24761 Self {
24762 snapshot,
24763 map: Default::default(),
24764 }
24765 }
24766 fn get_row_bounds(
24767 &self,
24768 sheet_id: SheetId,
24769 col_idx: usize,
24770 snapshot: u64,
24771 ) -> Option<(Option<u32>, Option<u32>)> {
24772 if self.snapshot != snapshot {
24773 return None;
24774 }
24775 self.map.get(&(sheet_id as u32, col_idx)).copied()
24776 }
24777 fn put_row_bounds(
24778 &mut self,
24779 sheet_id: SheetId,
24780 col_idx: usize,
24781 snapshot: u64,
24782 bounds: (Option<u32>, Option<u32>),
24783 ) {
24784 if self.snapshot != snapshot {
24785 self.snapshot = snapshot;
24786 self.map.clear();
24787 }
24788 self.map.insert((sheet_id as u32, col_idx), bounds);
24789 }
24790}
24791
24792struct UsedAxisBoundsCache {
24793 snapshot: u64,
24794 row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
24795 col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
24796 #[cfg(test)]
24797 row_hits: std::sync::atomic::AtomicUsize,
24798 #[cfg(test)]
24799 row_misses: std::sync::atomic::AtomicUsize,
24800 #[cfg(test)]
24801 col_hits: std::sync::atomic::AtomicUsize,
24802 #[cfg(test)]
24803 col_misses: std::sync::atomic::AtomicUsize,
24804}
24805
24806impl UsedAxisBoundsCache {
24807 fn new(snapshot: u64) -> Self {
24808 Self {
24809 snapshot,
24810 row_bounds_by_col_span: Default::default(),
24811 col_bounds_by_row_span: Default::default(),
24812 #[cfg(test)]
24813 row_hits: std::sync::atomic::AtomicUsize::new(0),
24814 #[cfg(test)]
24815 row_misses: std::sync::atomic::AtomicUsize::new(0),
24816 #[cfg(test)]
24817 col_hits: std::sync::atomic::AtomicUsize::new(0),
24818 #[cfg(test)]
24819 col_misses: std::sync::atomic::AtomicUsize::new(0),
24820 }
24821 }
24822
24823 fn reset_for_snapshot(&mut self, snapshot: u64) {
24824 if self.snapshot != snapshot {
24825 self.snapshot = snapshot;
24826 self.row_bounds_by_col_span.clear();
24827 self.col_bounds_by_row_span.clear();
24828 }
24829 }
24830
24831 fn get_row_bounds(
24832 &self,
24833 sheet_id: SheetId,
24834 start_col: u32,
24835 end_col: u32,
24836 snapshot: u64,
24837 ) -> Option<Option<(u32, u32)>> {
24838 if self.snapshot != snapshot {
24839 return None;
24840 }
24841 let cached = self
24842 .row_bounds_by_col_span
24843 .get(&(sheet_id, start_col, end_col))
24844 .copied();
24845 #[cfg(test)]
24846 if cached.is_some() {
24847 self.row_hits.fetch_add(1, Ordering::Relaxed);
24848 }
24849 cached
24850 }
24851
24852 fn put_row_bounds(
24853 &mut self,
24854 sheet_id: SheetId,
24855 start_col: u32,
24856 end_col: u32,
24857 snapshot: u64,
24858 bounds: Option<(u32, u32)>,
24859 ) {
24860 self.reset_for_snapshot(snapshot);
24861 self.row_bounds_by_col_span
24862 .insert((sheet_id, start_col, end_col), bounds);
24863 #[cfg(test)]
24864 self.row_misses.fetch_add(1, Ordering::Relaxed);
24865 }
24866
24867 fn get_col_bounds(
24868 &self,
24869 sheet_id: SheetId,
24870 start_row: u32,
24871 end_row: u32,
24872 snapshot: u64,
24873 ) -> Option<Option<(u32, u32)>> {
24874 if self.snapshot != snapshot {
24875 return None;
24876 }
24877 let cached = self
24878 .col_bounds_by_row_span
24879 .get(&(sheet_id, start_row, end_row))
24880 .copied();
24881 #[cfg(test)]
24882 if cached.is_some() {
24883 self.col_hits.fetch_add(1, Ordering::Relaxed);
24884 }
24885 cached
24886 }
24887
24888 fn put_col_bounds(
24889 &mut self,
24890 sheet_id: SheetId,
24891 start_row: u32,
24892 end_row: u32,
24893 snapshot: u64,
24894 bounds: Option<(u32, u32)>,
24895 ) {
24896 self.reset_for_snapshot(snapshot);
24897 self.col_bounds_by_row_span
24898 .insert((sheet_id, start_row, end_row), bounds);
24899 #[cfg(test)]
24900 self.col_misses.fetch_add(1, Ordering::Relaxed);
24901 }
24902}
24903
24904#[derive(Default)]
24906pub struct ShimSpillManager {
24907 region_locks: RegionLockManager,
24908 pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
24909}
24910
24911impl ShimSpillManager {
24912 pub(crate) fn reserve(
24913 &mut self,
24914 owner: VertexId,
24915 anchor_cell: CellRef,
24916 shape: SpillShape,
24917 _meta: SpillMeta,
24918 ) -> Result<(), ExcelError> {
24919 let region = crate::engine::spill::Region {
24921 sheet_id: anchor_cell.sheet_id as u32,
24922 row_start: anchor_cell.coord.row(),
24923 row_end: anchor_cell
24924 .coord
24925 .row()
24926 .saturating_add(shape.rows)
24927 .saturating_sub(1),
24928 col_start: anchor_cell.coord.col(),
24929 col_end: anchor_cell
24930 .coord
24931 .col()
24932 .saturating_add(shape.cols)
24933 .saturating_sub(1),
24934 };
24935 match self.region_locks.reserve(region, owner) {
24936 Ok(id) => {
24937 if id != 0 {
24938 self.active_locks.insert(owner, id);
24939 }
24940 Ok(())
24941 }
24942 Err(e) => Err(e),
24943 }
24944 }
24945
24946 pub(crate) fn release_owner(&mut self, owner: VertexId) {
24952 if let Some(id) = self.active_locks.remove(&owner) {
24953 self.region_locks.release(id);
24954 }
24955 }
24956
24957 pub(crate) fn commit_array_with_value_probe<F>(
24958 &mut self,
24959 graph: &mut DependencyGraph,
24960 anchor_vertex: VertexId,
24961 targets: &[CellRef],
24962 rows: Vec<Vec<LiteralValue>>,
24963 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24964 mut value_probe: F,
24965 ) -> Result<(), ExcelError>
24966 where
24967 F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
24968 {
24969 use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
24970
24971 let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
24975 anchor_vertex,
24976 targets,
24977 overwritable_formulas,
24978 );
24979 if let Err(e) = plan_res {
24980 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
24981 self.region_locks.release(id);
24982 }
24983 return Err(e);
24984 }
24985
24986 if !graph.value_cache_enabled() {
24987 let (expected_rows, expected_cols) = if targets.is_empty() {
24989 (0u32, 0u32)
24990 } else {
24991 let mut min_r = u32::MAX;
24992 let mut max_r = 0u32;
24993 let mut min_c = u32::MAX;
24994 let mut max_c = 0u32;
24995 for cell in targets {
24996 let r = cell.coord.row();
24997 let c = cell.coord.col();
24998 min_r = min_r.min(r);
24999 max_r = max_r.max(r);
25000 min_c = min_c.min(c);
25001 max_c = max_c.max(c);
25002 }
25003 (
25004 max_r.saturating_sub(min_r).saturating_add(1),
25005 max_c.saturating_sub(min_c).saturating_add(1),
25006 )
25007 };
25008
25009 let anchor_cell = graph
25010 .get_cell_ref(anchor_vertex)
25011 .expect("anchor cell ref for spill commit");
25012
25013 for cell in targets {
25014 if *cell == anchor_cell {
25016 continue;
25017 }
25018 if graph.spill_registry_anchor_for_cell(*cell).is_some() {
25020 continue;
25021 }
25022 if let Some(&vid) = graph.get_vertex_id_for_address(cell)
25024 && vid != anchor_vertex
25025 {
25026 match graph.get_vertex_kind(vid) {
25027 crate::engine::vertex::VertexKind::FormulaScalar
25028 | crate::engine::vertex::VertexKind::FormulaArray => {
25029 continue;
25031 }
25032 _ => {}
25033 }
25034 }
25035
25036 if let Some(v) = value_probe(graph, cell)
25037 && !matches!(v, LiteralValue::Empty)
25038 {
25039 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25040 self.region_locks.release(id);
25041 }
25042 return Err(ExcelError::new(ExcelErrorKind::Spill)
25043 .with_message("BlockedByValue")
25044 .with_extra(ExcelErrorExtra::Spill {
25045 expected_rows,
25046 expected_cols,
25047 }));
25048 }
25049 }
25050 }
25051
25052 let commit_res = graph.commit_spill_region_atomic_with_fault(
25053 anchor_vertex,
25054 targets.to_vec(),
25055 rows,
25056 None,
25057 );
25058 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25059 self.region_locks.release(id);
25060 }
25061 commit_res.map(|_| ())
25062 }
25063
25064 pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
25066 &mut self,
25067 engine: &mut Engine<R>,
25068 anchor_vertex: VertexId,
25069 targets: &[CellRef],
25070 rows: Vec<Vec<LiteralValue>>,
25071 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25072 ) -> Result<(), ExcelError> {
25073 let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
25075 anchor_vertex,
25076 targets,
25077 overwritable_formulas,
25078 );
25079 if let Err(e) = plan_res {
25080 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25081 self.region_locks.release(id);
25082 }
25083 return Err(e);
25084 }
25085
25086 let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
25087 anchor_vertex,
25088 targets.to_vec(),
25089 rows.clone(),
25090 None,
25091 );
25092 if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25093 self.region_locks.release(id);
25094 }
25095 commit_res.map(|_| ())?;
25096
25097 if engine.config.arrow_storage_enabled
25099 && engine.config.delta_overlay_enabled
25100 && engine.config.write_formula_overlay_enabled
25101 {
25102 for (idx, cell) in targets.iter().enumerate() {
25104 let (r_off, c_off) = {
25105 if rows.is_empty() || rows[0].is_empty() {
25106 (0usize, 0usize)
25107 } else {
25108 let width = rows[0].len();
25109 (idx / width, idx % width)
25110 }
25111 };
25112 let v = rows
25113 .get(r_off)
25114 .and_then(|r| r.get(c_off))
25115 .cloned()
25116 .unwrap_or(LiteralValue::Empty);
25117 let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
25118 engine.mirror_value_to_computed_overlay(
25119 &sheet_name,
25120 cell.coord.row() + 1,
25121 cell.coord.col() + 1,
25122 &v,
25123 );
25124 }
25125 }
25126 Ok(())
25127 }
25128}
25129
25130impl<R> Engine<R>
25131where
25132 R: EvaluationContext,
25133{
25134 fn resolve_shared_ref(
25135 &self,
25136 reference: &ReferenceType,
25137 current_sheet: &str,
25138 ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
25139 use formualizer_common::{
25140 SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
25141 SheetRef as SharedRef,
25142 };
25143
25144 let sr = match reference {
25146 ReferenceType::Cell {
25147 sheet,
25148 row,
25149 col,
25150 row_abs,
25151 col_abs,
25152 } => {
25153 let row0 = row
25154 .checked_sub(1)
25155 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25156 let col0 = col
25157 .checked_sub(1)
25158 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25159 let sheet_loc = match sheet.as_deref() {
25160 Some(name) => SheetLocator::from_name(name),
25161 None => SheetLocator::Current,
25162 };
25163 let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
25164 SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
25165 }
25166 ReferenceType::Range {
25167 sheet,
25168 start_row,
25169 start_col,
25170 end_row,
25171 end_col,
25172 start_row_abs,
25173 start_col_abs,
25174 end_row_abs,
25175 end_col_abs,
25176 } => {
25177 let sheet_loc = match sheet.as_deref() {
25178 Some(name) => SheetLocator::from_name(name),
25179 None => SheetLocator::Current,
25180 };
25181 let sr = start_row
25182 .map(|r| {
25183 r.checked_sub(1)
25184 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25185 })
25186 .transpose()?;
25187 let sc = start_col
25188 .map(|c| {
25189 c.checked_sub(1)
25190 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25191 })
25192 .transpose()?;
25193 let er = end_row
25194 .map(|r| {
25195 r.checked_sub(1)
25196 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25197 })
25198 .transpose()?;
25199 let ec = end_col
25200 .map(|c| {
25201 c.checked_sub(1)
25202 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25203 })
25204 .transpose()?;
25205 let range = SharedRangeRef::from_parts(
25206 sheet_loc,
25207 sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
25208 sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
25209 er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
25210 ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
25211 )
25212 .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
25213 SharedRef::Range(range)
25214 }
25215 _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
25216 };
25217
25218 let current_id = self
25219 .graph
25220 .sheet_id(current_sheet)
25221 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25222
25223 let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
25224 match loc {
25225 SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
25226 SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
25227 SheetLocator::Name(name) => {
25228 let n = name.as_ref();
25229 self.graph
25230 .sheet_id(n)
25231 .map(SheetLocator::Id)
25232 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25233 }
25234 }
25235 };
25236
25237 match sr {
25238 SharedRef::Cell(cell) => {
25239 let owned = cell.into_owned();
25240 let sheet = resolve_loc(owned.sheet)?;
25241 Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
25242 }
25243 SharedRef::Range(range) => {
25244 let owned = range.into_owned();
25245 let sheet = resolve_loc(owned.sheet)?;
25246 Ok(SharedRef::Range(SharedRangeRef {
25247 sheet,
25248 start_row: owned.start_row,
25249 start_col: owned.start_col,
25250 end_row: owned.end_row,
25251 end_col: owned.end_col,
25252 }))
25253 }
25254 }
25255 }
25256}
25257
25258impl<R> crate::traits::ReferenceResolver for Engine<R>
25261where
25262 R: EvaluationContext,
25263{
25264 fn resolve_cell_reference(
25265 &self,
25266 sheet: Option<&str>,
25267 row: u32,
25268 col: u32,
25269 ) -> Result<LiteralValue, ExcelError> {
25270 let Some(sheet_name) = sheet else {
25281 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
25282 "Unqualified cell reference resolved without sheet context".to_string(),
25283 ));
25284 };
25285 if let Some(v) = self.get_cell_value(sheet_name, row, col) {
25288 Ok(v)
25289 } else {
25290 Ok(LiteralValue::Number(0.0))
25292 }
25293 }
25294}
25295
25296impl<R> crate::traits::RangeResolver for Engine<R>
25297where
25298 R: EvaluationContext,
25299{
25300 fn resolve_range_reference(
25301 &self,
25302 sheet: Option<&str>,
25303 sr: Option<u32>,
25304 sc: Option<u32>,
25305 er: Option<u32>,
25306 ec: Option<u32>,
25307 ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
25308 self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
25311 }
25312}
25313
25314impl<R> crate::traits::NamedRangeResolver for Engine<R>
25315where
25316 R: EvaluationContext,
25317{
25318 fn resolve_named_range_reference(
25319 &self,
25320 name: &str,
25321 ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
25322 self.resolver.resolve_named_range_reference(name)
25323 }
25324}
25325
25326impl<R> crate::traits::TableResolver for Engine<R>
25327where
25328 R: EvaluationContext,
25329{
25330 fn resolve_table_reference(
25331 &self,
25332 tref: &formualizer_parse::parser::TableReference,
25333 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
25334 self.resolver.resolve_table_reference(tref)
25335 }
25336}
25337
25338impl<R> crate::traits::SourceResolver for Engine<R>
25339where
25340 R: EvaluationContext,
25341{
25342 fn source_scalar_version(&self, name: &str) -> Option<u64> {
25343 self.resolver.source_scalar_version(name)
25344 }
25345
25346 fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
25347 self.resolver.resolve_source_scalar(name)
25348 }
25349
25350 fn source_table_version(&self, name: &str) -> Option<u64> {
25351 self.resolver.source_table_version(name)
25352 }
25353
25354 fn resolve_source_table(
25355 &self,
25356 name: &str,
25357 ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
25358 self.resolver.resolve_source_table(name)
25359 }
25360}
25361
25362impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
25364
25365impl<R> crate::traits::FunctionProvider for Engine<R>
25367where
25368 R: EvaluationContext,
25369{
25370 fn planning_semantic_revision(&self) -> Option<u64> {
25371 self.resolver.planning_semantic_revision()
25372 }
25373
25374 fn get_function(
25375 &self,
25376 prefix: &str,
25377 name: &str,
25378 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
25379 self.resolver.get_function(prefix, name)
25380 }
25381
25382 fn get_function_for_planning(
25383 &self,
25384 prefix: &str,
25385 name: &str,
25386 ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
25387 self.resolver.get_function_for_planning(prefix, name)
25388 }
25389}
25390
25391impl<R> Engine<R>
25392where
25393 R: EvaluationContext,
25394{
25395 pub(crate) fn semantic_used_rows_for_columns(
25403 &self,
25404 sheet: &str,
25405 start_col: u32,
25406 end_col: u32,
25407 ) -> Option<(u32, u32)> {
25408 let arrow_bounds = self
25409 .sheet_store()
25410 .sheet(sheet)
25411 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
25412 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
25413 Self::union_used_bounds(arrow_bounds, formula_bounds)
25414 }
25415
25416 pub(crate) fn semantic_used_cols_for_rows(
25417 &self,
25418 sheet: &str,
25419 start_row: u32,
25420 end_row: u32,
25421 ) -> Option<(u32, u32)> {
25422 let arrow_bounds = self
25423 .sheet_store()
25424 .sheet(sheet)
25425 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
25426 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
25427 Self::union_used_bounds(arrow_bounds, formula_bounds)
25428 }
25429}
25430
25431impl<R> crate::traits::EvaluationContext for Engine<R>
25433where
25434 R: EvaluationContext,
25435{
25436 fn clock(&self) -> &dyn crate::timezone::ClockProvider {
25437 &self.clock
25438 }
25439
25440 fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
25441 self.thread_pool.as_ref()
25442 }
25443
25444 fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
25445 self.active_cancel_flag.clone()
25446 }
25447
25448 fn chunk_hint(&self) -> Option<usize> {
25449 let hint =
25451 (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
25452 Some(hint.clamp(1024, 1 << 20)) }
25454
25455 fn volatile_level(&self) -> crate::traits::VolatileLevel {
25456 self.config.volatile_level
25457 }
25458
25459 fn workbook_seed(&self) -> u64 {
25460 self.config.workbook_seed
25461 }
25462
25463 fn recalc_epoch(&self) -> u64 {
25464 self.recalc_epoch
25465 }
25466
25467 fn workbook_sheet_count(&self) -> Option<usize> {
25468 Some(self.graph.sheet_reg().active_len())
25469 }
25470
25471 fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
25472 self.graph.sheet_reg().active_position(sheet)
25473 }
25474
25475 fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
25476 self.sheet_index_by_name(current_sheet)
25477 }
25478
25479 fn inspect_reference(
25480 &self,
25481 reference: &ReferenceType,
25482 current_sheet: &str,
25483 ) -> Result<Option<ReferenceInfo>, ExcelError> {
25484 let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
25485 let sheet_id = self
25486 .graph
25487 .sheet_id(sheet_name)
25488 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25489 let sheet_index = self
25490 .graph
25491 .sheet_reg()
25492 .active_position_by_id(sheet_id)
25493 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25494 Ok((sheet_id, sheet_index))
25495 };
25496
25497 let cell_info =
25498 |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
25499 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
25500 let row0 = row
25501 .checked_sub(1)
25502 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25503 let col0 = col
25504 .checked_sub(1)
25505 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25506 Ok(ReferenceInfo {
25507 first_sheet_index: Some(sheet_index),
25508 sheet_count: Some(1),
25509 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
25510 })
25511 };
25512
25513 let range_info = |sheet_name: &str,
25514 start_row: Option<u32>,
25515 start_col: Option<u32>|
25516 -> Result<ReferenceInfo, ExcelError> {
25517 let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
25518 let row = start_row.unwrap_or(1);
25519 let col = start_col.unwrap_or(1);
25520 let row0 = row
25521 .checked_sub(1)
25522 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25523 let col0 = col
25524 .checked_sub(1)
25525 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25526 Ok(ReferenceInfo {
25527 first_sheet_index: Some(sheet_index),
25528 sheet_count: Some(1),
25529 first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
25530 })
25531 };
25532
25533 let info = match reference {
25534 ReferenceType::Cell {
25535 sheet, row, col, ..
25536 } => {
25537 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
25538 cell_info(sheet_name, *row, *col)?
25539 }
25540 ReferenceType::Range {
25541 sheet,
25542 start_row,
25543 start_col,
25544 ..
25545 } => {
25546 let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
25547 range_info(sheet_name, *start_row, *start_col)?
25548 }
25549 ReferenceType::Cell3D {
25550 sheet_first,
25551 sheet_last,
25552 row,
25553 col,
25554 ..
25555 } => {
25556 let first = cell_info(sheet_first, *row, *col)?;
25557 ReferenceInfo {
25558 first_sheet_index: first.first_sheet_index,
25559 sheet_count: self
25560 .graph
25561 .sheet_reg()
25562 .active_span_len(sheet_first, sheet_last),
25563 first_cell: first.first_cell,
25564 }
25565 }
25566 ReferenceType::Range3D {
25567 sheet_first,
25568 sheet_last,
25569 start_row,
25570 start_col,
25571 ..
25572 } => {
25573 let first = range_info(sheet_first, *start_row, *start_col)?;
25574 ReferenceInfo {
25575 first_sheet_index: first.first_sheet_index,
25576 sheet_count: self
25577 .graph
25578 .sheet_reg()
25579 .active_span_len(sheet_first, sheet_last),
25580 first_cell: first.first_cell,
25581 }
25582 }
25583 ReferenceType::NamedRange(name) => {
25584 let current_id = self
25585 .graph
25586 .sheet_id(current_sheet)
25587 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25588 let named = self
25589 .graph
25590 .resolve_name_entry(name, current_id)
25591 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25592 match &named.definition {
25593 NamedDefinition::Cell(cell) => ReferenceInfo {
25594 first_sheet_index: self
25595 .graph
25596 .sheet_reg()
25597 .active_position_by_id(cell.sheet_id),
25598 sheet_count: Some(1),
25599 first_cell: Some(*cell),
25600 },
25601 NamedDefinition::Range(range) => ReferenceInfo {
25602 first_sheet_index: self
25603 .graph
25604 .sheet_reg()
25605 .active_position_by_id(range.start.sheet_id),
25606 sheet_count: Some(1),
25607 first_cell: Some(range.start),
25608 },
25609 NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
25610 ReferenceInfo {
25611 first_sheet_index: None,
25612 sheet_count: None,
25613 first_cell: None,
25614 }
25615 }
25616 }
25617 }
25618 ReferenceType::Table(tref) => {
25619 let table = self
25620 .graph
25621 .resolve_table_entry(&tref.name)
25622 .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25623 ReferenceInfo {
25624 first_sheet_index: self
25625 .graph
25626 .sheet_reg()
25627 .active_position_by_id(table.range.start.sheet_id),
25628 sheet_count: Some(1),
25629 first_cell: Some(table.range.start),
25630 }
25631 }
25632 ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
25633 };
25634
25635 Ok(Some(info))
25636 }
25637
25638 fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
25639 let sheet_name = self.graph.sheet_name(cell.sheet_id);
25640 if sheet_name.is_empty() {
25641 return Err(ExcelError::new(ExcelErrorKind::Ref));
25642 }
25643 let row = cell.coord.row() + 1;
25644 let col = cell.coord.col() + 1;
25645
25646 if let Some(entries) = self.staged_formulas.get(sheet_name)
25647 && let Some(text) = entries.get(row, col)
25648 {
25649 return Ok(Some(if text.starts_with('=') {
25650 text.to_owned()
25651 } else {
25652 format!("={text}")
25653 }));
25654 }
25655
25656 let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
25657 return Ok(None);
25658 };
25659 Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
25660 }
25661
25662 fn used_rows_for_columns(
25663 &self,
25664 sheet: &str,
25665 start_col: u32,
25666 end_col: u32,
25667 ) -> Option<(u32, u32)> {
25668 let sheet_id = self.graph.sheet_id(sheet)?;
25670 let snap = self.data_snapshot_id();
25671 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
25672 guard
25673 .as_ref()
25674 .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
25675 }) {
25676 return cached;
25677 }
25678
25679 let arrow_bounds = self
25680 .sheet_store()
25681 .sheet(sheet)
25682 .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
25683 let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
25684 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
25685 Some(bounds)
25686 } else {
25687 let sc0 = start_col.saturating_sub(1);
25688 let ec0 = end_col.saturating_sub(1);
25689 self.graph
25690 .used_row_bounds_for_columns(sheet_id, sc0, ec0)
25691 .map(|(a0, b0)| (a0 + 1, b0 + 1))
25692 };
25693
25694 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
25695 guard
25696 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
25697 .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
25698 }
25699
25700 computed
25701 }
25702
25703 fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
25704 let sheet_id = self.graph.sheet_id(sheet)?;
25706 let snap = self.data_snapshot_id();
25707 if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
25708 guard
25709 .as_ref()
25710 .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
25711 }) {
25712 return cached;
25713 }
25714
25715 let arrow_bounds = self
25716 .sheet_store()
25717 .sheet(sheet)
25718 .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
25719 let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
25720 let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
25721 Some(bounds)
25722 } else {
25723 let sr0 = start_row.saturating_sub(1);
25724 let er0 = end_row.saturating_sub(1);
25725 self.graph
25726 .used_col_bounds_for_rows(sheet_id, sr0, er0)
25727 .map(|(a0, b0)| (a0 + 1, b0 + 1))
25728 };
25729
25730 if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
25731 guard
25732 .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
25733 .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
25734 }
25735
25736 computed
25737 }
25738
25739 fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
25740 let _ = self.graph.sheet_id(sheet)?;
25741 Some((1_048_576, 16_384)) }
25745
25746 fn data_snapshot_id(&self) -> u64 {
25747 self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
25748 }
25749
25750 fn backend_caps(&self) -> crate::traits::BackendCaps {
25751 crate::traits::BackendCaps {
25752 streaming: true,
25753 used_region: true,
25754 write: false,
25755 tables: false,
25756 async_stream: false,
25757 }
25758 }
25759
25760 fn build_lookup_index(
25761 &self,
25762 view: &RangeView<'_>,
25763 axis: LookupAxis,
25764 ) -> Option<Arc<LookupIndex>> {
25765 self.build_lookup_index_impl(view, axis)
25766 }
25767
25768 fn date_system(&self) -> crate::engine::DateSystem {
25771 self.config.date_system
25772 }
25773 fn resolve_range_view<'c>(
25775 &'c self,
25776 reference: &ReferenceType,
25777 current_sheet: &str,
25778 ) -> Result<RangeView<'c>, ExcelError> {
25779 match reference {
25780 ReferenceType::External(ext) => {
25781 let name = ext.raw.as_str();
25782 match ext.kind {
25783 formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
25784 let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
25785 return Err(ExcelError::new(ExcelErrorKind::Name)
25786 .with_message(format!("Undefined name: {name}")));
25787 };
25788 let version = source
25789 .version
25790 .or_else(|| self.resolver.source_scalar_version(name));
25791 let v = self.resolve_source_scalar_cached(name, version)?;
25792 Ok(RangeView::from_owned_rows(
25793 vec![vec![v]],
25794 self.config.date_system,
25795 ))
25796 }
25797 formualizer_parse::parser::ExternalRefKind::Range { .. } => {
25798 let Some(source) = self.graph.resolve_source_table_entry(name) else {
25799 return Err(ExcelError::new(ExcelErrorKind::Name)
25800 .with_message(format!("Undefined table: {name}")));
25801 };
25802 let version = source
25803 .version
25804 .or_else(|| self.resolver.source_table_version(name));
25805 let table = self.resolve_source_table_cached(name, version)?;
25806 let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
25807 self.source_table_to_range_view(table.as_ref(), &spec)
25808 }
25809 }
25810 }
25811 ReferenceType::Range { .. } => {
25812 let shared = self.resolve_shared_ref(reference, current_sheet)?;
25813 let formualizer_common::SheetRef::Range(range) = shared else {
25814 return Err(ExcelError::new(ExcelErrorKind::Ref));
25815 };
25816 let sheet_id = match range.sheet {
25819 formualizer_common::SheetLocator::Id(id) => id,
25820 formualizer_common::SheetLocator::Current
25821 | formualizer_common::SheetLocator::Name(_) => {
25822 return Err(ExcelError::new(ExcelErrorKind::Ref));
25823 }
25824 };
25825 let sheet_name = self.graph.sheet_name(sheet_id);
25826
25827 let bounded_range = if range.start_row.is_some()
25828 && range.start_col.is_some()
25829 && range.end_row.is_some()
25830 && range.end_col.is_some()
25831 {
25832 Some(RangeRef::try_from_shared(range.as_ref())?)
25833 } else {
25834 None
25835 };
25836
25837 let sr = bounded_range
25838 .as_ref()
25839 .map(|r| r.start.coord.row() + 1)
25840 .or_else(|| range.start_row.map(|b| b.index + 1));
25841 let sc = bounded_range
25842 .as_ref()
25843 .map(|r| r.start.coord.col() + 1)
25844 .or_else(|| range.start_col.map(|b| b.index + 1));
25845 let er = bounded_range
25846 .as_ref()
25847 .map(|r| r.end.coord.row() + 1)
25848 .or_else(|| range.end_row.map(|b| b.index + 1));
25849 let ec = bounded_range
25850 .as_ref()
25851 .map(|r| r.end.coord.col() + 1)
25852 .or_else(|| range.end_col.map(|b| b.index + 1));
25853
25854 let extent = resolve_used_extent_with_fallback(
25855 OpenRangeBounds {
25856 start_row: sr,
25857 start_column: sc,
25858 end_row: er,
25859 end_column: ec,
25860 },
25861 ExtentPolicy::EvaluationCompat {
25862 fallback_row: None,
25863 fallback_column: None,
25864 },
25865 || {
25866 self.sheet_bounds(sheet_name)
25867 .map(|_| self.config.max_open_ended_rows)
25868 },
25869 || {
25870 self.sheet_bounds(sheet_name)
25871 .map(|_| self.config.max_open_ended_cols)
25872 },
25873 |first, last| self.used_rows_for_columns(sheet_name, first, last),
25874 |first, last| self.used_cols_for_rows(sheet_name, first, last),
25875 );
25876 let (sr, sc, er, ec) = extent
25877 .map(|extent| {
25878 (
25879 extent.start_row,
25880 extent.start_column,
25881 extent.end_row,
25882 extent.end_column,
25883 )
25884 })
25885 .unwrap_or((1, 1, 0, 0));
25886
25887 if self.force_materialize_range_views {
25888 if er < sr || ec < sc {
25889 return Ok(RangeView::from_owned_rows(
25890 Vec::new(),
25891 self.config.date_system,
25892 ));
25893 }
25894 let h = (er - sr + 1) as u64;
25895 let w = (ec - sc + 1) as u64;
25896 let cell_count = h.saturating_mul(w);
25897 if cell_count <= self.config.spill.max_spill_cells as u64 {
25898 let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
25899 for r in sr..=er {
25900 let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
25901 for c in sc..=ec {
25902 rowv.push(
25903 self.get_cell_value(sheet_name, r, c)
25904 .unwrap_or(LiteralValue::Empty),
25905 );
25906 }
25907 rows.push(rowv);
25908 }
25909 return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
25910 }
25911 }
25912
25913 let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
25914 return Ok(RangeView::from_owned_rows(
25915 Vec::new(),
25916 self.config.date_system,
25917 ));
25918 };
25919
25920 let rv = if er < sr || ec < sc {
25921 asheet.range_view(1, 1, 0, 0)
25922 } else {
25923 let sr0 = sr.saturating_sub(1) as usize;
25924 let sc0 = sc.saturating_sub(1) as usize;
25925 let er0 = er.saturating_sub(1) as usize;
25926 let ec0 = ec.saturating_sub(1) as usize;
25927 asheet.range_view(sr0, sc0, er0, ec0)
25928 };
25929
25930 Ok(rv)
25931 }
25932 ReferenceType::Cell { .. } => {
25933 let shared = self.resolve_shared_ref(reference, current_sheet)?;
25934 let formualizer_common::SheetRef::Cell(cell) = shared else {
25935 return Err(ExcelError::new(ExcelErrorKind::Ref));
25936 };
25937 let addr = CellRef::try_from_shared(cell)?;
25938 let sheet_id = addr.sheet_id;
25939 let sheet_name = self.graph.sheet_name(sheet_id);
25940 let row = addr.coord.row() + 1;
25941 let col = addr.coord.col() + 1;
25942
25943 if self.force_materialize_range_views {
25944 let v = self
25945 .get_cell_value(sheet_name, row, col)
25946 .unwrap_or(LiteralValue::Empty);
25947 return Ok(RangeView::from_owned_rows(
25948 vec![vec![v]],
25949 self.config.date_system,
25950 ));
25951 }
25952
25953 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25954 let r0 = row.saturating_sub(1) as usize;
25955 let c0 = col.saturating_sub(1) as usize;
25956 let rv = asheet.range_view(r0, c0, r0, c0);
25957 Ok(rv)
25958 } else {
25959 let v = self
25960 .get_cell_value(sheet_name, row, col)
25961 .unwrap_or(LiteralValue::Empty);
25962 Ok(RangeView::from_owned_rows(
25963 vec![vec![v]],
25964 self.config.date_system,
25965 ))
25966 }
25967 }
25968 ReferenceType::NamedRange(name) => {
25969 if let Some(current_id) = self.graph.sheet_id(current_sheet)
25970 && let Some(named) = self.graph.resolve_name_entry(name, current_id)
25971 {
25972 match &named.definition {
25973 NamedDefinition::Cell(cell_ref) => {
25974 let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
25975 if self.force_materialize_range_views {
25976 let v = self
25977 .get_cell_value(
25978 sheet_name,
25979 cell_ref.coord.row() + 1,
25980 cell_ref.coord.col() + 1,
25981 )
25982 .unwrap_or(LiteralValue::Empty);
25983 return Ok(RangeView::from_owned_rows(
25984 vec![vec![v]],
25985 self.config.date_system,
25986 ));
25987 } else {
25988 let asheet = self
25989 .sheet_store()
25990 .sheet(sheet_name)
25991 .expect("Arrow sheet missing for named cell");
25992 let r0 = cell_ref.coord.row() as usize;
25993 let c0 = cell_ref.coord.col() as usize;
25994 let rv = asheet.range_view(r0, c0, r0, c0);
25995 return Ok(rv);
25996 }
25997 }
25998 NamedDefinition::Range(range_ref) => {
25999 let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
26000 let sr = range_ref.start.coord.row() + 1;
26001 let sc = range_ref.start.coord.col() + 1;
26002 let er = range_ref.end.coord.row() + 1;
26003 let ec = range_ref.end.coord.col() + 1;
26004 if self.force_materialize_range_views {
26005 let h = (er.saturating_sub(sr) + 1) as u64;
26006 let w = (ec.saturating_sub(sc) + 1) as u64;
26007 let cell_count = h.saturating_mul(w);
26008 if cell_count <= self.config.spill.max_spill_cells as u64 {
26009 let mut rows: Vec<Vec<LiteralValue>> =
26010 Vec::with_capacity(h as usize);
26011 for r in sr..=er {
26012 let mut rowv: Vec<LiteralValue> =
26013 Vec::with_capacity(w as usize);
26014 for c in sc..=ec {
26015 rowv.push(
26016 self.get_cell_value(sheet_name, r, c)
26017 .unwrap_or(LiteralValue::Empty),
26018 );
26019 }
26020 rows.push(rowv);
26021 }
26022 return Ok(RangeView::from_owned_rows(
26023 rows,
26024 self.config.date_system,
26025 ));
26026 }
26027 }
26028 let asheet = self
26029 .sheet_store()
26030 .sheet(sheet_name)
26031 .expect("Arrow sheet missing for named range");
26032 let sr0 = range_ref.start.coord.row() as usize;
26033 let sc0 = range_ref.start.coord.col() as usize;
26034 let er0 = range_ref.end.coord.row() as usize;
26035 let ec0 = range_ref.end.coord.col() as usize;
26036 let rv = asheet.range_view(sr0, sc0, er0, ec0);
26037 return Ok(rv);
26038 }
26039 NamedDefinition::Literal(v) => {
26040 return Ok(RangeView::from_owned_rows(
26041 vec![vec![v.clone()]],
26042 self.config.date_system,
26043 ));
26044 }
26045 NamedDefinition::Formula { .. } => {
26046 if let Some(value) = self.graph.get_value(named.vertex) {
26047 return Ok(RangeView::from_owned_rows(
26048 vec![vec![value]],
26049 self.config.date_system,
26050 ));
26051 }
26052 }
26053 }
26054 }
26055
26056 if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
26057 let version = source
26058 .version
26059 .or_else(|| self.resolver.source_scalar_version(name));
26060 let v = self.resolve_source_scalar_cached(name, version)?;
26061 return Ok(RangeView::from_owned_rows(
26062 vec![vec![v]],
26063 self.config.date_system,
26064 ));
26065 }
26066
26067 let data = self.resolver.resolve_named_range_reference(name)?;
26068 Ok(RangeView::from_owned_rows(data, self.config.date_system))
26069 }
26070 ReferenceType::Table(tref) => {
26071 if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
26072 let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
26073 let asheet = self
26074 .sheet_store()
26075 .sheet(sheet_name)
26076 .expect("Arrow sheet missing for table reference");
26077
26078 let sr0 = table.range.start.coord.row() as usize;
26079 let sc0 = table.range.start.coord.col() as usize;
26080 let er0 = table.range.end.coord.row() as usize;
26081 let ec0 = table.range.end.coord.col() as usize;
26082
26083 let has_totals = table.totals_row;
26084 let has_headers = table.header_row;
26085 let data_sr = if has_headers {
26086 sr0.saturating_add(1)
26087 } else {
26088 sr0
26089 };
26090 let data_er = if has_totals {
26091 er0.saturating_sub(1)
26092 } else {
26093 er0
26094 };
26095
26096 let select = |sr: usize, sc: usize, er: usize, ec: usize| {
26097 if sr > er || sc > ec {
26098 asheet.range_view(1, 1, 0, 0)
26099 } else {
26100 asheet.range_view(sr, sc, er, ec)
26101 }
26102 };
26103
26104 let av = match &tref.specifier {
26105 None => {
26106 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26107 "Table reference without specifier is unsupported".to_string(),
26108 ));
26109 }
26110 Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
26111 let Some(idx) = table.col_index(col) else {
26112 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26113 "Column refers to unknown table column".to_string(),
26114 ));
26115 };
26116 let c0 = sc0 + idx;
26117 select(data_sr, c0, data_er, c0)
26118 }
26119 Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
26120 start,
26121 end,
26122 )) => {
26123 let Some(si) = table.col_index(start) else {
26124 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26125 "Column range refers to unknown column(s)".to_string(),
26126 ));
26127 };
26128 let Some(ei) = table.col_index(end) else {
26129 return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26130 "Column range refers to unknown column(s)".to_string(),
26131 ));
26132 };
26133 let (mut a, mut b) = (si, ei);
26134 if a > b {
26135 std::mem::swap(&mut a, &mut b);
26136 }
26137 let c_start = sc0 + a;
26138 let c_end = sc0 + b;
26139 select(data_sr, c_start, data_er, c_end)
26140 }
26141 Some(formualizer_parse::parser::TableSpecifier::All)
26142 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26143 formualizer_parse::parser::SpecialItem::All,
26144 )) => select(sr0, sc0, er0, ec0),
26145 Some(formualizer_parse::parser::TableSpecifier::Data)
26146 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26147 formualizer_parse::parser::SpecialItem::Data,
26148 )) => select(data_sr, sc0, data_er, ec0),
26149 Some(formualizer_parse::parser::TableSpecifier::Headers)
26150 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26151 formualizer_parse::parser::SpecialItem::Headers,
26152 )) => {
26153 if !has_headers {
26154 asheet.range_view(1, 1, 0, 0)
26155 } else {
26156 select(sr0, sc0, sr0, ec0)
26157 }
26158 }
26159 Some(formualizer_parse::parser::TableSpecifier::Totals)
26160 | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26161 formualizer_parse::parser::SpecialItem::Totals,
26162 )) => {
26163 if !has_totals {
26164 asheet.range_view(1, 1, 0, 0)
26165 } else {
26166 select(er0, sc0, er0, ec0)
26167 }
26168 }
26169 Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26170 formualizer_parse::parser::SpecialItem::ThisRow,
26171 )) => {
26172 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26173 "@ (This Row) requires table-aware context; not yet supported"
26174 .to_string(),
26175 ));
26176 }
26177 Some(formualizer_parse::parser::TableSpecifier::Row(_))
26178 | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
26179 return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26180 "Complex structured references not yet supported".to_string(),
26181 ));
26182 }
26183 };
26184
26185 return Ok(av);
26186 }
26187
26188 if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
26189 let version = source
26190 .version
26191 .or_else(|| self.resolver.source_table_version(&tref.name));
26192 let table = self.resolve_source_table_cached(&tref.name, version)?;
26193 return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
26194 }
26195
26196 let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
26198 let owned = boxed.materialise().into_owned();
26199 Ok(RangeView::from_owned_rows(owned, self.config.date_system))
26200 }
26201 ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
26202 Err(ExcelError::new(ExcelErrorKind::NImpl)
26203 .with_message("3D references are not yet supported".to_string()))
26204 }
26205 }
26206 }
26207
26208 fn resolve_cell_format(
26209 &self,
26210 sheet: Option<&str>,
26211 row: u32,
26212 col: u32,
26213 current_sheet: &str,
26214 ) -> Option<crate::format::FormatId> {
26215 self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
26216 }
26217
26218 fn format_class(
26219 &self,
26220 format: crate::format::FormatId,
26221 ) -> Option<formualizer_common::numfmt::FormatClass> {
26222 self.format_registry.class(format).cloned()
26223 }
26224
26225 fn record_cell_derived_format(
26226 &self,
26227 sheet: &str,
26228 row: u32,
26229 col: u32,
26230 format: Option<crate::format::FormatId>,
26231 ) {
26232 if let Some(sheet_id) = self.graph.sheet_id(sheet) {
26233 let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
26234 self.record_derived_format_at(cell, format);
26235 }
26236 }
26237
26238 fn resolve_cell_reference_value(
26239 &self,
26240 sheet: Option<&str>,
26241 row: u32,
26242 col: u32,
26243 current_sheet: &str,
26244 ) -> Result<LiteralValue, ExcelError> {
26245 let sheet_name = sheet.unwrap_or(current_sheet);
26246 if self.graph.sheet_id(sheet_name).is_none() {
26247 return Err(ExcelError::new(ExcelErrorKind::Ref));
26248 }
26249 Ok(self
26250 .get_cell_value(sheet_name, row, col)
26251 .unwrap_or(LiteralValue::Empty))
26252 }
26253
26254 fn build_criteria_mask(
26255 &self,
26256 view: &RangeView<'_>,
26257 col_in_view: usize,
26258 pred: &crate::args::CriteriaPredicate,
26259 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
26260 #[cfg(any(test, feature = "test-support"))]
26261 criteria_mask_test_hooks::note_mask(view.dims().0);
26262 if view.dims().1 == 0 {
26263 return None;
26264 }
26265 let sheet_rows = view.sheet().nrows as usize;
26268 if sheet_rows == 0 || view.start_row() >= sheet_rows {
26269 return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
26270 }
26271 compute_criteria_mask(view, col_in_view, pred)
26272 }
26273
26274 fn build_row_visibility_mask(
26275 &self,
26276 view: &RangeView<'_>,
26277 mode: VisibilityMaskMode,
26278 ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
26279 self.build_row_visibility_mask_for_view(view, mode)
26280 }
26281}
26282
26283impl<R> Engine<R>
26284where
26285 R: EvaluationContext,
26286{
26287 fn clear_spill_projection_and_mirror(
26288 &mut self,
26289 anchor_vertex: VertexId,
26290 delta: Option<&mut DeltaCollector>,
26291 ) {
26292 let spill_cells = self
26293 .graph
26294 .spill_cells_for_anchor(anchor_vertex)
26295 .map(|cells| cells.to_vec())
26296 .unwrap_or_default();
26297 if spill_cells.is_empty() {
26298 return;
26299 }
26300
26301 if let Some(delta) = delta
26302 && delta.mode != DeltaMode::Off
26303 {
26304 let empty = LiteralValue::Empty;
26305 for cell in spill_cells.iter() {
26306 let sheet_name = self.graph.sheet_name(cell.sheet_id);
26307 let old = self
26308 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26309 .unwrap_or(LiteralValue::Empty);
26310 if old != empty {
26311 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26312 }
26313 }
26314 }
26315
26316 self.graph.clear_spill_region(anchor_vertex);
26317 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
26318 self.record_formula_plane_structural_change(scope);
26319 }
26320
26321 if self.config.arrow_storage_enabled
26322 && self.config.delta_overlay_enabled
26323 && self.config.write_formula_overlay_enabled
26324 {
26325 let empty = LiteralValue::Empty;
26326 for cell in spill_cells.iter() {
26327 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
26328 self.mirror_value_to_computed_overlay(
26329 &sheet_name,
26330 cell.coord.row() + 1,
26331 cell.coord.col() + 1,
26332 &empty,
26333 );
26334 }
26335 }
26336 }
26337
26338 fn apply_cycle_outcome(
26352 &mut self,
26353 cycle: &[VertexId],
26354 mut delta: Option<&mut DeltaCollector>,
26355 dirty_filter: Option<&FxHashSet<VertexId>>,
26356 ) -> usize {
26357 let circ_error = LiteralValue::Error(
26358 ExcelError::new(ExcelErrorKind::Circ)
26359 .with_message("Circular dependency detected".to_string()),
26360 );
26361 let mut stamped = 0usize;
26362 for &vertex_id in cycle {
26363 if let Some(filter) = dirty_filter
26364 && !filter.contains(&vertex_id)
26365 {
26366 continue;
26367 }
26368 self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
26369 stamped += 1;
26370 }
26371 stamped
26372 }
26373
26374 fn stamp_cycle_error(
26389 &mut self,
26390 vertex_id: VertexId,
26391 circ_error: &LiteralValue,
26392 mut delta: Option<&mut DeltaCollector>,
26393 ) {
26394 if self.graph.spill_registry_has_anchor(vertex_id) {
26396 self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
26397 }
26398 self.spill_mgr.release_owner(vertex_id);
26401
26402 if let Some(d) = delta
26404 && d.mode != DeltaMode::Off
26405 && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
26406 {
26407 let sheet_name = self.graph.sheet_name(cell.sheet_id);
26408 let old = self
26409 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26410 .unwrap_or(LiteralValue::Empty);
26411 if old != *circ_error {
26412 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26413 }
26414 }
26415
26416 self.graph
26417 .update_vertex_value(vertex_id, circ_error.clone());
26418 self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
26419 }
26420
26421 fn handle_cycle_unit(
26434 &mut self,
26435 cycle: &[VertexId],
26436 mut delta: Option<&mut DeltaCollector>,
26437 dirty_filter: Option<&FxHashSet<VertexId>>,
26438 cancel_flag: Option<&AtomicBool>,
26439 ) -> Result<usize, ExcelError> {
26440 self.resource_checkpoint(cycle.len() as u64)?;
26441 match self.config.cycle.detection {
26442 CycleDetection::Static => {
26443 Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
26444 }
26445 CycleDetection::Runtime => {
26446 if let Some(filter) = dirty_filter
26447 && !cycle.iter().any(|v| filter.contains(v))
26448 {
26449 return Ok(0);
26450 }
26451 self.evaluate_scc_unit(cycle, delta, cancel_flag)
26455 }
26456 }
26457 }
26458
26459 pub(crate) fn evaluate_scc_unit(
26479 &mut self,
26480 cycle: &[VertexId],
26481 mut delta: Option<&mut DeltaCollector>,
26482 cancel_flag: Option<&AtomicBool>,
26483 ) -> Result<usize, ExcelError> {
26484 struct SccMember {
26485 vertex: VertexId,
26486 cell: Option<CellRef>,
26487 }
26488
26489 let task_start = crate::instant::FzInstant::now();
26490
26491 let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
26496 let mut name_members: Vec<(VertexId, String)> = Vec::new();
26497 let mut other_members: Vec<VertexId> = Vec::new();
26498 for &v in cycle {
26499 match self.graph.get_vertex_kind(v) {
26500 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
26501 match self.graph.get_cell_ref(v) {
26502 Some(cell) => cell_members.push((v, cell)),
26503 None => other_members.push(v),
26504 }
26505 }
26506 VertexKind::NamedScalar | VertexKind::NamedArray => {
26507 match self.graph.name_key_for_vertex(v) {
26508 Some(key) => name_members.push((v, key)),
26509 None => other_members.push(v),
26510 }
26511 }
26512 _ => other_members.push(v),
26513 }
26514 }
26515 cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
26516 name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
26517 other_members.sort_unstable();
26518
26519 let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
26520 let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
26521 let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
26522 for (v, c) in &cell_members {
26523 members.push(SccMember {
26524 vertex: *v,
26525 cell: Some(*c),
26526 });
26527 }
26528 for (v, _) in &name_members {
26529 members.push(SccMember {
26530 vertex: *v,
26531 cell: None,
26532 });
26533 }
26534 for v in &other_members {
26535 members.push(SccMember {
26536 vertex: *v,
26537 cell: None,
26538 });
26539 }
26540 let n = members.len();
26541 let recordable = cell_refs.len() + name_keys.len();
26544
26545 let circ_error = LiteralValue::Error(
26546 ExcelError::new(ExcelErrorKind::Circ)
26547 .with_message("Circular dependency detected".to_string()),
26548 );
26549
26550 if !self.iterative_state_values.is_empty() {
26561 let restore: Vec<(VertexId, LiteralValue)> = members
26562 .iter()
26563 .filter_map(|m| {
26564 let cell = m.cell?;
26565 let persisted = self.iterative_state_values.get(&m.vertex)?;
26566 let sheet_name = self.graph.sheet_name(cell.sheet_id);
26567 let overlay = self
26568 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26569 .unwrap_or(LiteralValue::Empty);
26570 if matches!(overlay, LiteralValue::Empty) {
26571 Some((m.vertex, persisted.clone()))
26572 } else {
26573 None
26574 }
26575 })
26576 .collect();
26577 for (vertex, value) in restore {
26578 self.mirror_vertex_value_to_overlay(vertex, &value);
26579 }
26580 }
26581
26582 let snapshot: Vec<LiteralValue> = members
26585 .iter()
26586 .map(|m| match m.cell {
26587 Some(cell) => {
26588 let sheet_name = self.graph.sheet_name(cell.sheet_id);
26589 self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26590 .unwrap_or(LiteralValue::Empty)
26591 }
26592 None => self
26593 .graph
26594 .get_value(m.vertex)
26595 .unwrap_or(LiteralValue::Empty),
26596 })
26597 .collect();
26598
26599 let mut excluded = vec![false; n];
26604 let mut last_value = snapshot.clone();
26605 let mut stamped = 0usize;
26606 for (i, m) in members.iter().enumerate() {
26607 match self.graph.get_vertex_kind(m.vertex) {
26608 VertexKind::FormulaArray => {
26609 self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
26613 excluded[i] = true;
26614 last_value[i] = circ_error.clone();
26615 stamped += 1;
26616 }
26617 VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
26618 _ => excluded[i] = true,
26619 }
26620 }
26621
26622 let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
26623
26624 let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
26626 let mut pos: Vec<i64> = vec![-1; n];
26628 let mut changed = vec![false; n];
26630
26631 macro_rules! run_member {
26634 ($i:expr) => {{
26635 let i: usize = $i;
26636 let m = &members[i];
26637 if i < recordable {
26638 collector.set_current(i as u32);
26639 }
26640 let value = {
26641 let ctx = RecordingContext::new(&*self, &collector);
26642 match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
26643 Ok(v) => v,
26644 Err(e) => LiteralValue::Error(e),
26645 }
26646 };
26647 let is_cell_formula = m.cell.is_some();
26648 if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
26649 self.stamp_cycle_error(m.vertex, &circ_error, None);
26654 excluded[i] = true;
26655 stamped += 1;
26656 changed[i] = last_value[i] != circ_error;
26657 last_value[i] = circ_error.clone();
26658 } else {
26659 self.graph.update_vertex_value(m.vertex, value.clone());
26660 self.mirror_vertex_value_to_overlay(m.vertex, &value);
26661 #[cfg(debug_assertions)]
26665 if let Some(cell) = m.cell {
26666 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
26667 debug_assert!(
26668 self.read_delta_overlay_cell(
26669 &sheet_name,
26670 cell.coord.row() + 1,
26671 cell.coord.col() + 1
26672 )
26673 .is_none(),
26674 "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
26675 cell.coord.row() + 1,
26676 cell.coord.col() + 1
26677 );
26678 }
26679 changed[i] = last_value[i] != value;
26680 last_value[i] = value;
26681 }
26682 }};
26683 }
26684
26685 let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
26686 if let Some(flag) = flag
26687 && flag.load(Ordering::Relaxed)
26688 {
26689 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26690 .with_message("Evaluation cancelled during SCC evaluation".to_string()));
26691 }
26692 Ok(())
26693 };
26694
26695 check_cancel(cancel_flag)?;
26697 let mut passes = 1usize;
26698 {
26699 let mut p = 0i64;
26700 for i in 0..n {
26701 if excluded[i] {
26702 continue;
26703 }
26704 run_member!(i);
26705 pos[i] = p;
26706 p += 1;
26707 }
26708 }
26709
26710 let policy = self.config.cycle.policy;
26726 let cap = n + 2;
26727 let mut witnessed_cycles = 0usize;
26728 let mut capped = false;
26729 let mut iterating = false;
26731 let mut converged = false;
26732 let mut exact_fixed_point = false;
26733 let mut prev_pass: Option<Vec<LiteralValue>> = None;
26737 let mut iter_max_delta = 0f64;
26740 let mut iter_nan_converged = 0usize;
26741 let mut settle_passes = 0usize;
26745 loop {
26746 let drained = collector.take_edges();
26749 for i in 0..n {
26750 if pos[i] >= 0 {
26751 out_edges[i].clear();
26752 }
26753 }
26754 for (from, to) in drained {
26755 debug_assert!(
26756 pos[from as usize] >= 0,
26757 "edge from a member that did not run"
26758 );
26759 out_edges[from as usize].push(to);
26760 }
26761 let mut edges: Vec<(u32, u32)> = Vec::new();
26762 for (i, outs) in out_edges.iter().enumerate() {
26763 if excluded[i] {
26764 continue;
26765 }
26766 for &t in outs {
26767 edges.push((i as u32, t));
26768 }
26769 }
26770 edges.sort_unstable();
26771 edges.dedup();
26772
26773 let analysis = analyze_live_graph(n, &edges);
26774
26775 if analysis.cycle_count > 0 {
26776 witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
26780 match policy {
26781 CyclePolicy::Error => {
26782 for i in 0..n {
26788 if analysis.in_cycle[i] && !excluded[i] {
26789 self.stamp_cycle_error(members[i].vertex, &circ_error, None);
26790 excluded[i] = true;
26791 last_value[i] = circ_error.clone();
26792 stamped += 1;
26793 }
26794 }
26795 check_cancel(cancel_flag)?;
26796 let order: Vec<usize> = analysis
26797 .topo
26798 .iter()
26799 .map(|&i| i as usize)
26800 .filter(|&i| !excluded[i])
26801 .collect();
26802 if !order.is_empty() {
26803 passes += 1;
26804 for i in order {
26805 run_member!(i);
26806 }
26807 }
26808 break;
26809 }
26810 CyclePolicy::Iterate {
26811 max_iterations,
26812 max_change,
26813 } => {
26814 iterating = true;
26816
26817 if let Some(prev) = &prev_pass {
26823 let mut round_max_delta = 0f64;
26824 let mut round_nan = 0usize;
26825 let mut all_converged = true;
26826 let mut round_exact = true;
26827 for i in 0..n {
26828 if excluded[i] {
26829 continue;
26833 }
26834 let out = crate::engine::convergence::values_converged(
26835 &prev[i],
26836 &last_value[i],
26837 max_change,
26838 self.config.date_system,
26839 );
26840 if out.nan_converged {
26841 round_nan += 1;
26842 }
26843 if let Some(d) = out.abs_delta {
26844 round_max_delta = round_max_delta.max(d);
26845 if d != 0.0 {
26846 round_exact = false;
26847 }
26848 }
26849 if !out.converged {
26850 all_converged = false;
26851 }
26852 }
26853 iter_max_delta = round_max_delta;
26856 iter_nan_converged = round_nan;
26857 if all_converged {
26858 exact_fixed_point = round_exact;
26859 converged = true;
26860 break;
26861 }
26862 }
26863
26864 if passes >= max_iterations as usize {
26876 capped = true;
26877 break;
26878 }
26879
26880 check_cancel(cancel_flag)?;
26881 prev_pass = Some(last_value.clone());
26889 for x in pos.iter_mut() {
26890 *x = -1;
26891 }
26892 changed.fill(false);
26893 passes += 1;
26894 let mut p = 0i64;
26895 for i in 0..n {
26896 if excluded[i] {
26897 continue;
26898 }
26899 run_member!(i);
26900 pos[i] = p;
26901 p += 1;
26902 }
26903 continue;
26904 }
26905 }
26906 }
26907
26908 let mut stale: Vec<usize> = Vec::new();
26911 for i in 0..n {
26912 if excluded[i] {
26913 continue;
26914 }
26915 let is_stale = out_edges[i].iter().any(|&t| {
26916 let t = t as usize;
26917 changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
26918 });
26919 if is_stale {
26920 stale.push(i);
26921 }
26922 }
26923 if stale.is_empty() {
26924 break; }
26926 if 1 + settle_passes >= cap {
26927 capped = true;
26929 for (i, m) in members.iter().enumerate() {
26930 if !excluded[i] {
26931 self.stamp_cycle_error(m.vertex, &circ_error, None);
26932 excluded[i] = true;
26933 last_value[i] = circ_error.clone();
26934 stamped += 1;
26935 }
26936 }
26937 break;
26938 }
26939
26940 check_cancel(cancel_flag)?;
26941 prev_pass = None;
26948 let topo_pos = analysis.topo_positions();
26949 stale.sort_unstable_by_key(|&i| topo_pos[i]);
26950 for x in pos.iter_mut() {
26951 *x = -1;
26952 }
26953 changed.fill(false);
26954 passes += 1;
26955 settle_passes += 1;
26956 for (p, i) in stale.into_iter().enumerate() {
26957 run_member!(i);
26958 pos[i] = p as i64;
26959 }
26960 }
26961
26962 if iterating && !converged && !capped {
26967 converged = true;
26968 }
26969
26970 collector.clear_current();
26973 if let Some(d) = delta
26974 && d.mode != DeltaMode::Off
26975 {
26976 for (i, m) in members.iter().enumerate() {
26977 if let Some(cell) = m.cell
26978 && last_value[i] != snapshot[i]
26979 {
26980 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26981 }
26982 }
26983 }
26984
26985 if !self.retained_scc_members.is_empty() {
27003 for m in members.iter() {
27004 self.retained_scc_members.remove(&m.vertex);
27005 }
27006 }
27007 if iterating {
27008 let retain = converged
27009 && !capped
27010 && exact_fixed_point
27011 && iter_nan_converged == 0
27012 && members
27013 .iter()
27014 .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
27015 if retain {
27016 if self.retained_scc_members.is_empty() {
27017 self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
27018 self.retained_scc_function_epoch_seen =
27019 crate::function_registry::semantic_epoch();
27020 self.retained_scc_provider_revision_seen =
27021 self.resolver.planning_semantic_revision();
27022 }
27023 let scc_id = self.next_retained_scc_id;
27024 self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
27025 for (i, m) in members.iter().enumerate() {
27026 self.retained_scc_members.insert(m.vertex, scc_id);
27027 if matches!(last_value[i], LiteralValue::Empty) {
27030 self.iterative_state_values.remove(&m.vertex);
27031 } else {
27032 self.iterative_state_values
27033 .insert(m.vertex, last_value[i].clone());
27034 }
27035 }
27036 } else {
27037 self.pending_iterative_redirty
27038 .extend(members.iter().map(|m| m.vertex));
27039 }
27040 } else if !self.iterative_state_values.is_empty() {
27041 for m in members.iter() {
27045 self.iterative_state_values.remove(&m.vertex);
27046 }
27047 }
27048
27049 {
27050 let t = &mut self.last_cycle_telemetry;
27051 t.static_sccs += 1;
27052 if witnessed_cycles == 0 && stamped == 0 && !capped {
27053 t.phantom_sccs += 1;
27054 }
27055 t.live_cycles_witnessed += witnessed_cycles;
27056 t.circ_cells_stamped += stamped;
27057 t.settle_passes_total += passes;
27058 t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
27059 if iterating {
27060 t.iterated_sccs += 1;
27061 if converged {
27062 t.converged_sccs += 1;
27063 }
27064 t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
27065 t.nan_converged += iter_nan_converged;
27066 }
27067 if capped {
27068 t.capped_sccs += 1;
27069 }
27070 t.elapsed_ms += task_start.elapsed().as_millis();
27071 }
27072
27073 Ok(stamped)
27074 }
27075
27076 fn evaluate_vertex_recorded(
27084 &self,
27085 vertex_id: VertexId,
27086 ctx: &RecordingContext<'_, R>,
27087 collector: &LiveEdgeCollector,
27088 ) -> Result<LiteralValue, ExcelError> {
27089 if !self.graph.vertex_exists(vertex_id) {
27090 return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
27091 .with_message(format!("Vertex not found: {vertex_id:?}")));
27092 }
27093
27094 let kind = self.graph.get_vertex_kind(vertex_id);
27095 let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
27096
27097 match kind {
27098 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
27099 let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
27100 return Ok(LiteralValue::Number(0.0)); };
27102 let sheet_name = self.graph.sheet_name(sheet_id);
27103 let cell_ref = self
27104 .graph
27105 .get_cell_ref(vertex_id)
27106 .expect("cell ref for vertex");
27107 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
27108 interpreter
27109 .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
27110 .map(|cv| {
27111 let format = cv.format_id();
27112 self.derived_format_results
27113 .write()
27114 .unwrap()
27115 .insert(vertex_id, format);
27116 self.record_derived_format(vertex_id, format);
27117 crate::engine::result_finalization::finalize_formula_result(
27118 cv.into_literal(),
27119 )
27120 })
27121 }
27122 VertexKind::NamedScalar | VertexKind::NamedArray => {
27123 let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
27124 ExcelError::new(ExcelErrorKind::Name)
27125 .with_message("Named range metadata missing".to_string())
27126 })?;
27127
27128 match &named_range.definition {
27129 NamedDefinition::Formula { ast, .. } => {
27130 let context_sheet = match named_range.scope {
27131 NameScope::Sheet(id) => id,
27132 NameScope::Workbook => sheet_id,
27133 };
27134 let sheet_name = self.graph.sheet_name(context_sheet);
27135 let cell_ref = self
27136 .graph
27137 .get_cell_ref(vertex_id)
27138 .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
27139 let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
27140 if kind == VertexKind::NamedScalar {
27141 interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
27142 } else {
27143 match interpreter.evaluate_ast(ast) {
27144 Ok(cv) => match cv.into_literal() {
27145 v @ LiteralValue::Array(_) => Ok(v),
27146 other => Ok(LiteralValue::Array(vec![vec![other]])),
27147 },
27148 Err(err) => Ok(LiteralValue::Error(err)),
27149 }
27150 }
27151 }
27152 NamedDefinition::Cell(cell_ref) => {
27153 collector.record_scalar(
27157 cell_ref.sheet_id,
27158 cell_ref.coord.row(),
27159 cell_ref.coord.col(),
27160 );
27161 self.evaluate_vertex_immutable(vertex_id)
27162 }
27163 NamedDefinition::Range(range_ref) => {
27164 if range_ref.start.sheet_id == range_ref.end.sheet_id {
27165 collector.record_rect(
27166 range_ref.start.sheet_id,
27167 range_ref.start.coord.row(),
27168 range_ref.start.coord.col(),
27169 range_ref.end.coord.row(),
27170 range_ref.end.coord.col(),
27171 );
27172 }
27173 self.evaluate_vertex_immutable(vertex_id)
27174 }
27175 NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
27176 }
27177 }
27178 _ => self.evaluate_vertex_immutable(vertex_id),
27179 }
27180 }
27181
27182 fn commit_spill_and_mirror(
27184 &mut self,
27185 anchor_vertex: VertexId,
27186 targets: &[CellRef],
27187 rows: Vec<Vec<LiteralValue>>,
27188 delta: Option<&mut DeltaCollector>,
27189 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27190 ) -> Result<(), ExcelError> {
27191 let prev_spill_cells = self
27192 .graph
27193 .spill_cells_for_anchor(anchor_vertex)
27194 .map(|cells| cells.to_vec())
27195 .unwrap_or_default();
27196
27197 if let Some(delta) = delta
27198 && delta.mode != DeltaMode::Off
27199 {
27200 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
27201 targets.iter().copied().collect();
27202 let empty = LiteralValue::Empty;
27203
27204 for cell in prev_spill_cells.iter() {
27206 if target_set.contains(cell) {
27207 continue;
27208 }
27209 let sheet_name = self.graph.sheet_name(cell.sheet_id);
27210 let old = self
27211 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27212 .unwrap_or(LiteralValue::Empty);
27213 if old != empty {
27214 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27215 }
27216 }
27217
27218 if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
27220 let width = rows[0].len();
27221 for (idx, cell) in targets.iter().enumerate() {
27222 let r_off = idx / width;
27223 let c_off = idx % width;
27224 let new = rows
27225 .get(r_off)
27226 .and_then(|r| r.get(c_off))
27227 .cloned()
27228 .unwrap_or(LiteralValue::Empty);
27229 let sheet_name = self.graph.sheet_name(cell.sheet_id);
27230 let old = self
27231 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27232 .unwrap_or(LiteralValue::Empty);
27233 if old != new {
27234 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27235 }
27236 }
27237 } else {
27238 for cell in targets.iter() {
27240 let sheet_name = self.graph.sheet_name(cell.sheet_id);
27241 let old = self
27242 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27243 .unwrap_or(LiteralValue::Empty);
27244 if !matches!(old, LiteralValue::Empty) {
27245 delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27246 }
27247 }
27248 }
27249 }
27250
27251 let arrow_sheets = &self.arrow_sheets;
27254 self.spill_mgr.commit_array_with_value_probe(
27255 &mut self.graph,
27256 anchor_vertex,
27257 targets,
27258 rows.clone(),
27259 overwritable_formulas,
27260 |g, cell| {
27261 let sheet_name = g.sheet_name(cell.sheet_id);
27262 let asheet = arrow_sheets.sheet(sheet_name)?;
27263 let r0 = cell.coord.row() as usize;
27264 let c0 = cell.coord.col() as usize;
27265 let v = asheet.get_cell_value(r0, c0);
27266 if matches!(v, LiteralValue::Empty) {
27267 None
27268 } else {
27269 Some(v)
27270 }
27271 },
27272 )?;
27273
27274 if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
27275 self.record_formula_plane_structural_change(scope);
27276 }
27277 if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
27278 self.record_formula_plane_structural_change(scope);
27279 }
27280
27281 if self.config.arrow_storage_enabled
27282 && self.config.delta_overlay_enabled
27283 && self.config.write_formula_overlay_enabled
27284 {
27285 if !prev_spill_cells.is_empty() {
27286 let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
27287 targets.iter().copied().collect();
27288 let empty = LiteralValue::Empty;
27289 for cell in prev_spill_cells.iter() {
27290 if !target_set.contains(cell) {
27291 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27292 self.mirror_value_to_computed_overlay(
27293 &sheet_name,
27294 cell.coord.row() + 1,
27295 cell.coord.col() + 1,
27296 &empty,
27297 );
27298 }
27299 }
27300 }
27301
27302 for (idx, cell) in targets.iter().enumerate() {
27303 if rows.is_empty() || rows[0].is_empty() {
27304 break;
27305 }
27306 let width = rows[0].len();
27307 let r_off = idx / width;
27308 let c_off = idx % width;
27309 let v = rows[r_off][c_off].clone();
27310 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27311 self.mirror_value_to_computed_overlay(
27312 &sheet_name,
27313 cell.coord.row() + 1,
27314 cell.coord.col() + 1,
27315 &v,
27316 );
27317 }
27318 }
27319 Ok(())
27320 }
27321}
27322
27323use crate::engine::effects::Effect;
27328use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
27329
27330impl<R> Engine<R>
27331where
27332 R: EvaluationContext,
27333{
27334 pub(crate) fn plan_vertex_effects(
27341 &mut self,
27342 vertex_id: VertexId,
27343 computed_value: LiteralValue,
27344 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27345 ) -> Result<Vec<Effect>, ExcelError> {
27346 let kind = self.graph.get_vertex_kind(vertex_id);
27347 let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
27348
27349 if !is_formula {
27353 if let Some(cell) = self.graph.get_cell_ref(vertex_id)
27354 && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
27355 && owner != vertex_id
27356 {
27357 return Ok(Vec::new());
27358 }
27359 return Ok(vec![Effect::WriteCell {
27361 vertex_id,
27362 value: computed_value,
27363 }]);
27364 }
27365
27366 match computed_value {
27367 LiteralValue::Array(rows) => {
27368 self.plan_array_effects(vertex_id, rows, overwritable_formulas)
27369 }
27370 other => self.plan_scalar_effects(vertex_id, other),
27371 }
27372 }
27373
27374 fn plan_scalar_effects(
27376 &self,
27377 vertex_id: VertexId,
27378 value: LiteralValue,
27379 ) -> Result<Vec<Effect>, ExcelError> {
27380 let has_spill = self
27381 .graph
27382 .spill_cells_for_anchor(vertex_id)
27383 .is_some_and(|c| !c.is_empty());
27384
27385 let mut effects = Vec::new();
27386 if has_spill {
27387 effects.push(Effect::SpillClear {
27388 anchor_vertex: vertex_id,
27389 });
27390 }
27391 effects.push(Effect::WriteCell { vertex_id, value });
27392 Ok(effects)
27393 }
27394
27395 fn plan_array_effects(
27397 &mut self,
27398 vertex_id: VertexId,
27399 rows: Vec<Vec<LiteralValue>>,
27400 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27401 ) -> Result<Vec<Effect>, ExcelError> {
27402 self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
27404
27405 let anchor = self
27406 .graph
27407 .get_cell_ref(vertex_id)
27408 .expect("cell ref for vertex");
27409 let sheet_id = anchor.sheet_id;
27410 let h = rows.len() as u32;
27411 let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
27412
27413 let spill_cells = (h as u64).saturating_mul(w as u64);
27415 if spill_cells > self.config.spill.max_spill_cells as u64 {
27416 return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
27417 }
27418
27419 const PACKED_MAX_ROW: u32 = 1_048_575;
27421 const PACKED_MAX_COL: u32 = 16_383;
27422 let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
27423 let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
27424 if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
27425 return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
27426 }
27427
27428 let mut targets = Vec::new();
27429 for r in 0..h {
27430 for c in 0..w {
27431 targets.push(self.graph.make_cell_ref_internal(
27432 sheet_id,
27433 anchor.coord.row() + r,
27434 anchor.coord.col() + c,
27435 ));
27436 }
27437 }
27438
27439 match self.spill_mgr.reserve(
27441 vertex_id,
27442 anchor,
27443 SpillShape { rows: h, cols: w },
27444 SpillMeta {
27445 epoch: self.recalc_epoch,
27446 config: self.config.spill,
27447 },
27448 ) {
27449 Ok(()) => {
27450 if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
27452 vertex_id,
27453 &targets,
27454 overwritable_formulas,
27455 ) {
27456 return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
27457 }
27458
27459 if !self.graph.value_cache_enabled() {
27463 let sheet_name = self.graph.sheet_name(sheet_id);
27464 if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
27465 for cell in targets.iter() {
27466 if *cell == anchor {
27468 continue;
27469 }
27470 if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
27472 continue;
27473 }
27474 if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
27476 && vid != vertex_id
27477 {
27478 match self.graph.get_vertex_kind(vid) {
27479 VertexKind::FormulaScalar | VertexKind::FormulaArray => {
27480 continue;
27481 }
27482 _ => {}
27483 }
27484 }
27485
27486 let v = asheet.get_cell_value(
27487 cell.coord.row() as usize,
27488 cell.coord.col() as usize,
27489 );
27490 if !matches!(v, LiteralValue::Empty) {
27491 return self.plan_spill_error_effects(
27492 vertex_id,
27493 "BlockedByValue",
27494 h,
27495 w,
27496 );
27497 }
27498 }
27499 }
27500 }
27501
27502 let top_left = rows
27503 .first()
27504 .and_then(|r| r.first())
27505 .cloned()
27506 .unwrap_or(LiteralValue::Empty);
27507
27508 let mut effects = Vec::new();
27509 let has_prev = self
27511 .graph
27512 .spill_cells_for_anchor(vertex_id)
27513 .is_some_and(|c| !c.is_empty());
27514 if has_prev {
27515 effects.push(Effect::SpillClear {
27516 anchor_vertex: vertex_id,
27517 });
27518 }
27519 effects.push(Effect::SpillCommit {
27520 anchor_vertex: vertex_id,
27521 anchor_cell: anchor,
27522 target_cells: targets,
27523 values: rows,
27524 });
27525 effects.push(Effect::WriteCell {
27526 vertex_id,
27527 value: top_left,
27528 });
27529 Ok(effects)
27530 }
27531 Err(e) => {
27532 let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
27533 self.plan_spill_error_effects(vertex_id, &msg, h, w)
27534 }
27535 }
27536 }
27537
27538 fn plan_spill_error_effects(
27540 &self,
27541 vertex_id: VertexId,
27542 message: &str,
27543 expected_rows: u32,
27544 expected_cols: u32,
27545 ) -> Result<Vec<Effect>, ExcelError> {
27546 let spill_err = ExcelError::new(ExcelErrorKind::Spill)
27547 .with_message(message)
27548 .with_extra(formualizer_common::ExcelErrorExtra::Spill {
27549 expected_rows,
27550 expected_cols,
27551 });
27552 let spill_val = LiteralValue::Error(spill_err);
27553
27554 let effects = vec![
27555 Effect::SpillClear {
27556 anchor_vertex: vertex_id,
27557 },
27558 Effect::WriteCell {
27559 vertex_id,
27560 value: spill_val,
27561 },
27562 ];
27563 Ok(effects)
27564 }
27565
27566 pub(crate) fn apply_effect(
27568 &mut self,
27569 effect: &Effect,
27570 delta: Option<&mut DeltaCollector>,
27571 log: Option<&mut ChangeLog>,
27572 ) -> Result<(), ExcelError> {
27573 self.apply_effect_with_computed_writes(effect, delta, log, None)
27574 }
27575
27576 fn apply_effect_with_computed_writes(
27577 &mut self,
27578 effect: &Effect,
27579 delta: Option<&mut DeltaCollector>,
27580 log: Option<&mut ChangeLog>,
27581 computed_writes: Option<&mut ComputedWriteBuffer>,
27582 ) -> Result<(), ExcelError> {
27583 match effect {
27584 Effect::WriteCell { vertex_id, value } => {
27585 self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
27586 }
27587 Effect::SpillClear { anchor_vertex } => {
27588 self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
27589 }
27590 Effect::SpillCommit {
27591 anchor_vertex,
27592 anchor_cell: _,
27593 target_cells,
27594 values,
27595 } => {
27596 self.apply_spill_commit(
27597 *anchor_vertex,
27598 target_cells,
27599 values.clone(),
27600 delta,
27601 log,
27602 computed_writes,
27603 )?;
27604 }
27605 }
27606 Ok(())
27607 }
27608
27609 fn apply_write_cell(
27611 &mut self,
27612 vertex_id: VertexId,
27613 value: &LiteralValue,
27614 delta: Option<&mut DeltaCollector>,
27615 mut computed_writes: Option<&mut ComputedWriteBuffer>,
27616 ) -> Result<(), ExcelError> {
27617 if let Some(d) = delta
27618 && d.mode != DeltaMode::Off
27619 {
27620 if let Some(buffer) = computed_writes.as_deref_mut() {
27621 self.flush_computed_write_buffer(buffer)?;
27622 }
27623 if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
27624 let sheet_name = self.graph.sheet_name(cell.sheet_id);
27625 let old = self
27626 .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27627 .unwrap_or(LiteralValue::Empty);
27628 if old != *value {
27629 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27630 }
27631 }
27632 }
27633 self.graph.update_vertex_value(vertex_id, value.clone());
27634 self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
27635 Ok(())
27636 }
27637
27638 fn apply_spill_clear(
27640 &mut self,
27641 anchor_vertex: VertexId,
27642 delta: Option<&mut DeltaCollector>,
27643 log: Option<&mut ChangeLog>,
27644 computed_writes: Option<&mut ComputedWriteBuffer>,
27645 ) -> Result<(), ExcelError> {
27646 if let Some(buffer) = computed_writes {
27647 self.flush_computed_write_buffer(buffer)?;
27648 }
27649
27650 let spill_cells = self
27651 .graph
27652 .spill_cells_for_anchor(anchor_vertex)
27653 .map(|cells| cells.to_vec())
27654 .unwrap_or_default();
27655 if spill_cells.is_empty() {
27656 return Ok(());
27657 }
27658
27659 let snapshot = if log.is_some() {
27661 self.snapshot_spill_for_anchor(anchor_vertex)
27662 } else {
27663 None
27664 };
27665
27666 if let Some(d) = delta
27668 && d.mode != DeltaMode::Off
27669 {
27670 let empty = LiteralValue::Empty;
27671 for cell in spill_cells.iter() {
27672 let sheet_name = self.graph.sheet_name(cell.sheet_id);
27673 let old = self
27674 .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27675 .unwrap_or(LiteralValue::Empty);
27676 if old != empty {
27677 d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27678 }
27679 }
27680 }
27681
27682 self.graph.clear_spill_region(anchor_vertex);
27683 if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
27684 self.record_formula_plane_structural_change(scope);
27685 }
27686
27687 if self.config.arrow_storage_enabled
27689 && self.config.delta_overlay_enabled
27690 && self.config.write_formula_overlay_enabled
27691 {
27692 let empty = LiteralValue::Empty;
27693 for cell in spill_cells.iter() {
27694 let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27695 self.mirror_value_to_computed_overlay(
27696 &sheet_name,
27697 cell.coord.row() + 1,
27698 cell.coord.col() + 1,
27699 &empty,
27700 );
27701 }
27702 }
27703
27704 if let Some(log) = log
27706 && let Some(old) = snapshot
27707 {
27708 log.record(ChangeEvent::SpillCleared {
27709 anchor: anchor_vertex,
27710 old,
27711 });
27712 }
27713 Ok(())
27714 }
27715
27716 fn apply_spill_commit(
27718 &mut self,
27719 anchor_vertex: VertexId,
27720 target_cells: &[CellRef],
27721 values: Vec<Vec<LiteralValue>>,
27722 delta: Option<&mut DeltaCollector>,
27723 log: Option<&mut ChangeLog>,
27724 computed_writes: Option<&mut ComputedWriteBuffer>,
27725 ) -> Result<(), ExcelError> {
27726 if let Some(buffer) = computed_writes {
27727 self.flush_computed_write_buffer(buffer)?;
27728 }
27729
27730 let old_snapshot = if log.is_some() {
27732 self.snapshot_spill_for_anchor(anchor_vertex)
27733 } else {
27734 None
27735 };
27736
27737 self.commit_spill_and_mirror(
27739 anchor_vertex,
27740 target_cells,
27741 values.clone(),
27742 delta,
27743 None, )?;
27745
27746 if let Some(log) = log {
27748 log.record(ChangeEvent::SpillCommitted {
27749 anchor: anchor_vertex,
27750 old: old_snapshot,
27751 new: SpillSnapshot {
27752 target_cells: target_cells.to_vec(),
27753 values,
27754 },
27755 });
27756 }
27757 Ok(())
27758 }
27759
27760 fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
27765 let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
27766 if cells.is_empty() {
27767 return None;
27768 }
27769
27770 let max = self.config.spill.max_spill_cells as usize;
27771 let mut cells = cells;
27772 if cells.len() > max {
27773 cells.truncate(max);
27774 }
27775
27776 let first = *cells.first().expect("non-empty spill cells");
27777 let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
27778 let row0 = first.coord.row();
27779 let col0 = first.coord.col();
27780
27781 let mut max_row = row0;
27782 let mut max_col = col0;
27783 let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
27784 for cell in &cells {
27785 max_row = max_row.max(cell.coord.row());
27786 max_col = max_col.max(cell.coord.col());
27787 let v = self
27788 .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27789 .unwrap_or(LiteralValue::Empty);
27790 by_coord.insert((cell.coord.row(), cell.coord.col()), v);
27791 }
27792
27793 let rows = (max_row - row0 + 1) as usize;
27794 let cols = (max_col - col0 + 1) as usize;
27795 let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
27796 for r in 0..rows {
27797 let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
27798 for c in 0..cols {
27799 row.push(
27800 by_coord
27801 .get(&(row0 + r as u32, col0 + c as u32))
27802 .cloned()
27803 .unwrap_or(LiteralValue::Empty),
27804 );
27805 }
27806 values.push(row);
27807 }
27808
27809 Some(SpillSnapshot {
27810 target_cells: cells,
27811 values,
27812 })
27813 }
27814
27815 fn flush_before_range_dependent_vertex(
27816 &mut self,
27817 vertex_id: VertexId,
27818 computed_writes: &mut ComputedWriteBuffer,
27819 ) -> Result<(), ExcelError> {
27820 if self.graph.get_range_dependencies(vertex_id).is_some() {
27821 self.flush_computed_write_buffer(computed_writes)?;
27822 }
27823 Ok(())
27824 }
27825
27826 fn plan_vertex_effects_with_computed_flush(
27827 &mut self,
27828 vertex_id: VertexId,
27829 computed_value: LiteralValue,
27830 overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27831 computed_writes: &mut ComputedWriteBuffer,
27832 ) -> Result<Vec<Effect>, ExcelError> {
27833 if matches!(&computed_value, LiteralValue::Array(_)) {
27834 self.flush_computed_write_buffer(computed_writes)?;
27835 }
27836 self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
27837 }
27838
27839 fn evaluate_small_layer_direct_effects(
27842 &mut self,
27843 layer: &super::scheduler::Layer,
27844 mut delta: Option<&mut DeltaCollector>,
27845 mut log: Option<&mut ChangeLog>,
27846 cancel_flag: Option<&AtomicBool>,
27847 cancel_check_every: usize,
27848 cancel_message: &'static str,
27849 ) -> Result<usize, ExcelError> {
27850 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
27851 if cancel_check_every > 0
27852 && i % cancel_check_every == 0
27853 && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
27854 {
27855 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
27856 .with_message(cancel_message.to_string()));
27857 }
27858 let value = match self.evaluate_vertex_immutable(vertex_id) {
27859 Ok(v) => v,
27860 Err(e) => LiteralValue::Error(e),
27861 };
27862 let effects = self.plan_vertex_effects(vertex_id, value, None)?;
27863 for effect in &effects {
27864 self.apply_effect_with_computed_writes(
27865 effect,
27866 delta.as_deref_mut(),
27867 log.as_deref_mut(),
27868 None,
27869 )?;
27870 }
27871 }
27872 Ok(layer.vertices.len())
27873 }
27874
27875 fn evaluate_layer_sequential_effects(
27877 &mut self,
27878 layer: &super::scheduler::Layer,
27879 ) -> Result<usize, ExcelError> {
27880 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27881 return self.evaluate_small_layer_direct_effects(
27882 layer,
27883 None,
27884 None,
27885 None,
27886 0,
27887 "Evaluation cancelled within layer",
27888 );
27889 }
27890
27891 let mut computed_writes = ComputedWriteBuffer::default();
27892 for &vertex_id in &layer.vertices {
27893 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
27894 let value = match self.evaluate_vertex_immutable(vertex_id) {
27895 Ok(v) => v,
27896 Err(e) => LiteralValue::Error(e),
27897 };
27898 let effects = match self.plan_vertex_effects_with_computed_flush(
27899 vertex_id,
27900 value,
27901 None,
27902 &mut computed_writes,
27903 ) {
27904 Ok(effects) => effects,
27905 Err(e) => {
27906 self.flush_computed_write_buffer(&mut computed_writes)?;
27907 return Err(e);
27908 }
27909 };
27910 for effect in &effects {
27911 if let Err(e) = self.apply_effect_with_computed_writes(
27912 effect,
27913 None,
27914 None,
27915 Some(&mut computed_writes),
27916 ) {
27917 self.flush_computed_write_buffer(&mut computed_writes)?;
27918 return Err(e);
27919 }
27920 }
27921 }
27922 self.flush_computed_write_buffer(&mut computed_writes)?;
27923 Ok(layer.vertices.len())
27924 }
27925
27926 fn evaluate_layer_sequential_with_delta_effects(
27928 &mut self,
27929 layer: &super::scheduler::Layer,
27930 delta: &mut DeltaCollector,
27931 ) -> Result<usize, ExcelError> {
27932 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27933 return self.evaluate_small_layer_direct_effects(
27934 layer,
27935 Some(delta),
27936 None,
27937 None,
27938 0,
27939 "Evaluation cancelled within layer",
27940 );
27941 }
27942
27943 let mut computed_writes = ComputedWriteBuffer::default();
27944 for &vertex_id in &layer.vertices {
27945 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
27946 let value = match self.evaluate_vertex_immutable(vertex_id) {
27947 Ok(v) => v,
27948 Err(e) => LiteralValue::Error(e),
27949 };
27950 let effects = match self.plan_vertex_effects_with_computed_flush(
27951 vertex_id,
27952 value,
27953 None,
27954 &mut computed_writes,
27955 ) {
27956 Ok(effects) => effects,
27957 Err(e) => {
27958 self.flush_computed_write_buffer(&mut computed_writes)?;
27959 return Err(e);
27960 }
27961 };
27962 for effect in &effects {
27963 if let Err(e) = self.apply_effect_with_computed_writes(
27964 effect,
27965 Some(delta),
27966 None,
27967 Some(&mut computed_writes),
27968 ) {
27969 self.flush_computed_write_buffer(&mut computed_writes)?;
27970 return Err(e);
27971 }
27972 }
27973 }
27974 self.flush_computed_write_buffer(&mut computed_writes)?;
27975 Ok(layer.vertices.len())
27976 }
27977
27978 fn evaluate_layer_sequential_cancellable_effects(
27980 &mut self,
27981 layer: &super::scheduler::Layer,
27982 cancel_flag: &AtomicBool,
27983 ) -> Result<usize, ExcelError> {
27984 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27985 return self.evaluate_small_layer_direct_effects(
27986 layer,
27987 None,
27988 None,
27989 Some(cancel_flag),
27990 256,
27991 "Evaluation cancelled within layer",
27992 );
27993 }
27994
27995 let mut computed_writes = ComputedWriteBuffer::default();
27996 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
27997 if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
27998 self.flush_computed_write_buffer(&mut computed_writes)?;
27999 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28000 .with_message("Evaluation cancelled within layer".to_string()));
28001 }
28002 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28003 let value = match self.evaluate_vertex_immutable(vertex_id) {
28004 Ok(v) => v,
28005 Err(e) => LiteralValue::Error(e),
28006 };
28007 let effects = match self.plan_vertex_effects_with_computed_flush(
28008 vertex_id,
28009 value,
28010 None,
28011 &mut computed_writes,
28012 ) {
28013 Ok(effects) => effects,
28014 Err(e) => {
28015 self.flush_computed_write_buffer(&mut computed_writes)?;
28016 return Err(e);
28017 }
28018 };
28019 for effect in &effects {
28020 if let Err(e) = self.apply_effect_with_computed_writes(
28021 effect,
28022 None,
28023 None,
28024 Some(&mut computed_writes),
28025 ) {
28026 self.flush_computed_write_buffer(&mut computed_writes)?;
28027 return Err(e);
28028 }
28029 }
28030 }
28031 self.flush_computed_write_buffer(&mut computed_writes)?;
28032 Ok(layer.vertices.len())
28033 }
28034
28035 fn evaluate_layer_sequential_cancellable_demand_driven_effects(
28037 &mut self,
28038 layer: &super::scheduler::Layer,
28039 cancel_flag: &AtomicBool,
28040 ) -> Result<usize, ExcelError> {
28041 if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
28042 return self.evaluate_small_layer_direct_effects(
28043 layer,
28044 None,
28045 None,
28046 Some(cancel_flag),
28047 128,
28048 "Demand-driven evaluation cancelled within layer",
28049 );
28050 }
28051
28052 let mut computed_writes = ComputedWriteBuffer::default();
28053 for (i, &vertex_id) in layer.vertices.iter().enumerate() {
28054 if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
28055 self.flush_computed_write_buffer(&mut computed_writes)?;
28056 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28057 .with_message("Demand-driven evaluation cancelled within layer".to_string()));
28058 }
28059 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28060 let value = match self.evaluate_vertex_immutable(vertex_id) {
28061 Ok(v) => v,
28062 Err(e) => LiteralValue::Error(e),
28063 };
28064 let effects = match self.plan_vertex_effects_with_computed_flush(
28065 vertex_id,
28066 value,
28067 None,
28068 &mut computed_writes,
28069 ) {
28070 Ok(effects) => effects,
28071 Err(e) => {
28072 self.flush_computed_write_buffer(&mut computed_writes)?;
28073 return Err(e);
28074 }
28075 };
28076 for effect in &effects {
28077 if let Err(e) = self.apply_effect_with_computed_writes(
28078 effect,
28079 None,
28080 None,
28081 Some(&mut computed_writes),
28082 ) {
28083 self.flush_computed_write_buffer(&mut computed_writes)?;
28084 return Err(e);
28085 }
28086 }
28087 }
28088 self.flush_computed_write_buffer(&mut computed_writes)?;
28089 Ok(layer.vertices.len())
28090 }
28091
28092 fn evaluate_layer_parallel_effects(
28094 &mut self,
28095 layer: &super::scheduler::Layer,
28096 ) -> Result<usize, ExcelError> {
28097 use rayon::prelude::*;
28098
28099 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28100
28101 let mut phase1: Vec<VertexId> = Vec::new();
28102 let mut phase2: Vec<VertexId> = Vec::new();
28103 for &vid in &layer.vertices {
28104 if self.graph.get_range_dependencies(vid).is_some() {
28105 phase2.push(vid);
28106 } else {
28107 phase1.push(vid);
28108 }
28109 }
28110
28111 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28112 let mut applied = 0usize;
28113
28114 for group in [&phase1[..], &phase2[..]] {
28115 if group.is_empty() {
28116 continue;
28117 }
28118 let mut computed_writes = ComputedWriteBuffer::default();
28119
28120 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28121 thread_pool.install(|| {
28122 group
28123 .par_iter()
28124 .map(
28125 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
28126 Ok(v) => Ok((vertex_id, v)),
28127 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28128 },
28129 )
28130 .collect()
28131 });
28132
28133 match results {
28134 Ok(vertex_results) => {
28135 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28138 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28139 for (vertex_id, result) in vertex_results {
28140 if matches!(result, LiteralValue::Array(_)) {
28141 arrays.push((vertex_id, result));
28142 } else {
28143 others.push((vertex_id, result));
28144 }
28145 }
28146 for (vertex_id, result) in arrays {
28147 let effects = match self.plan_vertex_effects_with_computed_flush(
28148 vertex_id,
28149 result,
28150 Some(&inflight),
28151 &mut computed_writes,
28152 ) {
28153 Ok(effects) => effects,
28154 Err(e) => {
28155 self.flush_computed_write_buffer(&mut computed_writes)?;
28156 return Err(e);
28157 }
28158 };
28159 for effect in &effects {
28160 if let Err(e) = self.apply_effect_with_computed_writes(
28161 effect,
28162 None,
28163 None,
28164 Some(&mut computed_writes),
28165 ) {
28166 self.flush_computed_write_buffer(&mut computed_writes)?;
28167 return Err(e);
28168 }
28169 }
28170 applied = applied.saturating_add(1);
28171 }
28172 self.flush_computed_write_buffer(&mut computed_writes)?;
28174 for (vertex_id, result) in others {
28175 let effects = match self.plan_vertex_effects_with_computed_flush(
28176 vertex_id,
28177 result,
28178 Some(&inflight),
28179 &mut computed_writes,
28180 ) {
28181 Ok(effects) => effects,
28182 Err(e) => {
28183 self.flush_computed_write_buffer(&mut computed_writes)?;
28184 return Err(e);
28185 }
28186 };
28187 for effect in &effects {
28188 if let Err(e) = self.apply_effect_with_computed_writes(
28189 effect,
28190 None,
28191 None,
28192 Some(&mut computed_writes),
28193 ) {
28194 self.flush_computed_write_buffer(&mut computed_writes)?;
28195 return Err(e);
28196 }
28197 }
28198 applied = applied.saturating_add(1);
28199 }
28200 self.flush_computed_write_buffer(&mut computed_writes)?;
28202 }
28203 Err(e) => {
28204 self.flush_computed_write_buffer(&mut computed_writes)?;
28205 return Err(e);
28206 }
28207 }
28208 }
28209
28210 Ok(applied)
28211 }
28212
28213 fn evaluate_layer_parallel_with_delta_effects(
28215 &mut self,
28216 layer: &super::scheduler::Layer,
28217 delta: &mut DeltaCollector,
28218 ) -> Result<usize, ExcelError> {
28219 use rayon::prelude::*;
28220
28221 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28222
28223 let mut phase1: Vec<VertexId> = Vec::new();
28224 let mut phase2: Vec<VertexId> = Vec::new();
28225 for &vid in &layer.vertices {
28226 if self.graph.get_range_dependencies(vid).is_some() {
28227 phase2.push(vid);
28228 } else {
28229 phase1.push(vid);
28230 }
28231 }
28232
28233 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28234 let mut applied = 0usize;
28235
28236 for group in [&phase1[..], &phase2[..]] {
28237 if group.is_empty() {
28238 continue;
28239 }
28240 let mut computed_writes = ComputedWriteBuffer::default();
28241 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28242 thread_pool.install(|| {
28243 group
28244 .par_iter()
28245 .map(
28246 |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
28247 Ok(v) => Ok((vertex_id, v)),
28248 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28249 },
28250 )
28251 .collect()
28252 });
28253
28254 match results {
28255 Ok(vertex_results) => {
28256 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28257 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28258 for (vertex_id, result) in vertex_results {
28259 if matches!(result, LiteralValue::Array(_)) {
28260 arrays.push((vertex_id, result));
28261 } else {
28262 others.push((vertex_id, result));
28263 }
28264 }
28265 for (vertex_id, result) in arrays {
28266 let effects = match self.plan_vertex_effects_with_computed_flush(
28267 vertex_id,
28268 result,
28269 Some(&inflight),
28270 &mut computed_writes,
28271 ) {
28272 Ok(effects) => effects,
28273 Err(e) => {
28274 self.flush_computed_write_buffer(&mut computed_writes)?;
28275 return Err(e);
28276 }
28277 };
28278 for effect in &effects {
28279 if let Err(e) = self.apply_effect_with_computed_writes(
28280 effect,
28281 Some(delta),
28282 None,
28283 Some(&mut computed_writes),
28284 ) {
28285 self.flush_computed_write_buffer(&mut computed_writes)?;
28286 return Err(e);
28287 }
28288 }
28289 applied = applied.saturating_add(1);
28290 }
28291 self.flush_computed_write_buffer(&mut computed_writes)?;
28292 for (vertex_id, result) in others {
28293 let effects = match self.plan_vertex_effects_with_computed_flush(
28294 vertex_id,
28295 result,
28296 Some(&inflight),
28297 &mut computed_writes,
28298 ) {
28299 Ok(effects) => effects,
28300 Err(e) => {
28301 self.flush_computed_write_buffer(&mut computed_writes)?;
28302 return Err(e);
28303 }
28304 };
28305 for effect in &effects {
28306 if let Err(e) = self.apply_effect_with_computed_writes(
28307 effect,
28308 Some(delta),
28309 None,
28310 Some(&mut computed_writes),
28311 ) {
28312 self.flush_computed_write_buffer(&mut computed_writes)?;
28313 return Err(e);
28314 }
28315 }
28316 applied = applied.saturating_add(1);
28317 }
28318 self.flush_computed_write_buffer(&mut computed_writes)?;
28319 }
28320 Err(e) => {
28321 self.flush_computed_write_buffer(&mut computed_writes)?;
28322 return Err(e);
28323 }
28324 }
28325 }
28326
28327 Ok(applied)
28328 }
28329
28330 fn evaluate_layer_parallel_cancellable_effects(
28332 &mut self,
28333 layer: &super::scheduler::Layer,
28334 cancel_flag: &AtomicBool,
28335 ) -> Result<usize, ExcelError> {
28336 use rayon::prelude::*;
28337
28338 let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28339
28340 if cancel_flag.load(Ordering::Relaxed) {
28341 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28342 .with_message("Parallel evaluation cancelled before starting".to_string()));
28343 }
28344
28345 let mut phase1: Vec<VertexId> = Vec::new();
28346 let mut phase2: Vec<VertexId> = Vec::new();
28347 for &vid in &layer.vertices {
28348 if self.graph.get_range_dependencies(vid).is_some() {
28349 phase2.push(vid);
28350 } else {
28351 phase1.push(vid);
28352 }
28353 }
28354
28355 let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28356 let mut applied = 0usize;
28357
28358 for group in [&phase1[..], &phase2[..]] {
28359 if group.is_empty() {
28360 continue;
28361 }
28362 let mut computed_writes = ComputedWriteBuffer::default();
28363
28364 let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28365 thread_pool.install(|| {
28366 group
28367 .par_iter()
28368 .map(|&vertex_id| {
28369 if cancel_flag.load(Ordering::Relaxed) {
28370 return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28371 .with_message(
28372 "Parallel evaluation cancelled during execution"
28373 .to_string(),
28374 ));
28375 }
28376 match self.evaluate_vertex_immutable(vertex_id) {
28377 Ok(v) => Ok((vertex_id, v)),
28378 Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28379 }
28380 })
28381 .collect()
28382 });
28383
28384 match results {
28385 Ok(vertex_results) => {
28386 let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28387 let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28388 for (vertex_id, result) in vertex_results {
28389 if matches!(result, LiteralValue::Array(_)) {
28390 arrays.push((vertex_id, result));
28391 } else {
28392 others.push((vertex_id, result));
28393 }
28394 }
28395 for (vertex_id, result) in arrays {
28396 let effects = match self.plan_vertex_effects_with_computed_flush(
28397 vertex_id,
28398 result,
28399 Some(&inflight),
28400 &mut computed_writes,
28401 ) {
28402 Ok(effects) => effects,
28403 Err(e) => {
28404 self.flush_computed_write_buffer(&mut computed_writes)?;
28405 return Err(e);
28406 }
28407 };
28408 for effect in &effects {
28409 if let Err(e) = self.apply_effect_with_computed_writes(
28410 effect,
28411 None,
28412 None,
28413 Some(&mut computed_writes),
28414 ) {
28415 self.flush_computed_write_buffer(&mut computed_writes)?;
28416 return Err(e);
28417 }
28418 }
28419 applied = applied.saturating_add(1);
28420 }
28421 self.flush_computed_write_buffer(&mut computed_writes)?;
28422 for (vertex_id, result) in others {
28423 let effects = match self.plan_vertex_effects_with_computed_flush(
28424 vertex_id,
28425 result,
28426 Some(&inflight),
28427 &mut computed_writes,
28428 ) {
28429 Ok(effects) => effects,
28430 Err(e) => {
28431 self.flush_computed_write_buffer(&mut computed_writes)?;
28432 return Err(e);
28433 }
28434 };
28435 for effect in &effects {
28436 if let Err(e) = self.apply_effect_with_computed_writes(
28437 effect,
28438 None,
28439 None,
28440 Some(&mut computed_writes),
28441 ) {
28442 self.flush_computed_write_buffer(&mut computed_writes)?;
28443 return Err(e);
28444 }
28445 }
28446 applied = applied.saturating_add(1);
28447 }
28448 self.flush_computed_write_buffer(&mut computed_writes)?;
28449 }
28450 Err(e) => {
28451 self.flush_computed_write_buffer(&mut computed_writes)?;
28452 return Err(e);
28453 }
28454 }
28455 }
28456
28457 Ok(applied)
28458 }
28459
28460 pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
28467 self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
28468 engine.evaluate_all_logged_unobserved(log)
28469 })
28470 }
28471
28472 fn evaluate_all_logged_unobserved(
28473 &mut self,
28474 log: &mut ChangeLog,
28475 ) -> Result<EvalResult, ExcelError> {
28476 self.observe_function_semantic_epoch()?;
28477 let _source_cache = self.source_cache_session();
28478 self.validate_deterministic_mode()?;
28479 if self.config.defer_graph_building {
28480 self.build_graph_all()?;
28481 }
28482 self.transition_off_mode_spans_to_legacy()?;
28483 self.begin_evaluation_request();
28484 if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
28485 && self.graph.formula_authority().active_span_count() > 0
28486 {
28487 return self.evaluate_authoritative_formula_plane_all();
28488 }
28489 self.reset_virtual_dep_telemetry_if_disabled();
28490 let start = crate::instant::FzInstant::now();
28491 let mut computed_vertices = 0;
28492 let mut cycle_errors = 0;
28493
28494 let mut replan_iterations = 0;
28495 const MAX_REPLAN: usize = 5;
28496 let mut telemetry = self
28497 .config
28498 .enable_virtual_dep_telemetry
28499 .then(|| self.start_virtual_dep_telemetry());
28500
28501 log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
28502
28503 let result = (|| -> Result<EvalResult, ExcelError> {
28504 loop {
28505 let to_evaluate = self.graph.get_evaluation_vertices();
28506 if to_evaluate.is_empty() {
28507 if let Some(t) = telemetry.as_mut()
28508 && t.bailout_reason.is_none()
28509 {
28510 t.bailout_reason = Some("no_work");
28511 }
28512 break;
28513 }
28514
28515 let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
28516 if let Some(t) = telemetry.as_mut() {
28517 Self::accumulate_schedule_meta(t, &meta);
28518 }
28519
28520 for &unit in &schedule.units {
28523 match unit {
28524 ScheduleUnit::Cycle(i) => {
28525 if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
28536 {
28537 cycle_errors += 1;
28538 }
28539 }
28540 ScheduleUnit::Layer(i) => {
28541 computed_vertices +=
28542 self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
28543 }
28544 }
28545 }
28546
28547 let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
28548 if let Some(t) = telemetry.as_mut() {
28549 t.changed_vdeps_total += changed_vertices.len();
28550 }
28551 self.resource_checkpoint(0)?;
28552 self.graph.clear_dirty_flags(&to_evaluate);
28553 for v in &changed_vertices {
28554 self.graph.set_dirty(*v, true);
28555 }
28556
28557 if changed_vertices.is_empty() {
28558 if let Some(t) = telemetry.as_mut() {
28559 t.bailout_reason = Some("converged");
28560 }
28561 break;
28562 }
28563 if replan_iterations >= MAX_REPLAN {
28564 if let Some(mut t) = telemetry.take() {
28565 t.bailout_reason = Some("max_replan");
28566 t.replan_iterations = replan_iterations;
28567 self.last_virtual_dep_telemetry = t;
28568 }
28569 return Err(
28570 self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
28571 );
28572 }
28573 replan_iterations += 1;
28574 }
28575
28576 if let Some(mut t) = telemetry {
28577 t.replan_iterations = replan_iterations;
28578 self.last_virtual_dep_telemetry = t;
28579 }
28580
28581 self.redirty_for_next_recalc();
28582 self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
28583
28584 Ok(EvalResult {
28585 computed_vertices,
28586 cycle_errors,
28587 elapsed: start.elapsed(),
28588 })
28589 })();
28590 log.end_compound();
28591 result
28592 }
28593
28594 fn evaluate_layer_logged(
28596 &mut self,
28597 layer: &super::scheduler::Layer,
28598 log: &mut ChangeLog,
28599 ) -> Result<usize, ExcelError> {
28600 self.resource_checkpoint(layer.vertices.len() as u64)?;
28601 let mut computed_writes = ComputedWriteBuffer::default();
28602 for &vertex_id in &layer.vertices {
28603 self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28604 let value = match self.evaluate_vertex_immutable(vertex_id) {
28605 Ok(v) => v,
28606 Err(e) => LiteralValue::Error(e),
28607 };
28608 let effects = match self.plan_vertex_effects_with_computed_flush(
28609 vertex_id,
28610 value,
28611 None,
28612 &mut computed_writes,
28613 ) {
28614 Ok(effects) => effects,
28615 Err(e) => {
28616 self.flush_computed_write_buffer(&mut computed_writes)?;
28617 return Err(e);
28618 }
28619 };
28620 for effect in &effects {
28621 if let Err(e) = self.apply_effect_with_computed_writes(
28622 effect,
28623 None,
28624 Some(log),
28625 Some(&mut computed_writes),
28626 ) {
28627 self.flush_computed_write_buffer(&mut computed_writes)?;
28628 return Err(e);
28629 }
28630 }
28631 }
28632 self.flush_computed_write_buffer(&mut computed_writes)?;
28633 Ok(layer.vertices.len())
28634 }
28635}