Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::prepared_legacy_graph::{
8    PreparedLegacyGraphError, PreparedLegacyGraphPlan,
9};
10use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
11use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
12use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
13use crate::engine::live_graph::analyze_live_graph;
14use crate::engine::lookup_index_cache::{
15    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
16    LookupIndexKey, estimate_bytes,
17};
18use crate::engine::named_range::{NameScope, NamedDefinition};
19use crate::engine::range_view::RangeView;
20use crate::engine::row_visibility::RowVisibilityState;
21use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
22use crate::engine::target_preparation::{
23    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
24};
25use crate::engine::used_extent::{
26    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
27};
28use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
29use crate::engine::{
30    CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
31    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
32    EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
33    FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
34    FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
35    FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
36    FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
37    VertexId, VertexKind, VisibilityMaskMode,
38};
39use crate::formula_plane::placement::prepare_anchor_once_fragment;
40use crate::formula_plane::placement::{
41    CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
42    place_candidate_family_with_analyses, prepare_anchor_once_family,
43    split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
44    validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
45};
46use crate::formula_plane::producer::{
47    DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
48    FormulaProducerWork, ProducerDirtyDomain, SpanReadSummary,
49};
50use crate::formula_plane::region_index::{DirtyDomain, Region};
51use crate::formula_plane::runtime::{
52    FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
53};
54use crate::formula_plane::scheduler::{
55    MixedSchedule, MixedScheduleFallbackReason, MixedTopology, MixedTopologyCompileResult,
56    MixedTopologyCompileStats, MixedTopologyConfig, build_demand_closure_cached,
57    build_demand_closure_in_memory_runs, build_demand_closure_paged,
58    build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
59    schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
60    schedule_dirty_work_repeated_passes,
61};
62#[cfg(not(target_arch = "wasm32"))]
63use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
64#[cfg(not(target_arch = "wasm32"))]
65use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
66#[cfg(test)]
67use crate::formula_plane::span_eval::SpanEvalReport;
68use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
69use crate::formula_plane::structural::relocate_ast_for_template_placement;
70use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
71use crate::function::FnCaps;
72use crate::interpreter::Interpreter;
73use crate::reference::{CellRef, Coord, RangeRef};
74use crate::traits::FunctionProvider;
75use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
76use formualizer_common::{
77    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
78};
79use formualizer_parse::parser::ReferenceType;
80use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
81use rayon::ThreadPoolBuilder;
82use rustc_hash::{FxHashMap, FxHashSet};
83use std::collections::{BTreeMap, BTreeSet, VecDeque};
84use std::sync::Arc;
85use std::sync::atomic::{AtomicBool, Ordering};
86use std::time::Instant;
87
88type StagedFormulaEntry = (u32, u32, String);
89type StagedSheetParts = (
90    Vec<StagedFormulaEntry>,
91    Option<crate::engine::DeferredFormulaPackage>,
92);
93
94/// Per-sheet staged-formula store (NOTE(#126) follow-up).
95///
96/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
97/// entries in INSERTION order, so the order-preserving `Vec` stays the
98/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
99/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
100/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
101/// 50k stages, release, before the index). `stage`/`get` are O(1);
102/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
103#[derive(Default)]
104pub(crate) struct StagedSheet {
105    entries: Vec<StagedFormulaEntry>,
106    index: FxHashMap<(u32, u32), usize>,
107    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
108}
109
110impl StagedSheet {
111    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
112        package
113            .invalidated
114            .extend(package.families.iter().map(|family| family.source_id));
115        package.invalidated.extend(
116            package
117                .partitioned_families
118                .iter()
119                .map(|family| family.source_id),
120        );
121    }
122
123    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
124        let Some(package) = self.deferred_package.as_mut() else {
125            return;
126        };
127        let family = package
128            .replay
129            .lock()
130            .ok()
131            .and_then(|mut replay| replay.formula_at(row, col).ok())
132            .flatten()
133            .and_then(|record| record.partition_owner.or(record.family));
134        if let Some(family) = family {
135            package.invalidated.insert(family);
136        } else {
137            // A poisoned lock, replay failure, malformed spool, or missing
138            // lookup result must never let a possibly edited family commit.
139            // Fail closed at package scope before suppressing the edited cell.
140            Self::invalidate_all_deferred_families(package);
141        }
142        package.suppressed.insert((row, col));
143    }
144
145    fn reconcile_attached_deferred_package(&mut self) {
146        let Some(package) = self.deferred_package.as_mut() else {
147            return;
148        };
149        if self.entries.is_empty() {
150            return;
151        }
152
153        let replayed = (|| {
154            let mut disposition = crate::engine::FormulaReplayDisposition::default();
155            for partition in &package.partitioned_families {
156                disposition.register_partition(partition, false)?;
157            }
158            package
159                .replay
160                .lock()
161                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
162                .replay_partitioned(&disposition, &package.partitioned_families)
163        })();
164
165        let Ok(records) = replayed else {
166            Self::invalidate_all_deferred_families(package);
167            package
168                .suppressed
169                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
170            return;
171        };
172        let mut owners = FxHashMap::default();
173        for record in &records {
174            owners
175                .entry((record.row, record.col))
176                .or_insert(record.partition_owner.or(record.family));
177        }
178        for (row, col, _) in &self.entries {
179            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
180                package.invalidated.insert(family);
181            } else {
182                Self::invalidate_all_deferred_families(package);
183            }
184            package.suppressed.insert((*row, *col));
185        }
186        package.reconciliation_replay = Some(records);
187    }
188
189    fn stage(&mut self, row: u32, col: u32, text: String) {
190        self.invalidate_deferred_at(row, col);
191        match self.index.entry((row, col)) {
192            std::collections::hash_map::Entry::Occupied(slot) => {
193                self.entries[*slot.get()].2 = text;
194            }
195            std::collections::hash_map::Entry::Vacant(slot) => {
196                slot.insert(self.entries.len());
197                self.entries.push((row, col, text));
198            }
199        }
200    }
201
202    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
203        let deferred_text = self.get(row, col);
204        self.invalidate_deferred_at(row, col);
205        let Some(idx) = self.index.remove(&(row, col)) else {
206            return deferred_text;
207        };
208        let (_, _, text) = self.entries.remove(idx);
209        // `Vec::remove` shifted everything after `idx` left by one.
210        for slot in self.index.values_mut() {
211            if *slot > idx {
212                *slot -= 1;
213            }
214        }
215        Some(text)
216    }
217
218    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
219        self.index
220            .get(&(row, col))
221            .map(|&i| self.entries[i].2.as_str())
222    }
223
224    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
225        let idx = self.index.remove(&(row, col))?;
226        let (_, _, text) = self.entries.remove(idx);
227        for slot in self.index.values_mut() {
228            if *slot > idx {
229                *slot -= 1;
230            }
231        }
232        Some(text)
233    }
234
235    fn get(&self, row: u32, col: u32) -> Option<String> {
236        if let Some(text) = self.get_ordinary(row, col) {
237            return Some(text.to_string());
238        }
239        let package = self.deferred_package.as_ref()?;
240        if package.suppressed.contains(&(row, col)) {
241            return None;
242        }
243        package
244            .replay
245            .lock()
246            .ok()?
247            .formula_at(row, col)
248            .ok()?
249            .map(|record| record.text)
250    }
251
252    fn len(&self) -> usize {
253        self.deferred_package
254            .as_ref()
255            .map_or(self.entries.len(), |package| {
256                usize::try_from(package.report.source_formula_records_spooled)
257                    .unwrap_or(usize::MAX)
258                    .saturating_sub(package.suppressed.len())
259                    .saturating_add(self.entries.len())
260            })
261    }
262
263    fn is_empty(&self) -> bool {
264        self.entries.is_empty() && self.deferred_package.is_none()
265    }
266
267    fn into_parts(self) -> StagedSheetParts {
268        (self.entries, self.deferred_package)
269    }
270}
271
272type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
273
274#[derive(Clone, Debug, PartialEq, Eq, Hash)]
275struct PreparationRegion {
276    sheet: String,
277    sheet_id: SheetId,
278    start_row: u32,
279    start_col: u32,
280    end_row: u32,
281    end_col: u32,
282}
283
284// Span roots dedupe by exact demanded-region identity. Overlapping, non-identical
285// regions remain ordered entries; downstream demand treats those entries as a union.
286struct OrderedTargetProducers {
287    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
288    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
289}
290
291impl OrderedTargetProducers {
292    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
293        let mut ordered = Vec::new();
294        ordered.try_reserve(capacity)?;
295        let mut seen = FxHashSet::default();
296        seen.try_reserve(capacity)?;
297        Ok(Self { ordered, seen })
298    }
299
300    fn from_ordered(
301        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
302    ) -> Result<Self, std::collections::TryReserveError> {
303        let mut seen = FxHashSet::default();
304        seen.try_reserve(ordered.len())?;
305        seen.extend(ordered.iter().copied());
306        Ok(Self { ordered, seen })
307    }
308
309    fn push(
310        &mut self,
311        producer: crate::engine::target_preparation::TargetProducer,
312    ) -> Result<bool, std::collections::TryReserveError> {
313        #[cfg(test)]
314        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
315        if self.seen.contains(&producer) {
316            return Ok(false);
317        }
318        self.seen.try_reserve(1)?;
319        self.ordered.try_reserve(1)?;
320        self.seen.insert(producer);
321        self.ordered.push(producer);
322        Ok(true)
323    }
324
325    fn len(&self) -> usize {
326        self.ordered.len()
327    }
328
329    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
330        self.ordered
331    }
332}
333
334#[cfg(test)]
335thread_local! {
336    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
337}
338
339fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
340    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
341        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
342        limit: u64::MAX,
343        observed: observed as u64,
344        request_id,
345    })
346    .into_excel_error()
347}
348
349#[derive(Debug)]
350struct PreparedOrdinaryStagedFormula {
351    sheet: String,
352    sheet_id: SheetId,
353    lease: StagedFormulaLease,
354    ast_id: Option<AstNodeId>,
355    plan: Option<DependencyPlanRow>,
356}
357
358struct PreparedTargetSourcePackage {
359    sheet: String,
360    sheet_id: SheetId,
361    lease: StagedPackageLease,
362    source_report: crate::engine::FormulaCompressedSourceReport,
363    replay_records: Vec<crate::engine::DeferredReplayFormula>,
364    disposition: crate::engine::FormulaReplayDisposition,
365    placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
366    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
367    direct_families: usize,
368    direct_cells: u64,
369    direct_fragments: u64,
370    direct_complete_families: u64,
371    direct_complete_cells: u64,
372    direct_partition_families: u64,
373    direct_partition_cells: u64,
374    anchor_parses: u64,
375    anchor_asts: u64,
376    anchor_analyses: u64,
377}
378
379impl PreparedTargetSourcePackage {
380    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
381        self.replay_records.iter().filter(|record| {
382            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
383                return true;
384            };
385            let coord = crate::engine::SourceCoord { row, col };
386            let disposition = match record.family {
387                Some(family) => self.disposition.shared_disposition(family, coord),
388                None => self.disposition.ordinary_disposition(coord).0,
389            };
390            !matches!(
391                disposition,
392                crate::engine::FormulaReplayCoordinateDisposition::Direct
393                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
394            )
395        })
396    }
397}
398
399fn producer_dirty_to_span_dirty(
400    dirty: ProducerDirtyDomain,
401    span_ref: FormulaSpanRef,
402) -> DirtyDomain {
403    match dirty {
404        ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
405        ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
406        ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
407    }
408}
409type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
410type StagedFormulaBatches = Vec<(String, StagedSheet)>;
411#[derive(Debug)]
412pub(crate) struct SourceFamilyPreparationError {
413    reason: String,
414    parse_attempted: bool,
415    ast_created: bool,
416    analysis_created: bool,
417}
418
419impl SourceFamilyPreparationError {
420    fn contract(reason: &'static str) -> Self {
421        Self {
422            reason: reason.to_string(),
423            parse_attempted: false,
424            ast_created: false,
425            analysis_created: false,
426        }
427    }
428
429    fn parse(reason: &'static str) -> Self {
430        Self {
431            reason: reason.to_string(),
432            parse_attempted: true,
433            ast_created: false,
434            analysis_created: false,
435        }
436    }
437
438    fn ast(reason: impl Into<String>) -> Self {
439        Self {
440            reason: reason.into(),
441            parse_attempted: true,
442            ast_created: true,
443            analysis_created: false,
444        }
445    }
446
447    fn analysis(reason: impl Into<String>) -> Self {
448        Self {
449            reason: reason.into(),
450            parse_attempted: true,
451            ast_created: true,
452            analysis_created: true,
453        }
454    }
455}
456
457struct PreparedPartitionShadow {
458    fragment_count: u64,
459    direct_cells: u64,
460    fallback_cells: u64,
461    function_semantics_used: bool,
462}
463
464pub(crate) fn classify_mixed_topology_incomplete(
465    observed: &MixedTopologyCompileStats,
466) -> crate::engine::EvaluationIncompleteReason {
467    if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
468        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
469    } else if observed.memory_overflow_count > 0 {
470        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
471    } else if observed.edge_overflow_count > 0 {
472        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
473    } else if observed.candidate_overflow_count > 0 {
474        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
475    } else {
476        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
477    }
478}
479
480#[derive(Clone, Debug, Default)]
481struct LegacyIslandPlan {
482    membership: Vec<VertexId>,
483    dirty_vertices: Vec<VertexId>,
484    sheet_ids: Vec<SheetId>,
485    island_id: u64,
486    retained_bytes: usize,
487    boundary_relationships: usize,
488    omitted_relationships: usize,
489    omitted_direct_relationships: usize,
490}
491
492impl LegacyIslandPlan {
493    fn is_empty(&self) -> bool {
494        self.membership.is_empty()
495    }
496}
497
498type FormulaPlaneMixedScheduleBuild = (
499    MixedSchedule,
500    BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
501    u64,
502    Vec<VertexId>,
503    Vec<usize>,
504    LegacyIslandPlan,
505);
506
507#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub(crate) enum FormulaSpanDemotionFault {
509    #[default]
510    None,
511    AstPreparation,
512    LegacyGraphPreparation,
513    FinalLegacyGraphValidation,
514    FinalAuthorityValidation,
515    AllocationReservation,
516    BeforeFirstMutation,
517}
518
519#[derive(Debug)]
520pub(crate) enum FormulaSpanDemotionError {
521    StaleAuthority,
522    InvalidSpan,
523    CountOverflow,
524    Resource(ExcelError),
525    AstPreparation(ExcelError),
526    LegacyGraph(PreparedLegacyGraphError),
527    Injected(FormulaSpanDemotionFault),
528}
529
530impl std::fmt::Display for FormulaSpanDemotionError {
531    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
532        match self {
533            Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
534            Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
535            Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
536            Self::Resource(error) => write!(f, "{error}"),
537            Self::AstPreparation(error) => {
538                write!(f, "FormulaPlane demotion analysis failed: {error}")
539            }
540            Self::LegacyGraph(error) => {
541                write!(f, "FormulaPlane demotion graph plan failed: {error}")
542            }
543            Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
544        }
545    }
546}
547
548impl std::error::Error for FormulaSpanDemotionError {}
549
550pub(crate) struct PreparedFormulaSpanDemotion {
551    span_refs: Vec<FormulaSpanRef>,
552    expected_plane_epoch: u64,
553    expected_indexes_epoch: u64,
554    expected_indexed_plane_epoch: u64,
555    placement_count: usize,
556    legacy_graph: PreparedLegacyGraphPlan,
557    fault: FormulaSpanDemotionFault,
558}
559
560#[derive(Clone, Copy, Debug, PartialEq, Eq)]
561pub(crate) struct FormulaSpanDemotionReport {
562    pub(crate) spans_demoted: usize,
563    pub(crate) placements_materialized: usize,
564}
565
566type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
567type CompressedReplayBatch = (
568    FormulaIngestBatch,
569    crate::engine::FormulaCompressedSourceBatch,
570);
571type PreparedSourceBatch = (
572    FormulaIngestBatch,
573    crate::engine::FormulaCompressedSourceReport,
574    crate::engine::FormulaCompressedPreparation,
575);
576type PreparedStagedFormulaBatches = (
577    PreparedFormulaBatches,
578    Vec<CompressedReplayBatch>,
579    Vec<PreparedSourceBatch>,
580);
581
582/// Backend-neutral source-family ingress. Adapters may prepare candidates and
583/// submit exact replay, but FormulaPlane authority remains engine-owned.
584#[doc(hidden)]
585pub struct SourceFormulaIngress<'a, R> {
586    engine: &'a mut Engine<R>,
587}
588
589impl<R> SourceFormulaIngress<'_, R>
590where
591    R: EvaluationContext,
592{
593    pub fn prepare_families(
594        &mut self,
595        sheet_name: &str,
596        families: &[crate::engine::SourceFormulaFamily],
597    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
598        self.engine.observe_function_semantic_epoch()?;
599        Ok(self
600            .engine
601            .prepare_source_formula_families(sheet_name, families))
602    }
603
604    pub fn prepare_eager_proposals(
605        &mut self,
606        sheet_name: &str,
607        families: &[crate::engine::SourceFormulaFamily],
608        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
609        formula_record_count: u64,
610        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
611    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
612        self.engine.observe_function_semantic_epoch()?;
613        let replay = Arc::new(std::sync::Mutex::new(replay));
614        self.engine.prepare_source_formula_proposals(
615            sheet_name,
616            families,
617            partitions,
618            partitions,
619            formula_record_count,
620            replay,
621            &BTreeSet::new(),
622        )
623    }
624
625    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
626        self.engine.stage_deferred_formula_package(package);
627    }
628
629    pub fn ingest_replay_batches(
630        &mut self,
631        batches: Vec<(
632            FormulaIngestBatch,
633            crate::engine::FormulaCompressedSourceBatch,
634        )>,
635    ) -> Result<FormulaIngestReport, ExcelError> {
636        self.engine
637            .ingest_compressed_formula_source_batches(batches)
638    }
639
640    pub fn finish_prepared(
641        &mut self,
642        batches: Vec<(
643            FormulaIngestBatch,
644            crate::engine::FormulaCompressedSourceReport,
645            crate::engine::FormulaCompressedPreparation,
646        )>,
647    ) -> Result<FormulaIngestReport, ExcelError> {
648        self.engine.finish_compressed_formula_sources(batches)
649    }
650}
651
652// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
653// layers there is not enough work to amortize it, and the direct point-write path
654// is faster while preserving the same visibility semantics.
655const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
656
657#[derive(Debug, Clone, PartialEq)]
658pub(crate) enum ComputedWrite {
659    Cell {
660        seq: u64,
661        sheet_id: SheetId,
662        row0: u32,
663        col0: u32,
664        value: OverlayValue,
665    },
666    Rect {
667        seq: u64,
668        sheet_id: SheetId,
669        sr0: u32,
670        sc0: u32,
671        values: Vec<Vec<OverlayValue>>,
672    },
673}
674
675impl ComputedWrite {
676    #[inline]
677    pub(crate) fn seq(&self) -> u64 {
678        match self {
679            ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685pub(crate) struct ComputedWriteBuffer {
686    writes: Vec<ComputedWrite>,
687    next_seq: u64,
688    estimated_bytes: usize,
689}
690
691impl ComputedWriteBuffer {
692    const ENTRY_BASE_BYTES: usize = 32;
693
694    #[inline]
695    pub(crate) fn is_empty(&self) -> bool {
696        self.writes.is_empty()
697    }
698
699    #[inline]
700    pub(crate) fn len(&self) -> usize {
701        self.writes.len()
702    }
703
704    #[inline]
705    pub(crate) fn estimated_bytes(&self) -> usize {
706        self.estimated_bytes
707    }
708
709    #[inline]
710    pub(crate) fn writes(&self) -> &[ComputedWrite] {
711        &self.writes
712    }
713
714    pub(crate) fn push_cell(
715        &mut self,
716        sheet_id: SheetId,
717        row0: u32,
718        col0: u32,
719        value: OverlayValue,
720    ) {
721        let seq = self.next_sequence();
722        self.estimated_bytes = self
723            .estimated_bytes
724            .saturating_add(Self::estimate_value_bytes(&value));
725        self.writes.push(ComputedWrite::Cell {
726            seq,
727            sheet_id,
728            row0,
729            col0,
730            value,
731        });
732    }
733
734    pub(crate) fn push_rect(
735        &mut self,
736        sheet_id: SheetId,
737        sr0: u32,
738        sc0: u32,
739        values: Vec<Vec<OverlayValue>>,
740    ) {
741        let seq = self.next_sequence();
742        let added = values
743            .iter()
744            .flat_map(|row| row.iter())
745            .map(Self::estimate_value_bytes)
746            .fold(0usize, usize::saturating_add);
747        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
748        self.writes.push(ComputedWrite::Rect {
749            seq,
750            sheet_id,
751            sr0,
752            sc0,
753            values,
754        });
755    }
756
757    pub(crate) fn clear(&mut self) {
758        self.writes.clear();
759        self.estimated_bytes = 0;
760    }
761
762    fn take_writes(&mut self) -> Vec<ComputedWrite> {
763        self.estimated_bytes = 0;
764        std::mem::take(&mut self.writes)
765    }
766
767    fn next_sequence(&mut self) -> u64 {
768        let seq = self.next_seq;
769        self.next_seq = self.next_seq.wrapping_add(1);
770        seq
771    }
772
773    #[inline]
774    fn estimate_value_bytes(value: &OverlayValue) -> usize {
775        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
776    }
777}
778
779#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
780struct ComputedWriteChunkKey {
781    sheet_id: SheetId,
782    col0: u32,
783    chunk_idx: usize,
784    chunk_start_row0: u32,
785}
786
787#[derive(Debug, Clone, PartialEq)]
788pub(crate) struct ComputedWriteChunkEntryPlan {
789    pub(crate) row_in_chunk: usize,
790    pub(crate) seq: u64,
791    pub(crate) value: OverlayValue,
792}
793
794#[derive(Debug, Clone, PartialEq, Eq)]
795pub(crate) enum ComputedWriteChunkPlanShape {
796    Point,
797    SparseOffsets {
798        entries: usize,
799        span_len: usize,
800    },
801    DenseRange {
802        start: usize,
803        len: usize,
804    },
805    RunRange {
806        start: usize,
807        len: usize,
808        runs: usize,
809    },
810}
811
812#[derive(Debug, Clone, PartialEq)]
813pub(crate) struct ComputedWriteChunkPlan {
814    pub(crate) sheet_id: SheetId,
815    pub(crate) col0: u32,
816    pub(crate) chunk_idx: usize,
817    pub(crate) chunk_start_row0: u32,
818    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
819    pub(crate) shape: ComputedWriteChunkPlanShape,
820}
821
822#[derive(Debug, Clone, Default, PartialEq)]
823pub(crate) struct ComputedWriteCoalescingPlan {
824    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
825    pub(crate) input_cells: usize,
826    pub(crate) coalesced_cells: usize,
827    pub(crate) overwritten_cells: usize,
828}
829
830impl ComputedWriteCoalescingPlan {
831    #[inline]
832    pub(crate) fn is_empty(&self) -> bool {
833        self.chunks.is_empty()
834    }
835}
836
837impl ComputedWriteChunkPlan {
838    fn from_group(
839        key: ComputedWriteChunkKey,
840        mut entries: Vec<ComputedWriteChunkEntryPlan>,
841    ) -> (Self, usize) {
842        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
843        let input_len = entries.len();
844        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
845        for entry in entries {
846            if let Some(prev) = coalesced.last_mut()
847                && prev.row_in_chunk == entry.row_in_chunk
848            {
849                *prev = entry;
850                continue;
851            }
852            coalesced.push(entry);
853        }
854        let overwritten = input_len.saturating_sub(coalesced.len());
855        let shape = Self::classify_shape(&coalesced);
856        (
857            Self {
858                sheet_id: key.sheet_id,
859                col0: key.col0,
860                chunk_idx: key.chunk_idx,
861                chunk_start_row0: key.chunk_start_row0,
862                entries: coalesced,
863                shape,
864            },
865            overwritten,
866        )
867    }
868
869    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
870        debug_assert!(!entries.is_empty());
871        if entries.len() == 1 {
872            return ComputedWriteChunkPlanShape::Point;
873        }
874
875        let start = entries[0].row_in_chunk;
876        let end = entries[entries.len() - 1].row_in_chunk;
877        let span_len = end.saturating_sub(start).saturating_add(1);
878        if span_len != entries.len() {
879            return ComputedWriteChunkPlanShape::SparseOffsets {
880                entries: entries.len(),
881                span_len,
882            };
883        }
884
885        let runs = Self::run_count(entries);
886        if runs < entries.len() {
887            ComputedWriteChunkPlanShape::RunRange {
888                start,
889                len: entries.len(),
890                runs,
891            }
892        } else {
893            ComputedWriteChunkPlanShape::DenseRange {
894                start,
895                len: entries.len(),
896            }
897        }
898    }
899
900    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
901        let mut runs = 0usize;
902        let mut prev: Option<&OverlayValue> = None;
903        for entry in entries {
904            if prev != Some(&entry.value) {
905                runs = runs.saturating_add(1);
906                prev = Some(&entry.value);
907            }
908        }
909        runs
910    }
911}
912
913pub struct Engine<R> {
914    pub(crate) graph: DependencyGraph,
915    resolver: R,
916    pub config: EvalConfig,
917    workbook_load_limits: crate::engine::WorkbookLoadLimits,
918    /// Clock for volatile date/time builtins, wrapped in a per-recalc
919    /// snapshot: sampled once at the start of every evaluation request
920    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
921    /// one recalc — including SCC iteration passes — agree (spec §7.11).
922    clock: crate::timezone::SnapshotClock,
923    thread_pool: Option<Arc<rayon::ThreadPool>>,
924    pub recalc_epoch: u64,
925    snapshot_id: std::sync::atomic::AtomicU64,
926    topology_epoch: u64,
927    /// False after a structural axis operation. While #171 remains open we
928    /// must not use relocated span read summaries to prove disconnection.
929    legacy_island_structural_summaries_trusted: bool,
930    cached_static_schedule: Option<CachedScheduleEntry>,
931    cached_mixed_topology: Option<CachedMixedTopology>,
932    mixed_topology_cache_builds: u64,
933    mixed_topology_cache_hits: u64,
934    mixed_topology_cache_overflows: u64,
935    mixed_topology_cache_skip_streak: u64,
936    spill_mgr: ShimSpillManager,
937    /// Arrow-backed storage for sheet values (Phase A)
938    arrow_sheets: SheetStore,
939    /// True if any edit after bulk load; disables Arrow reads for parity
940    has_edited: bool,
941    /// Overlay compaction counter (Phase C instrumentation)
942    overlay_compactions: u64,
943
944    // Overlay memory observability / budget (ticket 503)
945    computed_overlay_bytes_estimate: usize,
946    computed_overlay_mirroring_disabled: bool,
947    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
948    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
949    pub(crate) force_materialize_range_views: bool,
950    // Pass-scoped cache for Arrow used-row bounds per column
951    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
952    // Snapshot-scoped final used-axis bounds for open-ended references.
953    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
954    lookup_index_cache: LookupIndexCache,
955    source_cache: Arc<std::sync::RwLock<SourceCache>>,
956    /// Identity binding for opaque source-family preparations.
957    source_formula_token: Arc<()>,
958    /// Dedicated identity binding for reusable recalculation plans.
959    recalc_plan_token: Arc<()>,
960    /// Staged formulas by sheet when `defer_graph_building` is enabled.
961    staged_formulas: StagedFormulaMap,
962    /// Presence and generation authority for ordinary staged formula discovery.
963    staged_formula_index: StagedFormulaIndex,
964    /// Per-sheet row visibility sidecar state.
965    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
966    /// Cached row visibility masks keyed by sheet/span/mode/version.
967    row_visibility_mask_cache: std::sync::RwLock<
968        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
969    >,
970    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
971    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
972    /// Last centralized formula ingest report.
973    last_formula_ingest_report: Option<FormulaIngestReport>,
974    /// Aggregate centralized formula ingest report for this engine.
975    formula_ingest_report_total: FormulaIngestReport,
976    /// Count of FormulaPlane spans demoted to legacy because one or more of
977    /// their member cells participate in a statically-cyclic SCC. A span member
978    /// must never be span-evaluated (gotcha G8 of the cycle-architecture track,
979    /// refs #112): under `CycleDetection::Static` the cycle stamping would race
980    /// span writes, and under `Runtime` SCC members must be evaluated by the
981    /// legacy `evaluate_scc_unit` path. Cyclic spans are demoted at
982    /// schedule-build time (the earliest point cross-cell cycles through span
983    /// producers become visible) so the cycle members land on the legacy graph
984    /// path. Observational only.
985    formula_plane_cycle_member_span_demotions: u64,
986    /// Successfully completed non-cycle unsafe mixed requests. Incremented
987    /// only after every scheduled span has committed demotion and the single
988    /// legacy completion pass succeeds; failed attempts are not counted.
989    /// Observational only.
990    formula_plane_capacity_bailouts: u64,
991    /// Exact span candidates classified across structural mutations.
992    formula_plane_structural_span_candidates: u64,
993    /// Transient cancellation flag used during evaluation
994    active_cancel_flag: Option<crate::engine::CancelToken>,
995    /// Transient absolute deadline used by composed target and plan requests.
996    active_evaluation_deadline: Option<Instant>,
997
998    /// Engine-level action depth.
999    ///
1000    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1001    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1002    action_depth: u32,
1003
1004    // Phase 3b virtual-dependency convergence telemetry
1005    last_virtual_dep_telemetry: VirtualDepTelemetry,
1006    virtual_dep_fallback_activations: u64,
1007
1008    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1009    last_cycle_telemetry: CycleTelemetry,
1010
1011    // C0 evaluation-resource observability. IDs are never reset or reused.
1012    next_evaluation_resource_request_id: u64,
1013    evaluation_resource_request_depth: usize,
1014    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1015    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1016    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1017    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1018    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1019    evaluation_resource_config_diagnostic:
1020        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1021    active_resource_ledger: Option<ResourceLedger>,
1022
1023    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1024    /// with a witnessed live cycle) during the current evaluation request.
1025    ///
1026    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1027    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1028    /// members clean after a recalc and would otherwise skip them forever.
1029    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1030    /// the end of the same recalc that iterated them
1031    /// ([`Self::redirty_for_next_recalc`], called wherever
1032    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1033    /// if an edit breaks the cycle, the next recalc's SCC task either does
1034    /// not exist or settles as phantom, nothing re-registers, and the
1035    /// redirty chain stops by itself.
1036    pending_iterative_redirty: Vec<VertexId>,
1037
1038    /// Final committed values of iterating-SCC members as of the end of the
1039    /// most recent recalc (spec §4 persistence). In canonical (value-cache
1040    /// disabled) mode the computed overlay is the ONLY home of a formula's
1041    /// value, and structural edits clear computed overlays wholesale
1042    /// (`clear_computed_overlay_after_row/_col`) — destroying iteration
1043    /// state (accumulators reset to 0; found by the iterate edge corpus).
1044    /// This snapshot, refreshed by [`Self::redirty_for_next_recalc`], lets
1045    /// the next SCC task re-seed members whose overlay entry vanished.
1046    /// Empty unless something iterated — zero cost otherwise.
1047    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1048
1049    /// Global function-registry semantic epoch observed after the latest
1050    /// conservative FormulaPlane invalidation.
1051    function_semantic_epoch_seen: u64,
1052    /// Runtime-provider semantic revision observed after the latest conservative
1053    /// FormulaPlane invalidation.
1054    function_provider_revision_seen: Option<u64>,
1055
1056    #[cfg(test)]
1057    last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1058    #[cfg(test)]
1059    evaluation_request_begin_count_for_test: u64,
1060    #[cfg(any(test, feature = "test-support"))]
1061    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1062    #[cfg(test)]
1063    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1064    #[cfg(test)]
1065    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1066    #[cfg(test)]
1067    inject_target_semantic_stale_once_for_test: bool,
1068    #[cfg(test)]
1069    force_virtual_dep_changes_remaining_for_test: usize,
1070    #[cfg(test)]
1071    fail_evaluation_commit_preflight_once_for_test: bool,
1072    #[cfg(test)]
1073    force_source_family_fallback: bool,
1074    #[cfg(test)]
1075    rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1076    #[cfg(test)]
1077    fragmented_commit_fault_for_test:
1078        Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1079    #[cfg(test)]
1080    formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1081    #[cfg(test)]
1082    target_preparation_fault_for_test:
1083        Option<crate::engine::target_preparation::TargetPreparationFault>,
1084    #[cfg(test)]
1085    force_non_cycle_schedule_fallback_for_test: bool,
1086    #[cfg(test)]
1087    mixed_topology_index_builds_for_test: u64,
1088    #[cfg(test)]
1089    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1090    #[cfg(test)]
1091    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1092}
1093
1094/// Minimal edit surface used by `Engine::action`.
1095///
1096fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1097    fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1098        let node_type = match &ast.node_type {
1099            ASTNodeType::Literal(_) if !in_array => {
1100                let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1101                *next = next.saturating_add(1);
1102                ASTNodeType::Literal(value)
1103            }
1104            ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1105            ASTNodeType::Omitted => ASTNodeType::Omitted,
1106            ASTNodeType::Reference {
1107                original,
1108                reference,
1109            } => ASTNodeType::Reference {
1110                original: original.clone(),
1111                reference: reference.clone(),
1112            },
1113            ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1114                op: op.clone(),
1115                expr: Box::new(visit(expr, binding, next, in_array)),
1116            },
1117            ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1118                op: op.clone(),
1119                left: Box::new(visit(left, binding, next, in_array)),
1120                right: Box::new(visit(right, binding, next, in_array)),
1121            },
1122            ASTNodeType::Function { name, args } => ASTNodeType::Function {
1123                name: name.clone(),
1124                args: args
1125                    .iter()
1126                    .map(|arg| visit(arg, binding, next, in_array))
1127                    .collect(),
1128            },
1129            ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1130                callee: Box::new(visit(callee, binding, next, in_array)),
1131                args: args
1132                    .iter()
1133                    .map(|arg| visit(arg, binding, next, in_array))
1134                    .collect(),
1135            },
1136            ASTNodeType::Array(rows) => ASTNodeType::Array(
1137                rows.iter()
1138                    .map(|row| {
1139                        row.iter()
1140                            .map(|cell| visit(cell, binding, next, true))
1141                            .collect()
1142                    })
1143                    .collect(),
1144            ),
1145        };
1146        ASTNode::new(node_type, ast.source_token.clone())
1147    }
1148
1149    let mut next = 0;
1150    visit(ast, binding, &mut next, false)
1151}
1152
1153/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1154/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1155impl<R: EvaluationContext> Engine<R> {
1156    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1157        self.graph.ingest_pipeline(&self.resolver)
1158    }
1159}
1160
1161pub struct EngineAction<'a, R>
1162where
1163    R: EvaluationContext,
1164{
1165    engine: &'a mut Engine<R>,
1166    name: String,
1167    // Optional external ChangeLog pointer used by `Engine::action_with_logger`.
1168    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1169    log: Option<*mut crate::engine::ChangeLog>,
1170    // Optional Arrow undo journal used by `Engine::action_atomic`.
1171    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1172    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1173    // True when this EngineAction must enforce conservative atomic transaction policy.
1174    atomic_policy: bool,
1175}
1176
1177impl<'a, R> EngineAction<'a, R>
1178where
1179    R: EvaluationContext,
1180{
1181    #[inline]
1182    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1183        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1184        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1185        crate::reference::CellRef::new(sheet_id, coord)
1186    }
1187
1188    #[inline]
1189    pub fn name(&self) -> &str {
1190        &self.name
1191    }
1192
1193    #[inline]
1194    pub fn set_cell_value(
1195        &mut self,
1196        sheet: &str,
1197        row: u32,
1198        col: u32,
1199        value: LiteralValue,
1200    ) -> Result<(), crate::engine::EditorError> {
1201        if self.log.is_some() {
1202            let old_value = self.engine.read_cell_value(sheet, row, col);
1203            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1204            let addr = self.addr_for(sheet, row, col);
1205            let Some(log_ptr) = self.log else {
1206                return Err(crate::engine::EditorError::TransactionFailed {
1207                    reason: "action_with_logger: missing ChangeLog".to_string(),
1208                });
1209            };
1210
1211            // For atomic journal mode, record computed overlay effects for this cell.
1212            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1213            let old_comp = if self.arrow_undo.is_some() {
1214                self.engine.read_computed_overlay_cell(sheet, row, col)
1215            } else {
1216                None
1217            };
1218
1219            if self.engine.graph_admission_enabled() {
1220                let admission =
1221                    self.engine
1222                        .graph
1223                        .preview_value_mutation(addr.sheet_id, row, col)?;
1224                self.engine.preflight_graph_admission(admission)?;
1225            }
1226            self.engine.demote_span_containing_cell_for_write(
1227                addr.sheet_id,
1228                addr.coord.row(),
1229                addr.coord.col(),
1230            )?;
1231            if old_formula.is_none() {
1232                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1233            }
1234
1235            let delta_old_sem = if old_formula.is_some() {
1236                None
1237            } else {
1238                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1239            };
1240
1241            let start_len = unsafe { (&*log_ptr).len() };
1242
1243            // Safety: `log_ptr` comes from a unique `&mut ChangeLog` in `Engine::action_with_logger`.
1244            let log = unsafe { &mut *log_ptr };
1245            self.engine.edit_with_logger(log, |editor| {
1246                editor.set_cell_value_with_old_state(
1247                    addr,
1248                    value.clone(),
1249                    old_value.clone(),
1250                    old_formula.clone(),
1251                );
1252            })?;
1253            self.engine
1254                .record_formula_plane_structural_change(StructuralScope::Cell {
1255                    sheet: addr.sheet_id,
1256                    row: addr.coord.row(),
1257                    col: addr.coord.col(),
1258                });
1259
1260            if let Some(undo_ptr) = self.arrow_undo {
1261                // 1) Spill snapshot operations (computed overlay rect restore).
1262                let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1263                let undo = unsafe { &mut *undo_ptr };
1264                self.engine
1265                    .record_spill_ops_into_arrow_undo(undo, new_events);
1266
1267                // 2) Delta/computed overlay single-cell deltas.
1268                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1269                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1270                let row0 = row.saturating_sub(1);
1271                let col0 = col.saturating_sub(1);
1272                let delta_new_sem = Some(value.clone());
1273                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1274                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1275            }
1276            Ok(())
1277        } else {
1278            self.engine
1279                .set_cell_value(sheet, row, col, value)
1280                .map_err(crate::engine::EditorError::from)
1281        }
1282    }
1283
1284    #[inline]
1285    pub fn set_cell_formula(
1286        &mut self,
1287        sheet: &str,
1288        row: u32,
1289        col: u32,
1290        ast: ASTNode,
1291    ) -> Result<(), crate::engine::EditorError> {
1292        if self.log.is_some() {
1293            let old_value = self.engine.read_cell_value(sheet, row, col);
1294            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1295            let addr = self.addr_for(sheet, row, col);
1296            let Some(log_ptr) = self.log else {
1297                return Err(crate::engine::EditorError::TransactionFailed {
1298                    reason: "action_with_logger: missing ChangeLog".to_string(),
1299                });
1300            };
1301
1302            let admitted_formula = if self.engine.graph_admission_enabled() {
1303                let placement =
1304                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1305                let ingested = self.engine.ingest_pipeline().ingest_formula(
1306                    FormulaAstInput::Tree(ast.clone()),
1307                    placement,
1308                    None,
1309                )?;
1310                let admission = self.engine.graph.preview_formula_mutations(&[(
1311                    addr.sheet_id,
1312                    row,
1313                    col,
1314                    ingested.dep_plan.clone(),
1315                )])?;
1316                self.engine.preflight_graph_admission(admission)?;
1317                Some((ingested.ast_id, ingested.dep_plan))
1318            } else {
1319                None
1320            };
1321            self.engine.demote_span_containing_cell_for_write(
1322                addr.sheet_id,
1323                addr.coord.row(),
1324                addr.coord.col(),
1325            )?;
1326            if old_formula.is_none() {
1327                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328            }
1329            let delta_old = if self.arrow_undo.is_some() {
1330                if old_formula.is_some() {
1331                    None
1332                } else {
1333                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334                }
1335            } else {
1336                None
1337            };
1338            let start_len = unsafe { (&*log_ptr).len() };
1339
1340            // Safety: `log_ptr` comes from a unique `&mut ChangeLog` in `Engine::action_with_logger`.
1341            let log = unsafe { &mut *log_ptr };
1342            self.engine.edit_with_logger(log, |editor| {
1343                if let Some((ast_id, plan)) = admitted_formula {
1344                    editor.set_cell_formula_with_prepared_plan(
1345                        addr,
1346                        ast.clone(),
1347                        old_value,
1348                        old_formula,
1349                        ast_id,
1350                        plan,
1351                    );
1352                } else {
1353                    editor.set_cell_formula_with_old_state(
1354                        addr,
1355                        ast.clone(),
1356                        old_value,
1357                        old_formula,
1358                    );
1359                }
1360            })?;
1361            self.engine
1362                .record_formula_plane_structural_change(StructuralScope::Cell {
1363                    sheet: addr.sheet_id,
1364                    row: addr.coord.row(),
1365                    col: addr.coord.col(),
1366                });
1367
1368            if let Some(undo_ptr) = self.arrow_undo {
1369                let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1370                let undo = unsafe { &mut *undo_ptr };
1371                self.engine
1372                    .record_spill_ops_into_arrow_undo(undo, new_events);
1373                let delta_new: Option<LiteralValue> = None;
1374                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1375                let row0 = row.saturating_sub(1);
1376                let col0 = col.saturating_sub(1);
1377                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1378            }
1379            Ok(())
1380        } else {
1381            self.engine
1382                .set_cell_formula(sheet, row, col, ast)
1383                .map_err(crate::engine::EditorError::from)
1384        }
1385    }
1386
1387    #[inline]
1388    pub fn set_row_hidden(
1389        &mut self,
1390        sheet: &str,
1391        row_1based: u32,
1392        hidden: bool,
1393        source: RowVisibilitySource,
1394    ) -> Result<(), crate::engine::EditorError> {
1395        if self.log.is_some() {
1396            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1397            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1398            let old_hidden = self
1399                .engine
1400                .row_visibility
1401                .get(&sheet_id)
1402                .map(|state| state.is_row_hidden(row0, Some(source)))
1403                .unwrap_or(false);
1404            if old_hidden == hidden {
1405                return Ok(());
1406            }
1407
1408            let _ = self
1409                .engine
1410                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1411
1412            let Some(log_ptr) = self.log else {
1413                return Err(crate::engine::EditorError::TransactionFailed {
1414                    reason: "action_with_logger: missing ChangeLog".to_string(),
1415                });
1416            };
1417            unsafe { &mut *log_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1418                sheet_id,
1419                row0,
1420                source,
1421                old_hidden,
1422                new_hidden: hidden,
1423            });
1424
1425            Ok(())
1426        } else {
1427            self.engine
1428                .set_row_hidden(sheet, row_1based, hidden, source)
1429        }
1430    }
1431
1432    #[inline]
1433    pub fn set_rows_hidden(
1434        &mut self,
1435        sheet: &str,
1436        start_row_1based: u32,
1437        end_row_1based: u32,
1438        hidden: bool,
1439        source: RowVisibilitySource,
1440    ) -> Result<(), crate::engine::EditorError> {
1441        if self.log.is_some() {
1442            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1443            let (start_row0, end_row0) =
1444                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1445
1446            let Some(log_ptr) = self.log else {
1447                return Err(crate::engine::EditorError::TransactionFailed {
1448                    reason: "action_with_logger: missing ChangeLog".to_string(),
1449                });
1450            };
1451            let log = unsafe { &mut *log_ptr };
1452
1453            for row0 in start_row0..=end_row0 {
1454                let old_hidden = self
1455                    .engine
1456                    .row_visibility
1457                    .get(&sheet_id)
1458                    .map(|state| state.is_row_hidden(row0, Some(source)))
1459                    .unwrap_or(false);
1460                if old_hidden == hidden {
1461                    continue;
1462                }
1463
1464                let _ = self
1465                    .engine
1466                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1467
1468                log.record(crate::engine::ChangeEvent::SetRowVisibility {
1469                    sheet_id,
1470                    row0,
1471                    source,
1472                    old_hidden,
1473                    new_hidden: hidden,
1474                });
1475            }
1476
1477            Ok(())
1478        } else {
1479            self.engine
1480                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1481        }
1482    }
1483
1484    #[inline]
1485    pub fn insert_rows(
1486        &mut self,
1487        sheet: &str,
1488        before: u32,
1489        count: u32,
1490    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1491        if count == 0 {
1492            return Ok(crate::engine::ShiftSummary::default());
1493        }
1494        if self.log.is_some() {
1495            let Some(log_ptr) = self.log else {
1496                return Err(crate::engine::EditorError::TransactionFailed {
1497                    reason: "action_atomic: missing ChangeLog".to_string(),
1498                });
1499            };
1500
1501            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1502            let before0 = before.saturating_sub(1);
1503            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1504            // Authority geometry is not journaled. Materialize affected spans
1505            // before the logged graph shift so undo/redo remains exact instead
1506            // of leaving an unlogged shifted span behind.
1507            self.engine
1508                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1509
1510            // Graph structural insert (logged) - no snapshot bump.
1511            let summary = {
1512                let log = unsafe { &mut *log_ptr };
1513                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1514                    Ok(crate::engine::ShiftSummary::default());
1515                self.engine.edit_with_logger(log, |editor| {
1516                    out = editor.insert_rows(sheet_id, before0, count);
1517                })?;
1518                out?
1519            };
1520
1521            // Arrow insert (truth) + undo op.
1522            self.engine.ensure_arrow_sheet(sheet);
1523            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1524                asheet.insert_rows(before0 as usize, count as usize);
1525            }
1526            self.engine
1527                .shift_row_visibility_insert(sheet_id, before0, count);
1528            self.engine.mark_moved_formula_vertices_dirty(&summary);
1529            self.engine
1530                .clear_computed_overlay_after_row(sheet, before0 as usize);
1531            self.engine
1532                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1533            self.engine.mark_topology_edited();
1534            if let Some(undo_ptr) = self.arrow_undo {
1535                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1536            }
1537            Ok(summary)
1538        } else {
1539            self.engine.insert_rows(sheet, before, count)
1540        }
1541    }
1542
1543    #[inline]
1544    pub fn delete_rows(
1545        &mut self,
1546        sheet: &str,
1547        start: u32,
1548        count: u32,
1549    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1550        if count == 0 {
1551            return Ok(crate::engine::ShiftSummary::default());
1552        }
1553        if self.atomic_policy {
1554            return Err(crate::engine::EditorError::TransactionUnsupported {
1555                reason:
1556                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1557                        .to_string(),
1558            });
1559        }
1560        self.engine.delete_rows(sheet, start, count)
1561    }
1562
1563    #[inline]
1564    pub fn insert_columns(
1565        &mut self,
1566        sheet: &str,
1567        before: u32,
1568        count: u32,
1569    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1570        if count == 0 {
1571            return Ok(crate::engine::ShiftSummary::default());
1572        }
1573        if self.log.is_some() {
1574            let Some(log_ptr) = self.log else {
1575                return Err(crate::engine::EditorError::TransactionFailed {
1576                    reason: "action_atomic: missing ChangeLog".to_string(),
1577                });
1578            };
1579
1580            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1581            let before0 = before.saturating_sub(1);
1582            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1583            self.engine
1584                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1585
1586            let summary = {
1587                let log = unsafe { &mut *log_ptr };
1588                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1589                    Ok(crate::engine::ShiftSummary::default());
1590                self.engine.edit_with_logger(log, |editor| {
1591                    out = editor.insert_columns(sheet_id, before0, count);
1592                })?;
1593                out?
1594            };
1595
1596            self.engine.ensure_arrow_sheet(sheet);
1597            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1598                asheet.insert_columns(before0 as usize, count as usize);
1599            }
1600            self.engine.mark_moved_formula_vertices_dirty(&summary);
1601            self.engine
1602                .clear_computed_overlay_after_col(sheet, before0 as usize);
1603            self.engine
1604                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1605            self.engine.mark_topology_edited();
1606            if let Some(undo_ptr) = self.arrow_undo {
1607                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1608            }
1609            Ok(summary)
1610        } else {
1611            self.engine.insert_columns(sheet, before, count)
1612        }
1613    }
1614
1615    #[inline]
1616    pub fn delete_columns(
1617        &mut self,
1618        sheet: &str,
1619        start: u32,
1620        count: u32,
1621    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1622        if count == 0 {
1623            return Ok(crate::engine::ShiftSummary::default());
1624        }
1625        if self.atomic_policy {
1626            return Err(crate::engine::EditorError::TransactionUnsupported {
1627                reason:
1628                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1629                        .to_string(),
1630            });
1631        }
1632        self.engine.delete_columns(sheet, start, count)
1633    }
1634
1635    /// Start an action from within an action.
1636    ///
1637    /// Nested actions are currently disallowed (ticket 614), so this will return a
1638    /// `EditorError::TransactionFailed` while an outer action is active.
1639    #[inline]
1640    pub fn action<T>(
1641        &mut self,
1642        name: impl AsRef<str>,
1643        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1644    ) -> Result<T, crate::engine::EditorError> {
1645        self.engine.action(name, f)
1646    }
1647}
1648
1649struct ActionDepthGuard<'a, R> {
1650    engine: *mut Engine<R>,
1651    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1652}
1653
1654impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1655    fn drop(&mut self) {
1656        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1657        // than the surrounding `Engine::action` call.
1658        unsafe {
1659            let e = &mut *self.engine;
1660            e.action_depth = e.action_depth.saturating_sub(1);
1661        }
1662    }
1663}
1664
1665#[derive(Default)]
1666struct SourceCache {
1667    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1668    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1669}
1670
1671#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1672struct VisibilityMaskCacheKey {
1673    sheet_id: SheetId,
1674    start_row0: u32,
1675    end_row0: u32,
1676    mode: VisibilityMaskMode,
1677    version: u64,
1678}
1679
1680#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1681enum StructuralScope {
1682    Cell { sheet: SheetId, row: u32, col: u32 },
1683    Region(Region),
1684    Sheet(SheetId),
1685    RemovedSheet(SheetId),
1686    OpaqueGlobal,
1687    AllSheets,
1688}
1689
1690struct SourceCacheSession {
1691    cache: Arc<std::sync::RwLock<SourceCache>>,
1692}
1693
1694impl Drop for SourceCacheSession {
1695    fn drop(&mut self) {
1696        if let Ok(mut g) = self.cache.write() {
1697            *g = SourceCache::default();
1698        }
1699    }
1700}
1701
1702#[derive(Debug)]
1703#[non_exhaustive]
1704pub struct EvalResult {
1705    pub computed_vertices: usize,
1706    pub cycle_errors: usize,
1707    pub elapsed: std::time::Duration,
1708}
1709
1710#[derive(Clone, Debug, PartialEq, Eq)]
1711#[non_exhaustive]
1712pub struct TableMetadata {
1713    pub name: String,
1714    pub sheet: String,
1715    pub start_row: u32,
1716    pub start_col: u32,
1717    pub end_row: u32,
1718    pub end_col: u32,
1719    pub header_row: bool,
1720    pub headers: Vec<String>,
1721    pub totals_row: bool,
1722}
1723
1724/// Read-only engine counters used by benchmark/instrumentation tooling.
1725///
1726/// These counters are deliberately observational: collecting them must not mutate engine state or
1727/// alter formula evaluation semantics.
1728#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1729#[non_exhaustive]
1730pub struct EngineBaselineStats {
1731    pub graph_vertex_count: usize,
1732    pub graph_formula_vertex_count: usize,
1733    pub graph_edge_count: usize,
1734    pub dirty_vertex_count: usize,
1735    pub evaluation_vertex_count: usize,
1736    pub formula_ast_root_count: usize,
1737    pub formula_ast_node_count: usize,
1738    pub staged_formula_count: usize,
1739    pub formula_plane_active_span_count: usize,
1740    pub formula_plane_producer_result_entries: usize,
1741    pub formula_plane_consumer_read_entries: usize,
1742    pub formula_plane_mixed_topology_cache_builds: u64,
1743    pub formula_plane_mixed_topology_cache_hits: u64,
1744    pub formula_plane_mixed_topology_cache_overflows: u64,
1745    pub formula_plane_dirty_pending_events: usize,
1746    pub formula_plane_dirty_region_events_recorded: u64,
1747    pub formula_plane_dirty_span_region_events_recorded: u64,
1748    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1749    pub formula_plane_dirty_global_invalidations: u64,
1750    pub formula_plane_structural_span_candidates: u64,
1751    /// Number of spans demoted to legacy because a member participated in a
1752    /// statically-cyclic SCC (gotcha G8, refs #112).
1753    pub formula_plane_cycle_member_span_demotions: u64,
1754}
1755
1756#[derive(Debug, Clone, Default)]
1757#[non_exhaustive]
1758pub struct VirtualDepTelemetry {
1759    pub candidate_vertices_total: usize,
1760    pub vdeps_vertices_total: usize,
1761    pub vdeps_edges_total: usize,
1762    pub builder_elapsed_ms_total: u128,
1763    pub schedule_virtual_passes: usize,
1764    pub schedule_static_passes: usize,
1765    pub schedule_cache_hits: usize,
1766    pub schedule_cache_misses: usize,
1767    pub reused_schedule_vertices_total: usize,
1768    pub replan_iterations: usize,
1769    pub changed_vdeps_total: usize,
1770    pub bailout_reason: Option<&'static str>,
1771    pub fallback_mode_activations: u64,
1772}
1773
1774/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
1775/// (spec `formualizer-cycle-semantics-spec.md` §10).
1776///
1777/// Collection is unconditional: SCC tasks are rare relative to ordinary
1778/// vertex evaluation and the counters are a handful of integer adds per
1779/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
1780/// which pays per-schedule costs). Counters reset at the start of every
1781/// evaluation request.
1782#[derive(Debug, Clone, Default, PartialEq)]
1783#[non_exhaustive]
1784pub struct CycleTelemetry {
1785    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
1786    pub static_sccs: usize,
1787    /// SCC tasks whose live subgraph was acyclic — values produced.
1788    pub phantom_sccs: usize,
1789    /// Distinct live cycles witnessed across all SCC tasks.
1790    pub live_cycles_witnessed: usize,
1791    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
1792    pub circ_cells_stamped: usize,
1793    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
1794    /// (pass 1 included).
1795    pub settle_passes_total: usize,
1796    /// Largest pass count any single SCC task needed.
1797    pub max_passes_single_scc: usize,
1798    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
1799    /// with a witnessed live cycle). RFC #113, Stage 3.
1800    pub iterated_sccs: usize,
1801    /// Iterating SCC tasks that stopped because every member passed the
1802    /// spec-§6 convergence test.
1803    pub converged_sccs: usize,
1804    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
1805    /// this is the Excel `max_iterations` cap (NOT an error — last values
1806    /// are kept; includes the no-convergence-test `max_iterations: 1`
1807    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
1808    /// settle cap (|SCC| + 2), which only a bug can hit.
1809    pub capped_sccs: usize,
1810    /// Largest `|Δ|` observed in any member's final-pass convergence
1811    /// comparison across iterating SCC tasks (numeric-class members only).
1812    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
1813    pub max_abs_delta_at_stop: f64,
1814    /// Identical-bit NaN vs NaN member comparisons that were treated as
1815    /// converged (spec §6 NaN rule).
1816    pub nan_converged: usize,
1817    /// Total wall-clock time spent inside Runtime SCC tasks.
1818    pub elapsed_ms: u128,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct ScheduleBuildMeta {
1823    candidate_vertices: usize,
1824    vdeps_vertices: usize,
1825    vdeps_edges: usize,
1826    builder_elapsed_ms: u128,
1827    used_virtual_schedule: bool,
1828    schedule_cache_hit: bool,
1829    schedule_cache_eligible: bool,
1830}
1831
1832#[derive(Debug, Clone)]
1833struct CachedScheduleEntry {
1834    topology_epoch: u64,
1835    candidate_vertices: Vec<VertexId>,
1836    schedule: crate::engine::scheduler::Schedule,
1837}
1838
1839#[derive(Clone, Debug, PartialEq, Eq)]
1840struct MixedTopologyCacheKey {
1841    engine_topology_epoch: u64,
1842    graph_topology_revision: u64,
1843    authority_indexes_epoch: u64,
1844    /// Explicit revisions for retained island membership and boundary views.
1845    /// They currently share the authoritative graph/index clocks but remain
1846    /// distinct key fields so future independent indexes cannot omit them.
1847    legacy_island_revision: u64,
1848    boundary_index_revision: u64,
1849    function_semantic_epoch: u64,
1850    function_provider_revision: Option<u64>,
1851    max_candidates: usize,
1852    max_edges: usize,
1853    max_memory_bytes: usize,
1854}
1855
1856#[derive(Debug)]
1857struct CachedMixedTopology {
1858    key: MixedTopologyCacheKey,
1859    topology: MixedTopology,
1860    producer_results: FormulaProducerResultIndex,
1861    consumer_reads: FormulaConsumerReadIndex,
1862    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1863    plane_epoch: u64,
1864    island: LegacyIslandPlan,
1865}
1866
1867#[derive(Debug)]
1868struct SkippedMixedTopology {
1869    reason: MixedScheduleFallbackReason,
1870    observed: MixedTopologyCompileStats,
1871    producer_results: FormulaProducerResultIndex,
1872    consumer_reads: FormulaConsumerReadIndex,
1873    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1874    plane_epoch: u64,
1875    island: LegacyIslandPlan,
1876}
1877
1878#[derive(Debug)]
1879enum FormulaPlaneTopologyCompileResult {
1880    Cached(CachedMixedTopology),
1881    CacheSkipped(SkippedMixedTopology),
1882}
1883
1884#[derive(Clone, Copy, Debug)]
1885struct ExactDemandScratchEstimates {
1886    paged: u64,
1887    runs: u64,
1888    native: u64,
1889    repeated: u64,
1890}
1891
1892fn exact_demand_scratch_estimates(
1893    read_count: usize,
1894    closure_base: u64,
1895) -> ExactDemandScratchEstimates {
1896    let read_count = read_count as u64;
1897    let page_count = read_count.div_ceil(128);
1898    let run_count = read_count.div_ceil(256);
1899    let pointer_bytes = std::mem::size_of::<usize>() as u64;
1900    let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
1901        + std::mem::size_of::<Vec<usize>>()
1902        + 12 * std::mem::size_of::<usize>()) as u64;
1903    let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
1904        page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
1905    );
1906    let runs_memory = read_count
1907        .saturating_mul(pointer_bytes)
1908        .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
1909    ExactDemandScratchEstimates {
1910        paged: closure_base.saturating_add(paged_memory),
1911        runs: closure_base.saturating_add(runs_memory),
1912        native: closure_base
1913            .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
1914        repeated: closure_base.saturating_add(512),
1915    }
1916}
1917
1918fn native_exact_demand_allowed(
1919    policy: Option<crate::engine::DiskScratchPolicy>,
1920    target_supports_native: bool,
1921) -> bool {
1922    target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
1923}
1924
1925fn select_exact_demand_strategy(
1926    estimates: ExactDemandScratchEstimates,
1927    native_allowed: bool,
1928    mut can_reserve: impl FnMut(u64) -> bool,
1929) -> (FormulaPlaneTopologyStrategy, u64) {
1930    if can_reserve(estimates.paged) {
1931        (
1932            FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1933            estimates.paged,
1934        )
1935    } else if can_reserve(estimates.runs) {
1936        (
1937            FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1938            estimates.runs,
1939        )
1940    } else if native_allowed && can_reserve(estimates.native) {
1941        (
1942            FormulaPlaneTopologyStrategy::ExactNativeScratch,
1943            estimates.native,
1944        )
1945    } else {
1946        (
1947            FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1948            estimates.repeated,
1949        )
1950    }
1951}
1952
1953#[cfg(test)]
1954mod exact_demand_strategy_tests {
1955    use super::*;
1956
1957    #[test]
1958    fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
1959        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
1960        assert!(estimates.paged > estimates.runs);
1961        assert!(estimates.runs > estimates.native);
1962        assert!(estimates.native > estimates.repeated);
1963        for (limit, native_allowed, expected) in [
1964            (
1965                estimates.paged,
1966                true,
1967                FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1968            ),
1969            (
1970                estimates.runs,
1971                true,
1972                FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1973            ),
1974            (
1975                estimates.native,
1976                true,
1977                FormulaPlaneTopologyStrategy::ExactNativeScratch,
1978            ),
1979            (
1980                estimates.repeated,
1981                true,
1982                FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1983            ),
1984        ] {
1985            assert_eq!(
1986                select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
1987                expected
1988            );
1989        }
1990    }
1991
1992    #[cfg(not(target_arch = "wasm32"))]
1993    #[test]
1994    fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
1995        let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
1996        let path = scratch.path.clone();
1997        let auxiliary_path = scratch.auxiliary_path.clone();
1998        assert!(path.exists());
1999        assert!(auxiliary_path.exists());
2000        drop(scratch);
2001        assert!(!path.exists());
2002        assert!(!auxiliary_path.exists());
2003    }
2004
2005    #[test]
2006    fn native_policy_and_wasm_target_support_gate_native_rung() {
2007        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2008        assert!(!native_exact_demand_allowed(
2009            Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2010            true,
2011        ));
2012        assert!(!native_exact_demand_allowed(
2013            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2014            false,
2015        ));
2016        let native_allowed = native_exact_demand_allowed(
2017            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2018            !cfg!(target_arch = "wasm32"),
2019        );
2020        let expected = if cfg!(target_arch = "wasm32") {
2021            FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2022        } else {
2023            FormulaPlaneTopologyStrategy::ExactNativeScratch
2024        };
2025        assert_eq!(
2026            select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2027                bytes <= estimates.native
2028            })
2029            .0,
2030            expected
2031        );
2032    }
2033}
2034
2035#[cfg(not(target_arch = "wasm32"))]
2036struct NativeTopologyScratch {
2037    path: std::path::PathBuf,
2038    file: Option<std::fs::File>,
2039    auxiliary_path: std::path::PathBuf,
2040    auxiliary_file: Option<std::fs::File>,
2041}
2042
2043#[cfg(not(target_arch = "wasm32"))]
2044impl NativeTopologyScratch {
2045    fn create(request_id: u64) -> std::io::Result<Self> {
2046        use std::fs::OpenOptions;
2047        for attempt in 0..32_u64 {
2048            let path = std::env::temp_dir().join(format!(
2049                "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2050                std::process::id()
2051            ));
2052            match OpenOptions::new()
2053                .read(true)
2054                .write(true)
2055                .create_new(true)
2056                .open(&path)
2057            {
2058                Ok(file) => {
2059                    let auxiliary_path = path.with_extension("aux.tmp");
2060                    match OpenOptions::new()
2061                        .read(true)
2062                        .write(true)
2063                        .create_new(true)
2064                        .open(&auxiliary_path)
2065                    {
2066                        Ok(auxiliary_file) => {
2067                            return Ok(Self {
2068                                path,
2069                                file: Some(file),
2070                                auxiliary_path,
2071                                auxiliary_file: Some(auxiliary_file),
2072                            });
2073                        }
2074                        Err(error) => {
2075                            drop(file);
2076                            let _ = std::fs::remove_file(&path);
2077                            if error.kind() == std::io::ErrorKind::AlreadyExists {
2078                                continue;
2079                            }
2080                            return Err(error);
2081                        }
2082                    }
2083                }
2084                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2085                Err(error) => return Err(error),
2086            }
2087        }
2088        Err(std::io::Error::new(
2089            std::io::ErrorKind::AlreadyExists,
2090            "could not allocate unique topology scratch file",
2091        ))
2092    }
2093}
2094
2095#[cfg(not(target_arch = "wasm32"))]
2096impl Drop for NativeTopologyScratch {
2097    fn drop(&mut self) {
2098        drop(self.file.take());
2099        drop(self.auxiliary_file.take());
2100        let _ = std::fs::remove_file(&self.path);
2101        let _ = std::fs::remove_file(&self.auxiliary_path);
2102    }
2103}
2104
2105fn estimated_span_bindings_bytes(
2106    bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2107) -> Option<usize> {
2108    const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2109    bindings.len().checked_mul(
2110        std::mem::size_of::<FormulaSpanId>()
2111            .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2112            .checked_add(TREE_ENTRY_OVERHEAD)?,
2113    )
2114}
2115
2116type ScheduleBuildOutput = (
2117    crate::engine::scheduler::Schedule,
2118    FxHashMap<VertexId, Vec<VertexId>>,
2119    ScheduleBuildMeta,
2120);
2121
2122/// Opaque, revision-bound recalculation recipe.
2123#[derive(Debug)]
2124pub struct RecalcPlan {
2125    key: RecalcPlanKey,
2126    kind: RecalcPlanKind,
2127}
2128
2129#[derive(Debug)]
2130struct RecalcPlanKey {
2131    engine_token: Arc<()>,
2132    revisions: PlanningRevisionSnapshot,
2133}
2134
2135#[derive(Clone, Debug, PartialEq, Eq)]
2136struct PlanningRevisionSnapshot {
2137    engine_topology_epoch: u64,
2138    graph_topology_revision: u64,
2139    authority: u64,
2140    authority_indexes: u64,
2141    authority_indexed_plane: u64,
2142    staged: u64,
2143    symbols: u64,
2144    semantic: u64,
2145    provider: Option<u64>,
2146    formula_plane_mode: FormulaPlaneMode,
2147    deterministic_mode: crate::engine::DeterministicMode,
2148    budgets: crate::engine::EvaluationBudgets,
2149    span_refs: Vec<FormulaSpanRef>,
2150}
2151
2152#[derive(Debug)]
2153enum RecalcPlanKind {
2154    CompatibilityFull {
2155        schedule: crate::engine::Schedule,
2156        has_dynamic_refs: bool,
2157    },
2158    Target {
2159        targets: Vec<crate::engine::EvaluationTarget>,
2160        scope: crate::engine::PrepareScope,
2161        topology: RecalcTopology,
2162        dynamic_policy: DynamicPlanPolicy,
2163    },
2164}
2165
2166#[derive(Debug)]
2167enum RecalcTopology {
2168    RunLocalRecipe,
2169    Workbook,
2170}
2171
2172#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2173enum DynamicPlanPolicy {
2174    BoundedTargetReplan,
2175}
2176
2177impl RecalcPlan {
2178    /// Returns the retained compatibility schedule depth. Target plans retain a
2179    /// run-local recipe rather than a schedule, so their layer count is zero.
2180    pub fn layer_count(&self) -> usize {
2181        match &self.kind {
2182            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2183            RecalcPlanKind::Target { .. } => 0,
2184        }
2185    }
2186
2187    pub fn has_dynamic_refs(&self) -> bool {
2188        match &self.kind {
2189            RecalcPlanKind::CompatibilityFull {
2190                has_dynamic_refs, ..
2191            } => *has_dynamic_refs,
2192            RecalcPlanKind::Target { .. } => false,
2193        }
2194    }
2195
2196    #[cfg(test)]
2197    pub(crate) fn force_stale_reasons_for_test(
2198        &mut self,
2199        reasons: &[formualizer_common::PlanStaleReason],
2200    ) {
2201        use formualizer_common::PlanStaleReason;
2202        for reason in reasons {
2203            match reason {
2204                PlanStaleReason::Engine => {
2205                    self.key.engine_token = Arc::new(());
2206                }
2207                PlanStaleReason::Provider => {
2208                    self.key.revisions.provider = Some(
2209                        self.key
2210                            .revisions
2211                            .provider
2212                            .unwrap_or_default()
2213                            .wrapping_add(1),
2214                    );
2215                }
2216                PlanStaleReason::Semantic => {
2217                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2218                }
2219                PlanStaleReason::Budget => {
2220                    let current = self.key.revisions.budgets.work.max_work_units;
2221                    self.key.revisions.budgets.work.max_work_units =
2222                        Some(current.unwrap_or_default().wrapping_add(1));
2223                }
2224                PlanStaleReason::Staged => {
2225                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2226                }
2227                PlanStaleReason::Symbols => {
2228                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2229                }
2230                PlanStaleReason::Authority => {
2231                    self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2232                }
2233                PlanStaleReason::SpanGeneration => {
2234                    self.key.revisions.span_refs.push(FormulaSpanRef {
2235                        id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2236                        generation: u32::MAX,
2237                        version: u32::MAX,
2238                    });
2239                }
2240                PlanStaleReason::Graph => {
2241                    self.key.revisions.graph_topology_revision =
2242                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2243                }
2244                _ => {}
2245            }
2246        }
2247    }
2248}
2249
2250#[cfg(test)]
2251pub(crate) mod criteria_mask_test_hooks {
2252    use std::cell::Cell;
2253
2254    thread_local! {
2255        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2256        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2257    }
2258
2259    pub fn reset_text_segment_counters() {
2260        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2261        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2262    }
2263
2264    pub fn text_segment_counters() -> (usize, usize) {
2265        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2266        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2267        (a, b)
2268    }
2269
2270    pub(crate) fn inc_total() {
2271        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2272    }
2273    pub(crate) fn inc_all_null() {
2274        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2275    }
2276}
2277
2278#[cfg(test)]
2279pub(crate) mod visibility_mask_test_hooks {
2280    use std::cell::Cell;
2281
2282    thread_local! {
2283        static HITS: Cell<usize> = const { Cell::new(0) };
2284        static MISSES: Cell<usize> = const { Cell::new(0) };
2285        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2286    }
2287
2288    pub fn reset() {
2289        HITS.with(|c| c.set(0));
2290        MISSES.with(|c| c.set(0));
2291        EVICTIONS.with(|c| c.set(0));
2292    }
2293
2294    pub fn counters() -> (usize, usize, usize) {
2295        let hits = HITS.with(|c| c.get());
2296        let misses = MISSES.with(|c| c.get());
2297        let evictions = EVICTIONS.with(|c| c.get());
2298        (hits, misses, evictions)
2299    }
2300
2301    pub(crate) fn inc_hit() {
2302        HITS.with(|c| c.set(c.get() + 1));
2303    }
2304
2305    pub(crate) fn inc_miss() {
2306        MISSES.with(|c| c.set(c.get() + 1));
2307    }
2308
2309    pub(crate) fn inc_eviction() {
2310        EVICTIONS.with(|c| c.set(c.get() + 1));
2311    }
2312}
2313
2314fn compute_criteria_mask(
2315    view: &RangeView<'_>,
2316    col_in_view: usize,
2317    pred: &crate::args::CriteriaPredicate,
2318) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2319    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2320    use arrow::compute::kernels::comparison::{ilike, nilike};
2321    use arrow_array::{
2322        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2323    };
2324
2325    // Helper: apply a numeric predicate to a single Float64Array chunk
2326    fn apply_numeric_pred(
2327        chunk: &Float64Array,
2328        pred: &crate::args::CriteriaPredicate,
2329    ) -> Option<BooleanArray> {
2330        match pred {
2331            crate::args::CriteriaPredicate::Gt(n) => {
2332                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2333            }
2334            crate::args::CriteriaPredicate::Ge(n) => {
2335                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2336            }
2337            crate::args::CriteriaPredicate::Lt(n) => {
2338                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2339            }
2340            crate::args::CriteriaPredicate::Le(n) => {
2341                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2342            }
2343            crate::args::CriteriaPredicate::Eq(v) => match v {
2344                formualizer_common::LiteralValue::Number(x) => {
2345                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2346                }
2347                formualizer_common::LiteralValue::Int(i) => {
2348                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2349                }
2350                _ => None,
2351            },
2352            crate::args::CriteriaPredicate::Ne(v) => match v {
2353                formualizer_common::LiteralValue::Number(x) => {
2354                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2355                }
2356                formualizer_common::LiteralValue::Int(i) => {
2357                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2358                }
2359                _ => None,
2360            },
2361            _ => None,
2362        }
2363    }
2364
2365    // Check if this is a numeric predicate that can be applied per-chunk
2366    let is_numeric_pred = matches!(
2367        pred,
2368        crate::args::CriteriaPredicate::Gt(_)
2369            | crate::args::CriteriaPredicate::Ge(_)
2370            | crate::args::CriteriaPredicate::Lt(_)
2371            | crate::args::CriteriaPredicate::Le(_)
2372            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2373            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2374            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2375            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2376    );
2377
2378    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2379    // This avoids materializing the full numeric column (64-bit per element) and instead
2380    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2381    if is_numeric_pred {
2382        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2383        for res in view.numbers_slices() {
2384            let (_rs, _rl, cols_seg) = res.ok()?;
2385            if col_in_view < cols_seg.len() {
2386                let chunk = cols_seg[col_in_view].as_ref();
2387                let mask = apply_numeric_pred(chunk, pred)?;
2388                bool_parts.push(mask);
2389            }
2390        }
2391
2392        if bool_parts.is_empty() {
2393            return None;
2394        } else if bool_parts.len() == 1 {
2395            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2396        } else {
2397            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2398            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2399                .iter()
2400                .map(|a| a as &dyn arrow_array::Array)
2401                .collect();
2402            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2403            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2404            return Some(std::sync::Arc::new(ba));
2405        }
2406    }
2407
2408    // TEXT PATH: build masks per row-chunk using lowered text slices.
2409    // This avoids concatenating full-string columns just to compute a boolean mask.
2410    let (text_kind, text_pat, empty_special) = match pred {
2411        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2412            (0u8, t.to_lowercase(), t.is_empty())
2413        }
2414        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2415            (1u8, t.to_lowercase(), false)
2416        }
2417        crate::args::CriteriaPredicate::TextLike {
2418            pattern,
2419            case_insensitive,
2420        } => {
2421            let p = if *case_insensitive {
2422                pattern.to_lowercase()
2423            } else {
2424                pattern.clone()
2425            };
2426            (2u8, p.replace('*', "%").replace('?', "_"), false)
2427        }
2428        _ => return None,
2429    };
2430
2431    let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2432    let pat = StringArray::new_scalar(text_pat);
2433    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2434
2435    for res in view.iter_row_chunks() {
2436        let cs = res.ok()?;
2437        if cs.row_len == 0 {
2438            continue;
2439        }
2440        #[cfg(test)]
2441        criteria_mask_test_hooks::inc_total();
2442
2443        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2444        if col_in_view >= slices.len() {
2445            return None;
2446        }
2447
2448        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2449        let seg = match seg_opt {
2450            Some(s) => s,
2451            None => {
2452                #[cfg(test)]
2453                criteria_mask_test_hooks::inc_all_null();
2454                if (text_kind == 0 && empty_special) || ne_matches_blank {
2455                    // Eq("") treats nulls (Empty) as equal.
2456                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2457                    bb.append_n(cs.row_len, true);
2458                    bool_parts.push(bb.finish());
2459                } else {
2460                    // For non-empty patterns, ilike/nilike return null on null inputs.
2461                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2462                }
2463                continue;
2464            }
2465        };
2466
2467        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2468        let mut m = match text_kind {
2469            0 => ilike(seg_sa, &pat).ok()?,
2470            1 => nilike(seg_sa, &pat).ok()?,
2471            2 => ilike(seg_sa, &pat).ok()?,
2472            _ => return None,
2473        };
2474
2475        // Only fold blank/Empty (null) cells into the mask when the segment
2476        // actually contains any. The null-fill loop + or_kleene are pure
2477        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2478        // over a column with no blanks stays fully vectorized on the ilike/
2479        // nilike result.
2480        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2481            // Treat nulls as equal to empty string
2482            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2483            for i in 0..seg_sa.len() {
2484                bb.append_value(seg_sa.is_null(i));
2485            }
2486            let nulls = bb.finish();
2487            m = boolean::or_kleene(&m, &nulls).ok()?;
2488        }
2489
2490        bool_parts.push(m);
2491    }
2492
2493    if bool_parts.is_empty() {
2494        None
2495    } else if bool_parts.len() == 1 {
2496        Some(std::sync::Arc::new(bool_parts.remove(0)))
2497    } else {
2498        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2499            .iter()
2500            .map(|a| a as &dyn arrow_array::Array)
2501            .collect();
2502        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2503        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2504        Some(std::sync::Arc::new(ba))
2505    }
2506}
2507
2508#[derive(Debug, Clone)]
2509pub struct LayerInfo {
2510    pub vertex_count: usize,
2511    pub parallel_eligible: bool,
2512    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2513}
2514
2515#[derive(Debug, Clone)]
2516pub struct EvalPlan {
2517    pub total_vertices_to_evaluate: usize,
2518    pub layers: Vec<LayerInfo>,
2519    pub cycles_detected: usize,
2520    pub dirty_count: usize,
2521    pub volatile_count: usize,
2522    pub parallel_enabled: bool,
2523    pub estimated_parallel_layers: usize,
2524    pub target_cells: Vec<String>,
2525}
2526
2527impl<R> Engine<R>
2528where
2529    R: EvaluationContext,
2530{
2531    /// # Panics
2532    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2533    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2534    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2535    /// rejects these at build; this re-validates configs assembled via
2536    /// struct literals.
2537    pub fn new(resolver: R, config: EvalConfig) -> Self {
2538        if let Err(msg) = config.cycle.validate() {
2539            panic!("invalid CycleConfig: {msg}");
2540        }
2541        crate::builtins::load_builtins();
2542        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2543            &config.evaluation_budgets,
2544            config.max_vertices,
2545            config.max_memory_mb,
2546            config.max_eval_time,
2547        );
2548
2549        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2550            #[cfg(feature = "system-clock")]
2551            {
2552                Arc::new(crate::timezone::SystemClock::new(
2553                    crate::timezone::TimeZoneSpec::default(),
2554                ))
2555            }
2556            #[cfg(not(feature = "system-clock"))]
2557            {
2558                Arc::new(crate::timezone::FixedClock::new(
2559                    chrono::DateTime::UNIX_EPOCH,
2560                    crate::timezone::TimeZoneSpec::Utc,
2561                ))
2562            }
2563        });
2564
2565        // Initialize thread pool based on config
2566        let thread_pool = if config.enable_parallel {
2567            let mut builder = ThreadPoolBuilder::new();
2568            if let Some(max_threads) = config.max_threads {
2569                builder = builder.num_threads(max_threads);
2570            }
2571
2572            match builder.build() {
2573                Ok(pool) => Some(Arc::new(pool)),
2574                Err(_) => {
2575                    // Fall back to sequential evaluation if thread pool creation fails
2576                    None
2577                }
2578            }
2579        } else {
2580            None
2581        };
2582
2583        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2584        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2585        let function_provider_revision_seen = resolver.planning_semantic_revision();
2586        let mut engine = Self {
2587            graph: DependencyGraph::new_with_config(config.clone()),
2588            resolver,
2589            config,
2590            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2591            clock: crate::timezone::SnapshotClock::new(clock),
2592            thread_pool,
2593            recalc_epoch: 0,
2594            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2595            topology_epoch: 0,
2596            legacy_island_structural_summaries_trusted: true,
2597            cached_static_schedule: None,
2598            cached_mixed_topology: None,
2599            mixed_topology_cache_builds: 0,
2600            mixed_topology_cache_hits: 0,
2601            mixed_topology_cache_overflows: 0,
2602            mixed_topology_cache_skip_streak: 0,
2603            spill_mgr: ShimSpillManager::default(),
2604            arrow_sheets: SheetStore::default(),
2605            has_edited: false,
2606            overlay_compactions: 0,
2607            computed_overlay_bytes_estimate: 0,
2608            computed_overlay_mirroring_disabled: false,
2609            force_materialize_range_views: false,
2610            row_bounds_cache: std::sync::RwLock::new(None),
2611            used_axis_bounds_cache: std::sync::RwLock::new(None),
2612            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2613            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2614            source_formula_token: Arc::new(()),
2615            recalc_plan_token: Arc::new(()),
2616            staged_formulas: std::collections::HashMap::new(),
2617            staged_formula_index: StagedFormulaIndex::default(),
2618            row_visibility: FxHashMap::default(),
2619            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2620            formula_parse_diagnostics: Vec::new(),
2621            last_formula_ingest_report: None,
2622            formula_ingest_report_total: FormulaIngestReport::default(),
2623            formula_plane_cycle_member_span_demotions: 0,
2624            formula_plane_capacity_bailouts: 0,
2625            formula_plane_structural_span_candidates: 0,
2626            active_cancel_flag: None,
2627            active_evaluation_deadline: None,
2628            action_depth: 0,
2629            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2630            virtual_dep_fallback_activations: 0,
2631            last_cycle_telemetry: CycleTelemetry::default(),
2632            next_evaluation_resource_request_id: 1,
2633            evaluation_resource_request_depth: 0,
2634            active_evaluation_resource_request: None,
2635            last_evaluation_resource_request: None,
2636            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2637            evaluation_resource_request_started_at: None,
2638            evaluation_resource_budgets: resolved_resources.budgets,
2639            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2640            active_resource_ledger: None,
2641            pending_iterative_redirty: Vec::new(),
2642            iterative_state_values: FxHashMap::default(),
2643            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2644            function_provider_revision_seen,
2645            #[cfg(test)]
2646            last_formula_plane_span_eval_report: None,
2647            #[cfg(test)]
2648            evaluation_request_begin_count_for_test: 0,
2649            #[cfg(any(test, feature = "test-support"))]
2650            before_prepared_span_commit_hook: None,
2651            #[cfg(test)]
2652            before_target_preparation_commit_hook: None,
2653            #[cfg(test)]
2654            before_target_planning_snapshot_hook: None,
2655            #[cfg(test)]
2656            inject_target_semantic_stale_once_for_test: false,
2657            #[cfg(test)]
2658            force_virtual_dep_changes_remaining_for_test: 0,
2659            #[cfg(test)]
2660            fail_evaluation_commit_preflight_once_for_test: false,
2661            #[cfg(test)]
2662            force_source_family_fallback: false,
2663            #[cfg(test)]
2664            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2665            #[cfg(test)]
2666            fragmented_commit_fault_for_test: None,
2667            #[cfg(test)]
2668            formula_span_demotion_fault_for_test: None,
2669            #[cfg(test)]
2670            target_preparation_fault_for_test: None,
2671            #[cfg(test)]
2672            force_non_cycle_schedule_fallback_for_test: false,
2673            #[cfg(test)]
2674            mixed_topology_index_builds_for_test: 0,
2675            #[cfg(test)]
2676            before_legacy_fallback_final_provider_sample_hook: None,
2677            #[cfg(test)]
2678            after_eager_proposal_commit_hook: None,
2679        };
2680        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
2681        engine.config.arrow_storage_enabled = true;
2682        engine.config.delta_overlay_enabled = true;
2683        engine.config.write_formula_overlay_enabled = true;
2684        let default_sheet = engine.graph.default_sheet_name().to_string();
2685        engine.ensure_arrow_sheet(&default_sheet);
2686        engine
2687    }
2688
2689    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
2690    ///
2691    /// # Panics
2692    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
2693    pub fn with_thread_pool(
2694        resolver: R,
2695        config: EvalConfig,
2696        thread_pool: Arc<rayon::ThreadPool>,
2697    ) -> Self {
2698        if let Err(msg) = config.cycle.validate() {
2699            panic!("invalid CycleConfig: {msg}");
2700        }
2701        crate::builtins::load_builtins();
2702        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2703            &config.evaluation_budgets,
2704            config.max_vertices,
2705            config.max_memory_mb,
2706            config.max_eval_time,
2707        );
2708        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2709            #[cfg(feature = "system-clock")]
2710            {
2711                Arc::new(crate::timezone::SystemClock::new(
2712                    crate::timezone::TimeZoneSpec::default(),
2713                ))
2714            }
2715            #[cfg(not(feature = "system-clock"))]
2716            {
2717                Arc::new(crate::timezone::FixedClock::new(
2718                    chrono::DateTime::UNIX_EPOCH,
2719                    crate::timezone::TimeZoneSpec::Utc,
2720                ))
2721            }
2722        });
2723        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2724        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2725        let function_provider_revision_seen = resolver.planning_semantic_revision();
2726        let mut engine = Self {
2727            graph: DependencyGraph::new_with_config(config.clone()),
2728            resolver,
2729            config,
2730            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2731            clock: crate::timezone::SnapshotClock::new(clock),
2732            thread_pool: Some(thread_pool),
2733            recalc_epoch: 0,
2734            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2735            topology_epoch: 0,
2736            legacy_island_structural_summaries_trusted: true,
2737            cached_static_schedule: None,
2738            cached_mixed_topology: None,
2739            mixed_topology_cache_builds: 0,
2740            mixed_topology_cache_hits: 0,
2741            mixed_topology_cache_overflows: 0,
2742            mixed_topology_cache_skip_streak: 0,
2743            spill_mgr: ShimSpillManager::default(),
2744            arrow_sheets: SheetStore::default(),
2745            has_edited: false,
2746            overlay_compactions: 0,
2747            computed_overlay_bytes_estimate: 0,
2748            computed_overlay_mirroring_disabled: false,
2749            force_materialize_range_views: false,
2750            row_bounds_cache: std::sync::RwLock::new(None),
2751            used_axis_bounds_cache: std::sync::RwLock::new(None),
2752            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2753            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2754            source_formula_token: Arc::new(()),
2755            recalc_plan_token: Arc::new(()),
2756            staged_formulas: std::collections::HashMap::new(),
2757            staged_formula_index: StagedFormulaIndex::default(),
2758            row_visibility: FxHashMap::default(),
2759            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2760            formula_parse_diagnostics: Vec::new(),
2761            last_formula_ingest_report: None,
2762            formula_ingest_report_total: FormulaIngestReport::default(),
2763            formula_plane_cycle_member_span_demotions: 0,
2764            formula_plane_capacity_bailouts: 0,
2765            formula_plane_structural_span_candidates: 0,
2766            active_cancel_flag: None,
2767            active_evaluation_deadline: None,
2768            action_depth: 0,
2769            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2770            virtual_dep_fallback_activations: 0,
2771            last_cycle_telemetry: CycleTelemetry::default(),
2772            next_evaluation_resource_request_id: 1,
2773            evaluation_resource_request_depth: 0,
2774            active_evaluation_resource_request: None,
2775            last_evaluation_resource_request: None,
2776            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2777            evaluation_resource_request_started_at: None,
2778            evaluation_resource_budgets: resolved_resources.budgets,
2779            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2780            active_resource_ledger: None,
2781            pending_iterative_redirty: Vec::new(),
2782            iterative_state_values: FxHashMap::default(),
2783            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2784            function_provider_revision_seen,
2785            #[cfg(test)]
2786            last_formula_plane_span_eval_report: None,
2787            #[cfg(test)]
2788            evaluation_request_begin_count_for_test: 0,
2789            #[cfg(any(test, feature = "test-support"))]
2790            before_prepared_span_commit_hook: None,
2791            #[cfg(test)]
2792            before_target_preparation_commit_hook: None,
2793            #[cfg(test)]
2794            before_target_planning_snapshot_hook: None,
2795            #[cfg(test)]
2796            inject_target_semantic_stale_once_for_test: false,
2797            #[cfg(test)]
2798            force_virtual_dep_changes_remaining_for_test: 0,
2799            #[cfg(test)]
2800            fail_evaluation_commit_preflight_once_for_test: false,
2801            #[cfg(test)]
2802            force_source_family_fallback: false,
2803            #[cfg(test)]
2804            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2805            #[cfg(test)]
2806            fragmented_commit_fault_for_test: None,
2807            #[cfg(test)]
2808            formula_span_demotion_fault_for_test: None,
2809            #[cfg(test)]
2810            target_preparation_fault_for_test: None,
2811            #[cfg(test)]
2812            force_non_cycle_schedule_fallback_for_test: false,
2813            #[cfg(test)]
2814            mixed_topology_index_builds_for_test: 0,
2815            #[cfg(test)]
2816            before_legacy_fallback_final_provider_sample_hook: None,
2817            #[cfg(test)]
2818            after_eager_proposal_commit_hook: None,
2819        };
2820        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
2821        engine.config.arrow_storage_enabled = true;
2822        engine.config.delta_overlay_enabled = true;
2823        engine.config.write_formula_overlay_enabled = true;
2824        let default_sheet = engine.graph.default_sheet_name().to_string();
2825        engine.ensure_arrow_sheet(&default_sheet);
2826        engine
2827    }
2828
2829    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
2830        &self.workbook_load_limits
2831    }
2832
2833    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
2834        self.workbook_load_limits = limits;
2835    }
2836
2837    fn clear_source_cache(&self) {
2838        if let Ok(mut g) = self.source_cache.write() {
2839            *g = SourceCache::default();
2840        }
2841    }
2842
2843    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
2844        &self.last_virtual_dep_telemetry
2845    }
2846
2847    /// Telemetry from Runtime SCC evaluation during the most recent
2848    /// evaluation request (always default-zero under `CycleDetection::Static`
2849    /// or when `enable_virtual_dep_telemetry` is off).
2850    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
2851        &self.last_cycle_telemetry
2852    }
2853
2854    /// Resource observations for the most recently completed public evaluation request.
2855    pub fn last_evaluation_resource_request_stats(
2856        &self,
2857    ) -> Option<&EvaluationResourceRequestStats> {
2858        self.last_evaluation_resource_request.as_ref()
2859    }
2860
2861    /// Cumulative resource observations since engine creation or the last telemetry reset.
2862    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
2863        self.evaluation_resource_baseline
2864    }
2865
2866    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
2867        &self.evaluation_resource_budgets
2868    }
2869
2870    /// At most one diagnostic is emitted for deprecated resource fields.
2871    pub fn evaluation_resource_config_diagnostic(
2872        &self,
2873    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
2874        self.evaluation_resource_config_diagnostic.as_ref()
2875    }
2876
2877    /// Reset accumulated and last-request observations without reusing request IDs.
2878    pub fn reset_evaluation_resource_telemetry(&mut self) {
2879        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
2880        self.last_evaluation_resource_request = None;
2881    }
2882
2883    fn duration_ns(duration: std::time::Duration) -> u64 {
2884        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
2885    }
2886
2887    fn observe_evaluation_resource_request<T>(
2888        &mut self,
2889        kind: EvaluationRequestKind,
2890        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
2891    ) -> Result<T, ExcelError> {
2892        let outermost = self.evaluation_resource_request_depth == 0;
2893        if outermost {
2894            let request_id = self.next_evaluation_resource_request_id;
2895            self.next_evaluation_resource_request_id = request_id
2896                .checked_add(1)
2897                .expect("evaluation resource request ID exhausted");
2898            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
2899                request_id,
2900                kind,
2901                self.config.formula_plane_mode,
2902                self.staged_formula_count(),
2903            ));
2904            self.evaluation_resource_baseline.record_started(request_id);
2905            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
2906            self.active_resource_ledger = Some(ResourceLedger::new(
2907                Some(request_id),
2908                self.evaluation_resource_budgets.clone(),
2909            ));
2910        }
2911        self.evaluation_resource_request_depth =
2912            self.evaluation_resource_request_depth.saturating_add(1);
2913        let result = if outermost {
2914            self.resource_checkpoint(0).and_then(|()| evaluate(self))
2915        } else {
2916            evaluate(self)
2917        };
2918        self.evaluation_resource_request_depth =
2919            self.evaluation_resource_request_depth.saturating_sub(1);
2920
2921        if outermost {
2922            let total_ns = self
2923                .evaluation_resource_request_started_at
2924                .take()
2925                .map(|start| Self::duration_ns(start.elapsed()))
2926                .unwrap_or(0);
2927            let mut stats = self
2928                .active_evaluation_resource_request
2929                .take()
2930                .expect("outer evaluation resource request has active stats");
2931            let mut ledger = self
2932                .active_resource_ledger
2933                .take()
2934                .expect("outer evaluation resource request has active ledger");
2935            ledger.release_all_scratch();
2936            stats.ledger.update(ledger.snapshot());
2937            stats.outcome = match &result {
2938                Ok(_) => EvaluationRequestOutcome::Success,
2939                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
2940                    EvaluationRequestOutcome::Cancelled
2941                }
2942                Err(_) => EvaluationRequestOutcome::Error,
2943            };
2944            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
2945                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
2946                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
2947                } else {
2948                    FormulaDirtyLeaseOutcome::RetainedOnError
2949                };
2950            }
2951            stats.phases.total_ns = total_ns;
2952            let attributed = stats
2953                .phases
2954                .staged_prepare_ns
2955                .saturating_add(stats.phases.topology_ns)
2956                .saturating_add(stats.phases.materialization_ns);
2957            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
2958            self.evaluation_resource_baseline.record_finished(&stats);
2959            self.last_evaluation_resource_request = Some(stats);
2960        }
2961        result
2962    }
2963
2964    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
2965        self.evaluation_resource_budgets = budgets.clone();
2966        self.config.evaluation_budgets = budgets.clone();
2967        self.graph.set_evaluation_budgets(budgets);
2968    }
2969
2970    #[cfg(test)]
2971    pub(crate) fn set_evaluation_budgets_for_test(
2972        &mut self,
2973        budgets: crate::engine::EvaluationBudgets,
2974    ) {
2975        self.set_evaluation_resource_budgets(budgets);
2976    }
2977
2978    fn resource_loop_checkpoint(
2979        ledger: &mut Option<ResourceLedger>,
2980        work_units: u64,
2981    ) -> Result<(), ExcelError> {
2982        ledger
2983            .as_mut()
2984            .map_or(Ok(()), |ledger| {
2985                ledger
2986                    .charge_work(work_units)
2987                    .and_then(|()| ledger.checkpoint_deadline())
2988            })
2989            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
2990    }
2991
2992    fn preflight_evaluation_commit_window(
2993        &mut self,
2994        bounded_writes: usize,
2995    ) -> Result<crate::instant::FzInstant, ExcelError> {
2996        #[cfg(test)]
2997        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
2998            return Err(crate::engine::ResourceLedgerError::Exhausted(
2999                formualizer_common::ResourceExhaustionDetail {
3000                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3001                    limit: 0,
3002                    observed: 1,
3003                    request_id: self
3004                        .active_evaluation_resource_request
3005                        .as_ref()
3006                        .map(|stats| stats.request_id),
3007                },
3008            )
3009            .into_excel_error());
3010        }
3011        let estimate = std::time::Duration::from_nanos(
3012            u64::try_from(bounded_writes)
3013                .unwrap_or(u64::MAX)
3014                .saturating_mul(100),
3015        );
3016        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3017            ledger
3018                .preflight_commit_window(estimate)
3019                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3020        }
3021        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3022            stats.evaluation_commit_preflight_count =
3023                stats.evaluation_commit_preflight_count.saturating_add(1);
3024            stats.evaluation_commit_estimated_ns = stats
3025                .evaluation_commit_estimated_ns
3026                .saturating_add(Self::duration_ns(estimate));
3027        }
3028        Ok(crate::instant::FzInstant::now())
3029    }
3030
3031    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3032        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3033            stats.evaluation_commit_actual_ns = stats
3034                .evaluation_commit_actual_ns
3035                .saturating_add(Self::duration_ns(started.elapsed()));
3036        }
3037    }
3038
3039    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3040        if self
3041            .active_cancel_flag
3042            .as_ref()
3043            .is_some_and(|cancel| cancel.is_cancelled())
3044        {
3045            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3046        }
3047        if self
3048            .active_evaluation_deadline
3049            .is_some_and(|deadline| Instant::now() >= deadline)
3050        {
3051            return Err(crate::engine::ResourceLedgerError::Exhausted(
3052                formualizer_common::ResourceExhaustionDetail {
3053                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3054                    limit: 0,
3055                    observed: 1,
3056                    request_id: self
3057                        .active_evaluation_resource_request
3058                        .as_ref()
3059                        .map(|request| request.request_id),
3060                },
3061            )
3062            .into_excel_error()
3063            .with_message(message));
3064        }
3065        Ok(())
3066    }
3067
3068    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3069        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3070            return Ok(());
3071        };
3072        ledger
3073            .charge_work(work_units)
3074            .and_then(|()| ledger.checkpoint_deadline())
3075            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3076    }
3077
3078    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3079        if work_units == 0 {
3080            return self.resource_checkpoint(0);
3081        }
3082        while work_units > 0 {
3083            let chunk = work_units.min(256);
3084            self.resource_checkpoint(chunk)?;
3085            work_units -= chunk;
3086        }
3087        Ok(())
3088    }
3089
3090    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3091        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3092            // Exact request topology and activation of the scratch cap are C1b. C1a records
3093            // scoped ownership but must not introduce a new skip or terminal path.
3094            ledger.observe_scratch(bytes);
3095        }
3096        Ok(())
3097    }
3098
3099    fn release_request_scratch(&mut self, bytes: u64) {
3100        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3101            let released = ledger.release_scratch(bytes);
3102            debug_assert!(
3103                released.is_ok(),
3104                "request scratch release exceeded the outstanding reservation"
3105            );
3106        }
3107    }
3108
3109    fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3110        self.active_resource_ledger
3111            .as_ref()
3112            .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3113    }
3114
3115    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3116        self.active_resource_ledger
3117            .as_mut()
3118            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3119            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3120    }
3121
3122    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3123        self.active_resource_ledger
3124            .as_mut()
3125            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3126            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3127    }
3128
3129    fn with_request_scratch<T>(
3130        &mut self,
3131        bytes: u64,
3132        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3133    ) -> Result<T, ExcelError> {
3134        self.reserve_request_scratch(bytes)?;
3135        let result = work(self);
3136        self.release_request_scratch(bytes);
3137        result
3138    }
3139
3140    fn observe_staged_preparation(
3141        &mut self,
3142        selected: usize,
3143        retained: usize,
3144        elapsed: std::time::Duration,
3145    ) {
3146        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3147            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3148            stats.staged_retained = retained as u64;
3149            stats.phases.staged_prepare_ns = stats
3150                .phases
3151                .staged_prepare_ns
3152                .saturating_add(Self::duration_ns(elapsed));
3153        }
3154    }
3155
3156    fn graph_admission_enabled(&self) -> bool {
3157        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3158    }
3159
3160    fn preflight_graph_admission(
3161        &mut self,
3162        usage: crate::engine::resource_ledger::GraphAdmission,
3163    ) -> Result<(), ExcelError> {
3164        let request_id = self
3165            .active_evaluation_resource_request
3166            .as_ref()
3167            .map(|stats| stats.request_id);
3168        crate::engine::resource_ledger::preflight_graph_admission(
3169            &self.evaluation_resource_budgets,
3170            usage,
3171            request_id,
3172        )
3173        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3174    }
3175
3176    fn prepared_legacy_admission(
3177        &mut self,
3178        plan: &PreparedLegacyGraphPlan,
3179        materialization_cells: u64,
3180    ) -> Result<(), ExcelError> {
3181        if !self.graph_admission_enabled() {
3182            return Ok(());
3183        }
3184        let stats = self.graph.baseline_stats();
3185        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3186            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3187        })?;
3188        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3189            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3190        })?;
3191        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3192            final_vertices: stats
3193                .graph_vertex_count
3194                .checked_add(plan.new_vertex_count())
3195                .ok_or_else(|| {
3196                    ExcelError::new(ExcelErrorKind::NImpl)
3197                        .with_message("graph vertex count overflow")
3198                })?,
3199            final_edges: stats
3200                .graph_edge_count
3201                .checked_sub(removed_edges)
3202                .and_then(|count| count.checked_add(added_edges))
3203                .ok_or_else(|| {
3204                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3205                })?,
3206            materialization_cells,
3207            added_vertices: plan.new_vertex_count(),
3208            added_edges,
3209        })
3210    }
3211
3212    fn observe_target_admission_failure(
3213        &mut self,
3214        reason: formualizer_common::ResourceExhaustionReason,
3215    ) {
3216        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3217            stats.target_admission_failure = Some(reason);
3218        }
3219    }
3220
3221    fn observe_target_preparation_report(
3222        &mut self,
3223        report: &crate::engine::PreparedTargetGraphReport,
3224    ) {
3225        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3226            let index = match reason {
3227                crate::engine::OpaqueReason::DynamicReference => 0,
3228                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3229                crate::engine::OpaqueReason::UnknownFunction => 2,
3230                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3231                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3232                crate::engine::OpaqueReason::UnresolvedName => 5,
3233                crate::engine::OpaqueReason::UnresolvedTable => 6,
3234                crate::engine::OpaqueReason::FormulaName => 7,
3235                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3236                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3237                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3238            };
3239            1u64 << index
3240        };
3241        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3242            stats.staged_selected = report.selected_staged_cells as u64;
3243            stats.staged_retained = report.retained_staged_cells as u64;
3244            stats.target_requested = report.requested_targets as u64;
3245            stats.target_normalized_regions = report.normalized_regions as u64;
3246            stats.target_scope_level = match &report.widened_scope {
3247                crate::engine::PrepareScope::Exact => 0,
3248                crate::engine::PrepareScope::Sheets(_) => 1,
3249                crate::engine::PrepareScope::Workbook => 2,
3250            };
3251            stats.target_widening_reason_bits = report
3252                .widening_reasons
3253                .iter()
3254                .copied()
3255                .fold(0, |bits, reason| bits | reason_bit(reason));
3256            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3257            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3258            stats.target_commit_estimated_work = report.estimated_commit_work;
3259            stats.target_commit_actual_work = report.actual_commit_work;
3260            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3261            stats.phases.staged_prepare_ns = stats
3262                .phases
3263                .staged_prepare_ns
3264                .saturating_add(Self::duration_ns(report.commit_window));
3265        }
3266    }
3267
3268    fn observe_topology(
3269        &mut self,
3270        outcome: FormulaPlaneTopologyCacheOutcome,
3271        strategy: FormulaPlaneTopologyStrategy,
3272        observed: &MixedTopologyCompileStats,
3273        elapsed: std::time::Duration,
3274    ) {
3275        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3276            if outcome.severity() > stats.topology.cache_outcome.severity() {
3277                stats.topology.cache_outcome = outcome;
3278            }
3279            if strategy.severity() > stats.topology.strategy.severity() {
3280                stats.topology.strategy = strategy;
3281            }
3282            match outcome {
3283                FormulaPlaneTopologyCacheOutcome::Hit => {
3284                    stats.topology.cache_hit_events =
3285                        stats.topology.cache_hit_events.saturating_add(1);
3286                }
3287                FormulaPlaneTopologyCacheOutcome::Built => {
3288                    stats.topology.cache_build_events =
3289                        stats.topology.cache_build_events.saturating_add(1);
3290                }
3291                FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3292                | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3293                    stats.topology.cache_build_events =
3294                        stats.topology.cache_build_events.saturating_add(1);
3295                    stats.topology.cache_skip_events =
3296                        stats.topology.cache_skip_events.saturating_add(1);
3297                }
3298                FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3299            }
3300            if matches!(
3301                outcome,
3302                FormulaPlaneTopologyCacheOutcome::Built
3303                    | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3304                    | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3305            ) {
3306                stats.topology.producers_observed = stats
3307                    .topology
3308                    .producers_observed
3309                    .saturating_add(observed.producers as u64);
3310                stats.topology.candidates_observed = stats
3311                    .topology
3312                    .candidates_observed
3313                    .saturating_add(observed.candidates as u64);
3314                stats.topology.edges_observed = stats
3315                    .topology
3316                    .edges_observed
3317                    .saturating_add(observed.relationships as u64);
3318                stats.topology.candidate_cap_hits = stats
3319                    .topology
3320                    .candidate_cap_hits
3321                    .saturating_add(observed.candidate_overflow_count as u64);
3322                stats.topology.edge_cap_hits = stats
3323                    .topology
3324                    .edge_cap_hits
3325                    .saturating_add(observed.edge_overflow_count as u64);
3326                stats.topology.byte_cap_hits = stats
3327                    .topology
3328                    .byte_cap_hits
3329                    .saturating_add(observed.memory_overflow_count as u64);
3330                stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3331                    Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3332                } else if stats.topology.edge_cap_hits > 0 {
3333                    Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3334                } else if stats.topology.candidate_cap_hits > 0 {
3335                    Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3336                } else {
3337                    None
3338                };
3339                stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3340                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3341                } else if stats.topology.edge_cap_hits > 0 {
3342                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3343                } else if stats.topology.candidate_cap_hits > 0 {
3344                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3345                } else {
3346                    None
3347                };
3348            }
3349            stats.topology.retained_bytes_observed = stats
3350                .topology
3351                .retained_bytes_observed
3352                .max(observed.estimated_memory_bytes as u64);
3353            stats.phases.topology_ns = stats
3354                .phases
3355                .topology_ns
3356                .saturating_add(Self::duration_ns(elapsed));
3357        }
3358    }
3359
3360    fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3361        if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3362            && strategy.severity() > stats.topology.strategy.severity()
3363        {
3364            stats.topology.strategy = strategy;
3365        }
3366    }
3367
3368    fn observe_formula_plane_route(
3369        &mut self,
3370        plan: &LegacyIslandPlan,
3371        phase: FormulaPlaneRoutePhase,
3372        reason: FormulaPlaneRouteTransitionReason,
3373        demotion_generation: usize,
3374        replan_generation: usize,
3375    ) {
3376        if plan.is_empty() {
3377            return;
3378        }
3379        let mut sheet_ids = [0_u16;
3380            crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3381        let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3382        sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3383        let authority = self.graph.formula_authority();
3384        let event = FormulaPlaneRouteEvent {
3385            island_id: plan.island_id,
3386            phase,
3387            route: FormulaPlaneRoute::ContractedLegacyIsland,
3388            sheet_ids,
3389            sheet_count: sheet_count as u8,
3390            authority_epoch: authority.plane.epoch().0,
3391            index_epoch: authority.indexes_epoch(),
3392            transition_reason: reason,
3393            demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3394            replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3395        };
3396        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3397            let index = usize::from(stats.topology.route_event_count);
3398            if index < stats.topology.route_events.len() {
3399                stats.topology.route_events[index] = event;
3400                stats.topology.route_event_count =
3401                    stats.topology.route_event_count.saturating_add(1);
3402            } else {
3403                stats.topology.route_events_dropped =
3404                    stats.topology.route_events_dropped.saturating_add(1);
3405            }
3406            stats.topology.route_event_bytes_observed = stats
3407                .topology
3408                .route_event_bytes_observed
3409                .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3410            stats.topology.island_membership_vertices = plan.membership.len() as u64;
3411            stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3412            stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3413            stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3414        }
3415    }
3416
3417    fn observe_materialization(
3418        &mut self,
3419        placements: usize,
3420        cycle: bool,
3421        elapsed: std::time::Duration,
3422    ) {
3423        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3424            if cycle {
3425                stats.cycle_materialized_cells = stats
3426                    .cycle_materialized_cells
3427                    .saturating_add(placements as u64);
3428            } else {
3429                stats.fallback_materialized_cells = stats
3430                    .fallback_materialized_cells
3431                    .saturating_add(placements as u64);
3432                stats.topology.strategy =
3433                    FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3434            }
3435            stats.phases.materialization_ns = stats
3436                .phases
3437                .materialization_ns
3438                .saturating_add(Self::duration_ns(elapsed));
3439        }
3440    }
3441
3442    fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3443        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3444            stats.dirty_lease = if empty {
3445                FormulaDirtyLeaseOutcome::Empty
3446            } else {
3447                FormulaDirtyLeaseOutcome::Acquired
3448            };
3449        }
3450    }
3451
3452    fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3453        let empty = lease.is_empty();
3454        let _ = self.graph.ack_formula_dirty(lease);
3455        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3456            stats.dirty_lease = if empty {
3457                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3458            } else {
3459                FormulaDirtyLeaseOutcome::Acknowledged
3460            };
3461        }
3462    }
3463
3464    fn checked_ack_formula_dirty_observed(
3465        &mut self,
3466        lease: FormulaDirtyLease,
3467    ) -> Result<(), ExcelError> {
3468        self.resource_checkpoint(0)?;
3469        self.ack_formula_dirty_observed(lease);
3470        Ok(())
3471    }
3472
3473    fn checked_ack_formula_dirty_sublease_observed(
3474        &mut self,
3475        lease: FormulaDirtyLease,
3476        owned_events: &[usize],
3477    ) -> Result<(), ExcelError> {
3478        self.resource_checkpoint(owned_events.len() as u64)?;
3479        let empty = lease.is_empty();
3480        let partial = owned_events.len() < lease.len();
3481        let sublease = lease.sublease(owned_events.iter().copied());
3482        if !self.graph.ack_formula_dirty_sublease(sublease) {
3483            return Err(ExcelError::new(ExcelErrorKind::NImpl)
3484                .with_message("FormulaPlane dirty sublease became stale"));
3485        }
3486        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3487            stats.dirty_lease = if empty {
3488                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3489            } else if partial {
3490                FormulaDirtyLeaseOutcome::AcknowledgedPartial
3491            } else {
3492                FormulaDirtyLeaseOutcome::Acknowledged
3493            };
3494        }
3495        Ok(())
3496    }
3497
3498    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3499    /// take the per-recalc volatile clock sample. Called at the start of
3500    /// every evaluation request that walks schedule units.
3501    fn begin_evaluation_request(&mut self) {
3502        #[cfg(test)]
3503        {
3504            self.evaluation_request_begin_count_for_test = self
3505                .evaluation_request_begin_count_for_test
3506                .saturating_add(1);
3507        }
3508        self.last_cycle_telemetry = CycleTelemetry::default();
3509        // Defensive: consumed at the end of the previous request; a request
3510        // that errored out mid-walk must not leak its members into this one.
3511        self.pending_iterative_redirty.clear();
3512        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3513        // read within this request (including SCC iteration passes) observes
3514        // this sample.
3515        self.clock.refresh();
3516    }
3517
3518    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3519    /// SCCs that iterated this recalc (`CyclePolicy::Iterate`), so circular
3520    /// cells re-evaluate on every recalc exactly like Excel's iterative
3521    /// calculation (spec §4 persistence / §7.6 accumulator / §7.11 volatile
3522    /// redirty). Replaces the bare `graph.redirty_volatiles()` call at every
3523    /// evaluation-flow exit; must run AFTER the flow's `clear_dirty_flags`.
3524    fn redirty_for_next_recalc(&mut self) {
3525        self.graph.redirty_volatiles();
3526        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3527        // Refresh the §4-persistence snapshot: these final values survive
3528        // structural edits that clear the computed overlay (the only value
3529        // home in canonical mode) so the next SCC task can re-seed from them
3530        // (see `iterative_state_values`). Replaced wholesale each recalc —
3531        // when nothing iterates the map empties and stays free.
3532        self.iterative_state_values.clear();
3533        for &vertex in &pending {
3534            if !self.graph.vertex_exists(vertex) {
3535                continue;
3536            }
3537            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3538                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3539                if let Some(value) =
3540                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
3541                    && !matches!(value, LiteralValue::Empty)
3542                {
3543                    self.iterative_state_values.insert(vertex, value);
3544                }
3545            }
3546        }
3547        if !pending.is_empty() {
3548            self.graph.redirty_iterative_members(&pending);
3549        }
3550    }
3551
3552    pub fn virtual_dep_fallback_activations(&self) -> u64 {
3553        self.virtual_dep_fallback_activations
3554    }
3555
3556    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3557        self.lookup_index_cache.report()
3558    }
3559
3560    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3561        let start_row = view.start_row();
3562        let end_row = view.end_row();
3563        let start_col = view.start_col();
3564        let end_col = view.end_col();
3565        for row in start_row..=end_row {
3566            let Ok(row_u32) = u32::try_from(row) else {
3567                return true;
3568            };
3569            for col in start_col..=end_col {
3570                let Ok(col_u32) = u32::try_from(col) else {
3571                    return true;
3572                };
3573                let cell_ref = self
3574                    .graph
3575                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3576                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3577                    && self.graph.is_volatile(*vertex_id)
3578                {
3579                    return true;
3580                }
3581            }
3582        }
3583        false
3584    }
3585
3586    fn build_lookup_index_impl(
3587        &self,
3588        view: &RangeView<'_>,
3589        axis: LookupAxis,
3590    ) -> Option<Arc<LookupIndex>> {
3591        let (rows, cols) = view.dims();
3592        if rows == 0 || cols == 0 {
3593            self.lookup_index_cache.note_skipped_tiny();
3594            return None;
3595        }
3596        let len = match axis {
3597            LookupAxis::ColumnInView(col) => {
3598                if col >= cols {
3599                    self.lookup_index_cache.note_skipped_tiny();
3600                    return None;
3601                }
3602                rows
3603            }
3604            LookupAxis::RowInView(row) => {
3605                if row >= rows {
3606                    self.lookup_index_cache.note_skipped_tiny();
3607                    return None;
3608                }
3609                cols
3610            }
3611        };
3612        if len < 64 {
3613            self.lookup_index_cache.note_skipped_tiny();
3614            return None;
3615        }
3616
3617        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3618        let key = LookupIndexKey {
3619            sheet_id,
3620            start_row: u32::try_from(view.start_row()).ok()?,
3621            start_col: u32::try_from(view.start_col()).ok()?,
3622            end_row: u32::try_from(view.end_row()).ok()?,
3623            end_col: u32::try_from(view.end_col()).ok()?,
3624            axis,
3625            snapshot_id: self.data_snapshot_id(),
3626        };
3627        if let Some(index) = self.lookup_index_cache.get(&key) {
3628            return Some(index);
3629        }
3630        if self
3631            .lookup_index_cache
3632            .would_exceed_cap(estimate_bytes(len, 0))
3633        {
3634            self.lookup_index_cache.note_skipped_cap();
3635            return None;
3636        }
3637        if !self.lookup_index_cache.should_build(key) {
3638            return None;
3639        }
3640        if self.lookup_index_cache.is_known_volatile(&key) {
3641            self.lookup_index_cache.note_skipped_volatile();
3642            return None;
3643        }
3644        if self.lookup_view_contains_volatile(view, sheet_id) {
3645            self.lookup_index_cache.note_volatile_key(key);
3646            self.lookup_index_cache.note_skipped_volatile();
3647            return None;
3648        }
3649        match LookupIndex::build(view, axis, self.config.date_system).ok()? {
3650            BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3651            BuildOutcome::ErrorInLookupAxis => {
3652                self.lookup_index_cache.note_skipped_error();
3653                None
3654            }
3655            BuildOutcome::Degenerate => {
3656                self.lookup_index_cache.note_skipped_tiny();
3657                None
3658            }
3659        }
3660    }
3661
3662    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
3663        if !self.config.enable_virtual_dep_telemetry {
3664            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
3665                fallback_mode_activations: self.virtual_dep_fallback_activations,
3666                ..VirtualDepTelemetry::default()
3667            };
3668        }
3669    }
3670
3671    fn source_cache_session(&self) -> SourceCacheSession {
3672        self.clear_source_cache();
3673        SourceCacheSession {
3674            cache: self.source_cache.clone(),
3675        }
3676    }
3677
3678    fn resolve_source_scalar_cached(
3679        &self,
3680        name: &str,
3681        version: Option<u64>,
3682    ) -> Result<LiteralValue, ExcelError> {
3683        let key = (name.to_string(), version);
3684        if let Ok(mut g) = self.source_cache.write() {
3685            if let Some(v) = g.scalars.get(&key) {
3686                return Ok(v.clone());
3687            }
3688
3689            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
3690                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3691                    ExcelError::new(ExcelErrorKind::Ref)
3692                        .with_message(format!("Unresolved source scalar: {name}"))
3693                } else {
3694                    err
3695                }
3696            })?;
3697            g.scalars.insert(key, v.clone());
3698            Ok(v)
3699        } else {
3700            self.resolver.resolve_source_scalar(name).map_err(|err| {
3701                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3702                    ExcelError::new(ExcelErrorKind::Ref)
3703                        .with_message(format!("Unresolved source scalar: {name}"))
3704                } else {
3705                    err
3706                }
3707            })
3708        }
3709    }
3710
3711    fn resolve_source_table_cached(
3712        &self,
3713        name: &str,
3714        version: Option<u64>,
3715    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
3716        let key = (name.to_string(), version);
3717        if let Ok(mut g) = self.source_cache.write() {
3718            if let Some(t) = g.tables.get(&key) {
3719                return Ok(t.clone());
3720            }
3721
3722            let t = self.resolver.resolve_source_table(name).map_err(|err| {
3723                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3724                    ExcelError::new(ExcelErrorKind::Ref)
3725                        .with_message(format!("Unresolved source table: {name}"))
3726                } else {
3727                    err
3728                }
3729            })?;
3730            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
3731            g.tables.insert(key, t.clone());
3732            Ok(t)
3733        } else {
3734            self.resolver
3735                .resolve_source_table(name)
3736                .map_err(|err| {
3737                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3738                        ExcelError::new(ExcelErrorKind::Ref)
3739                            .with_message(format!("Unresolved source table: {name}"))
3740                    } else {
3741                        err
3742                    }
3743                })
3744                .map(Arc::from)
3745        }
3746    }
3747
3748    fn source_table_to_range_view(
3749        &self,
3750        table: &dyn crate::traits::Table,
3751        spec: &Option<formualizer_parse::parser::TableSpecifier>,
3752    ) -> Result<RangeView<'static>, ExcelError> {
3753        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
3754
3755        let owned = match spec {
3756            Some(TableSpecifier::Column(c)) => {
3757                let c = c.trim();
3758                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
3759                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3760                        "Complex structured references not yet supported".to_string(),
3761                    ));
3762                }
3763                table.get_column(c)?.materialise().into_owned()
3764            }
3765            Some(TableSpecifier::ColumnRange(start, end)) => {
3766                let cols = table.columns();
3767                let start = start.trim();
3768                let end = end.trim();
3769                let start_key = start.to_lowercase();
3770                let end_key = end.to_lowercase();
3771                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
3772                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
3773                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
3774                    if si > ei {
3775                        std::mem::swap(&mut si, &mut ei);
3776                    }
3777                    let h = table.data_height();
3778                    let w = ei - si + 1;
3779                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
3780                    for (offset, ci) in (si..=ei).enumerate() {
3781                        let cname = &cols[ci];
3782                        let col_range = table.get_column(cname)?;
3783                        let (rh, _) = col_range.dimensions();
3784                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
3785                            row[offset] = col_range.get(r, 0)?;
3786                        }
3787                    }
3788                    rows
3789                } else {
3790                    return Err(ExcelError::new(ExcelErrorKind::Ref)
3791                        .with_message("Column range refers to unknown column(s)".to_string()));
3792                }
3793            }
3794            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
3795            | Some(TableSpecifier::Headers) => table
3796                .headers_row()
3797                .map(|r| r.materialise().into_owned())
3798                .unwrap_or_default(),
3799            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
3800            | Some(TableSpecifier::Totals) => table
3801                .totals_row()
3802                .map(|r| r.materialise().into_owned())
3803                .unwrap_or_default(),
3804            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
3805                table
3806                    .data_body()
3807                    .map(|r| r.materialise().into_owned())
3808                    .unwrap_or_default()
3809            }
3810            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
3811                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
3812                if let Some(h) = table.headers_row() {
3813                    out.extend(h.iter_rows());
3814                }
3815                if let Some(body) = table.data_body() {
3816                    out.extend(body.iter_rows());
3817                }
3818                if let Some(tr) = table.totals_row() {
3819                    out.extend(tr.iter_rows());
3820                }
3821                out
3822            }
3823            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
3824                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3825                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
3826                ));
3827            }
3828            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
3829                return Err(ExcelError::new(ExcelErrorKind::NImpl)
3830                    .with_message("Complex structured references not yet supported".to_string()));
3831            }
3832            None => {
3833                return Err(ExcelError::new(ExcelErrorKind::NImpl)
3834                    .with_message("Table reference without specifier is unsupported".to_string()));
3835            }
3836        };
3837
3838        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
3839    }
3840
3841    pub fn default_sheet_id(&self) -> SheetId {
3842        self.graph.default_sheet_id()
3843    }
3844
3845    pub fn default_sheet_name(&self) -> &str {
3846        self.graph.default_sheet_name()
3847    }
3848
3849    /// Update the workbook seed for deterministic RNGs in functions.
3850    pub fn set_workbook_seed(&mut self, seed: u64) {
3851        self.config.workbook_seed = seed;
3852    }
3853
3854    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
3855    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
3856        self.config.volatile_level = level;
3857    }
3858
3859    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
3860    pub fn set_deterministic_mode(
3861        &mut self,
3862        mode: crate::engine::DeterministicMode,
3863    ) -> Result<(), ExcelError> {
3864        let clock = mode.build_clock()?;
3865        self.config.deterministic_mode = mode;
3866        self.clock = crate::timezone::SnapshotClock::new(clock);
3867        Ok(())
3868    }
3869
3870    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
3871    /// volatile date/time builtins (`NOW()`, `TODAY()`).
3872    ///
3873    /// The provider is the clock *source*; per spec §7.11 the engine samples
3874    /// it once at the start of every evaluation request and all reads within
3875    /// that recalc (including SCC iteration passes) observe the frozen
3876    /// sample.
3877    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
3878        self.clock = crate::timezone::SnapshotClock::new(clock);
3879    }
3880
3881    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
3882        self.config.deterministic_mode.validate()
3883    }
3884
3885    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
3886        self.graph.sheet_id(name)
3887    }
3888
3889    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
3890        self.add_sheet(name)
3891            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
3892    }
3893
3894    pub fn sheet_name(&self, id: SheetId) -> &str {
3895        self.graph.sheet_name(id)
3896    }
3897
3898    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
3899        let id = self.graph.add_sheet(name)?;
3900        self.ensure_arrow_sheet(name);
3901        // Adding a sheet does not change any existing span result or dependency.
3902        self.mark_topology_edited();
3903        Ok(id)
3904    }
3905
3906    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
3907        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
3908            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
3909        })?;
3910        if new_name.is_empty() || new_name.len() > 255 {
3911            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
3912        }
3913        if self.graph.sheet_id(new_name).is_some() {
3914            return Err(ExcelError::new(ExcelErrorKind::Value)
3915                .with_message(format!("Sheet '{new_name}' already exists")));
3916        }
3917        // Materialize only spans on the source sheet so graph duplication sees
3918        // the formulas being copied. Spans on unrelated sheets remain active.
3919        self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
3920            .map_err(Self::editor_error_to_excel)?;
3921        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
3922
3923        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
3924            let mut copied_sheet = source_sheet;
3925            copied_sheet.name = Arc::<str>::from(new_name);
3926            self.arrow_sheets.sheets.push(copied_sheet);
3927        } else {
3928            self.ensure_arrow_sheet(new_name);
3929        }
3930
3931        let duplicated_formulas = self
3932            .graph
3933            .formula_vertices()
3934            .into_iter()
3935            .filter(|vertex| {
3936                self.graph
3937                    .get_cell_ref(*vertex)
3938                    .is_some_and(|cell| cell.sheet_id == new_id)
3939            })
3940            .collect::<Vec<_>>();
3941        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
3942        self.mark_topology_edited();
3943        Ok(new_id)
3944    }
3945
3946    fn ensure_arrow_sheet(&mut self, name: &str) {
3947        if self.arrow_sheets.sheet(name).is_some() {
3948            return;
3949        }
3950        self.arrow_sheets
3951            .sheets
3952            .push(crate::arrow_store::ArrowSheet {
3953                name: std::sync::Arc::<str>::from(name),
3954                date_system: self.config.date_system,
3955                columns: Vec::new(),
3956                nrows: 0,
3957                chunk_starts: Vec::new(),
3958                chunk_rows: 32 * 1024,
3959            });
3960    }
3961
3962    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
3963        let name = self.graph.sheet_name(sheet_id).to_string();
3964        // Removing a sheet only affects spans on that sheet and spans reading
3965        // from that sheet. Preserve spans on unrelated sheets so sheet
3966        // lifecycle operations do not collapse the whole FormulaPlane.
3967        self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
3968            .map_err(Self::editor_error_to_excel)?;
3969        self.graph.remove_sheet(sheet_id)?;
3970        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
3971        // Sheet removal can change cross-sheet refs, names, and default-sheet
3972        // resolution. Until those domains have a complete exact dependency
3973        // proof, retain the documented graph-owned global invalidation.
3974        self.clear_all_computed_overlays();
3975        self.mark_all_formula_vertices_dirty();
3976        self.clear_staged_formulas_for_sheet(&name);
3977        if self.row_visibility.remove(&sheet_id).is_some() {
3978            self.invalidate_row_visibility_mask_cache();
3979        }
3980        self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
3981        self.mark_topology_edited();
3982        Ok(())
3983    }
3984
3985    /// Helper to synchronize the Arrow-backed storage layer.
3986    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
3987        if let Some(asheet) = self
3988            .arrow_sheets
3989            .sheets
3990            .iter_mut()
3991            .find(|s| s.name.as_ref() == target_name)
3992        {
3993            asheet.name = std::sync::Arc::<str>::from(new_name);
3994            return true;
3995        }
3996        false
3997    }
3998
3999    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4000        let old_name = self.graph.sheet_name(sheet_id).to_string();
4001
4002        // Speculative Storage Update
4003        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4004        self.rename_sheet_in_arrow_store(&old_name, new_name);
4005
4006        // Graph Update (Metadata + Rescue Logic)
4007        match self.graph.rename_sheet(sheet_id, new_name) {
4008            Ok(_) => {
4009                self.rename_staged_formula_sheet(&old_name, new_name);
4010                // Success! Invalidate cache for the moved sheet
4011                let sheet_vertices: Vec<VertexId> =
4012                    self.graph.vertices_in_sheet(sheet_id).collect();
4013                for v_id in sheet_vertices {
4014                    self.graph.mark_vertex_dirty(v_id);
4015                }
4016                // Sheet rename preserves SheetId and therefore formula dependencies.
4017                self.mark_topology_edited();
4018                Ok(())
4019            }
4020            Err(e) => {
4021                // ROLLBACK: Revert storage if graph rejected the name
4022                self.rename_sheet_in_arrow_store(new_name, &old_name);
4023                Err(e)
4024            }
4025        }
4026    }
4027
4028    pub fn named_ranges_iter(
4029        &self,
4030    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4031        self.graph.named_ranges_iter()
4032    }
4033
4034    pub fn sheet_named_ranges_iter(
4035        &self,
4036    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4037        self.graph.sheet_named_ranges_iter()
4038    }
4039
4040    pub fn resolve_name_entry(
4041        &self,
4042        name: &str,
4043        current_sheet: SheetId,
4044    ) -> Option<&crate::engine::named_range::NamedRange> {
4045        self.graph.resolve_name_entry(name, current_sheet)
4046    }
4047
4048    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4049        let current_sheet = scope_sheet
4050            .and_then(|sheet| self.graph.sheet_id(sheet))
4051            .unwrap_or_else(|| self.graph.default_sheet_id());
4052        self.graph.resolve_name_entry(name, current_sheet).is_some()
4053    }
4054
4055    pub fn resolved_name_value(
4056        &self,
4057        name: &str,
4058        scope_sheet: Option<&str>,
4059    ) -> Option<LiteralValue> {
4060        let current_sheet = scope_sheet
4061            .and_then(|sheet| self.graph.sheet_id(sheet))
4062            .unwrap_or_else(|| self.graph.default_sheet_id());
4063        let entry = self.graph.resolve_name_entry(name, current_sheet)?;
4064        self.graph.get_value(entry.vertex)
4065    }
4066
4067    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4068        let entry = self.graph.resolve_table_entry(name)?;
4069        Some(TableMetadata {
4070            name: entry.name.clone(),
4071            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4072            start_row: entry.range.start.coord.row() + 1,
4073            start_col: entry.range.start.coord.col() + 1,
4074            end_row: entry.range.end.coord.row() + 1,
4075            end_col: entry.range.end.coord.col() + 1,
4076            header_row: entry.header_row,
4077            headers: entry.headers.clone(),
4078            totals_row: entry.totals_row,
4079        })
4080    }
4081
4082    /// Metadata for every defined table, ordered by name.
4083    pub fn tables(&self) -> Vec<TableMetadata> {
4084        self.graph
4085            .table_names()
4086            .into_iter()
4087            .filter_map(|name| self.table_metadata(&name))
4088            .collect()
4089    }
4090
4091    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4092        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4093
4094        for (name, named) in self.graph.named_ranges_iter() {
4095            out.push(crate::engine::named_range::NamedRangeSnapshot {
4096                name: name.clone(),
4097                scope: NameScope::Workbook,
4098                definition: named.definition.clone(),
4099            });
4100        }
4101
4102        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4103            out.push(crate::engine::named_range::NamedRangeSnapshot {
4104                name: name.clone(),
4105                scope: NameScope::Sheet(*sheet_id),
4106                definition: named.definition.clone(),
4107            });
4108        }
4109
4110        out.sort_by(|a, b| {
4111            let a_scope = match a.scope {
4112                NameScope::Workbook => (0u8, 0u32),
4113                NameScope::Sheet(id) => (1u8, u32::from(id)),
4114            };
4115            let b_scope = match b.scope {
4116                NameScope::Workbook => (0u8, 0u32),
4117                NameScope::Sheet(id) => (1u8, u32::from(id)),
4118            };
4119            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4120        });
4121
4122        out
4123    }
4124
4125    pub fn named_ranges_snapshot_for_sheet(
4126        &self,
4127        sheet_id: SheetId,
4128    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4129        self.named_ranges_snapshot()
4130            .into_iter()
4131            .filter(|entry| match entry.scope {
4132                NameScope::Workbook => true,
4133                NameScope::Sheet(id) => id == sheet_id,
4134            })
4135            .collect()
4136    }
4137
4138    pub fn define_name(
4139        &mut self,
4140        name: &str,
4141        definition: NamedDefinition,
4142        scope: NameScope,
4143    ) -> Result<(), ExcelError> {
4144        // A new define can flip resolution for spans that previously resolved
4145        // the same name through another scope (e.g. a sheet-scoped name
4146        // shadowing a workbook-scoped one). Demote those spans BEFORE the
4147        // registry changes so their cells re-ingest and re-resolve through
4148        // the normal legacy path.
4149        self.graph.validate_define_name(name, scope)?;
4150        let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4151        if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4152            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4153                "formula-name shadowing with active FormulaPlane dependents is not supported",
4154            ));
4155        }
4156        let prepared = self
4157            .prepare_name_dependent_span_demotion([name])
4158            .map_err(Self::editor_error_to_excel)?;
4159        let demoted = self
4160            .commit_name_dependent_span_demotion(prepared)
4161            .map_err(Self::editor_error_to_excel)?;
4162        self.graph.define_name(name, definition, scope)?;
4163        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4164        if !demoted {
4165            self.mark_topology_edited();
4166        }
4167        Ok(())
4168    }
4169
4170    pub fn update_name(
4171        &mut self,
4172        name: &str,
4173        definition: NamedDefinition,
4174        scope: NameScope,
4175    ) -> Result<(), ExcelError> {
4176        // Demote name-dependent spans BEFORE the registry update: the demoted
4177        // cells re-materialize as legacy vertices attached to the name vertex
4178        // (via their resolved-name dep plans), so the registry update's
4179        // dependent dirtying reaches them exactly like long-lived legacy
4180        // formulas.
4181        self.graph.validate_existing_name(name, scope)?;
4182        let prepared = self
4183            .prepare_name_dependent_span_demotion([name])
4184            .map_err(Self::editor_error_to_excel)?;
4185        let demoted = self
4186            .commit_name_dependent_span_demotion(prepared)
4187            .map_err(Self::editor_error_to_excel)?;
4188        self.graph.update_name(name, definition, scope)?;
4189        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4190        if !demoted {
4191            self.mark_topology_edited();
4192        }
4193        Ok(())
4194    }
4195
4196    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4197        // Demote first (see update_name): the demoted legacy vertices become
4198        // dependents of the name vertex, so delete_name dirties them and they
4199        // re-evaluate to #NAME? exactly as legacy formulas do.
4200        self.graph.validate_existing_name(name, scope)?;
4201        let prepared = self
4202            .prepare_name_dependent_span_demotion([name])
4203            .map_err(Self::editor_error_to_excel)?;
4204        let demoted = self
4205            .commit_name_dependent_span_demotion(prepared)
4206            .map_err(Self::editor_error_to_excel)?;
4207        self.graph.delete_name(name, scope)?;
4208        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4209        if !demoted {
4210            self.mark_topology_edited();
4211        }
4212        Ok(())
4213    }
4214
4215    fn exact_name_dependent_span_refs<'a>(
4216        &self,
4217        names: impl IntoIterator<Item = &'a str>,
4218    ) -> Vec<FormulaSpanRef> {
4219        let authority = self.graph.formula_authority();
4220        let mut refs = names
4221            .into_iter()
4222            .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4223            .collect::<Vec<_>>();
4224        refs.sort_unstable_by_key(|span_ref| {
4225            (span_ref.id.0, span_ref.generation, span_ref.version)
4226        });
4227        refs.dedup();
4228        refs
4229    }
4230
4231    fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4232        events
4233            .iter()
4234            .filter_map(|event| match event {
4235                ChangeEvent::DefineName { name, .. }
4236                | ChangeEvent::UpdateName { name, .. }
4237                | ChangeEvent::DeleteName { name, .. }
4238                | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4239                _ => None,
4240            })
4241            .collect()
4242    }
4243
4244    fn prepare_name_dependent_span_demotion<'a>(
4245        &mut self,
4246        names: impl IntoIterator<Item = &'a str>,
4247    ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4248    {
4249        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4250            return Ok(None);
4251        }
4252        let refs = self.exact_name_dependent_span_refs(names);
4253        if refs.is_empty() {
4254            return Ok(None);
4255        }
4256
4257        // Exact demotion creates dirty legacy vertices. Preserve the old
4258        // span's clean/dirty state so the following name mutation remains the
4259        // sole authority that dirties formulas whose resolution changed.
4260        let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4261        let clean_cells = {
4262            let authority = self.graph.formula_authority();
4263            refs.iter()
4264                .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4265                .flat_map(|span| span.domain.iter())
4266                .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4267                .map(|coord| {
4268                    CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4269                })
4270                .collect::<Vec<_>>()
4271        };
4272        self.prepare_formula_span_demotion(&refs)
4273            .map(|prepared| Some((prepared, clean_cells)))
4274            .map_err(|error| match error {
4275                FormulaSpanDemotionError::Resource(error) => error.into(),
4276                error => crate::engine::EditorError::TransactionFailed {
4277                    reason: format!(
4278                        "FormulaPlane name-dependent demotion preparation failed: {error}"
4279                    ),
4280                },
4281            })
4282    }
4283
4284    fn commit_name_dependent_span_demotion(
4285        &mut self,
4286        prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4287    ) -> Result<bool, crate::engine::EditorError> {
4288        let Some((prepared, clean_cells)) = prepared else {
4289            return Ok(false);
4290        };
4291        self.commit_prepared_formula_span_demotion(prepared)
4292            .map_err(|error| crate::engine::EditorError::TransactionFailed {
4293                reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4294            })?;
4295        for cell in clean_cells {
4296            if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4297                self.graph.set_dirty(vertex_id, false);
4298            }
4299        }
4300        Ok(true)
4301    }
4302
4303    pub fn define_table(
4304        &mut self,
4305        name: &str,
4306        range: crate::reference::RangeRef,
4307        header_row: bool,
4308        headers: Vec<String>,
4309        totals_row: bool,
4310    ) -> Result<(), ExcelError> {
4311        self.graph
4312            .define_table(name, range, header_row, headers, totals_row)?;
4313        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4314        self.mark_topology_edited();
4315        Ok(())
4316    }
4317
4318    pub fn define_source_scalar(
4319        &mut self,
4320        name: &str,
4321        version: Option<u64>,
4322    ) -> Result<(), ExcelError> {
4323        self.graph.define_source_scalar(name, version)?;
4324        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4325        self.mark_topology_edited();
4326        Ok(())
4327    }
4328
4329    pub fn define_source_table(
4330        &mut self,
4331        name: &str,
4332        version: Option<u64>,
4333    ) -> Result<(), ExcelError> {
4334        self.graph.define_source_table(name, version)?;
4335        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4336        self.mark_topology_edited();
4337        Ok(())
4338    }
4339
4340    pub fn set_source_scalar_version(
4341        &mut self,
4342        name: &str,
4343        version: Option<u64>,
4344    ) -> Result<(), ExcelError> {
4345        self.graph.set_source_scalar_version(name, version)?;
4346        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4347            self.graph
4348                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4349        }
4350        Ok(())
4351    }
4352
4353    pub fn set_source_table_version(
4354        &mut self,
4355        name: &str,
4356        version: Option<u64>,
4357    ) -> Result<(), ExcelError> {
4358        self.graph.set_source_table_version(name, version)?;
4359        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4360            self.graph
4361                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4362        }
4363        Ok(())
4364    }
4365
4366    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4367        self.graph.invalidate_source(name)?;
4368        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4369            self.graph
4370                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4371        }
4372        Ok(())
4373    }
4374
4375    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4376        self.graph.get_value(vertex)
4377    }
4378
4379    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4380        self.graph.get_cell_value(sheet, row, col)
4381    }
4382
4383    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4384        self.graph.get_vertex_for_cell(cell)
4385    }
4386
4387    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4388        self.graph.get_evaluation_vertices()
4389    }
4390
4391    /// Return read-only baseline counters for FormulaPlane/dispatch benchmarking.
4392    pub fn baseline_stats(&self) -> EngineBaselineStats {
4393        let graph = self.graph.baseline_stats();
4394        let formula_authority = self.graph.formula_authority();
4395        let formula_dirty = self.graph.formula_dirty_stats();
4396        EngineBaselineStats {
4397            graph_vertex_count: graph.graph_vertex_count,
4398            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4399            graph_edge_count: graph.graph_edge_count,
4400            dirty_vertex_count: graph.dirty_vertex_count,
4401            evaluation_vertex_count: graph.evaluation_vertex_count,
4402            formula_ast_root_count: graph.formula_ast_root_count,
4403            formula_ast_node_count: graph.formula_ast_node_count,
4404            staged_formula_count: self.staged_formula_count(),
4405            formula_plane_active_span_count: formula_authority.active_span_count(),
4406            formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4407            formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4408            formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4409            formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4410            formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4411            formula_plane_dirty_pending_events: formula_dirty.pending_events,
4412            formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4413            formula_plane_dirty_span_region_events_recorded: formula_dirty
4414                .span_region_events_recorded,
4415            formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4416            formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4417            formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4418            formula_plane_cycle_member_span_demotions: self
4419                .formula_plane_cycle_member_span_demotions,
4420        }
4421    }
4422
4423    /// Mutation revision captured by read-only engine reports.
4424    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4425        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4426    }
4427
4428    #[cfg(test)]
4429    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4430        self.used_axis_bounds_cache
4431            .read()
4432            .ok()
4433            .and_then(|guard| {
4434                guard.as_ref().map(|cache| {
4435                    (
4436                        cache.row_hits.load(Ordering::Relaxed),
4437                        cache.row_misses.load(Ordering::Relaxed),
4438                        cache.col_hits.load(Ordering::Relaxed),
4439                        cache.col_misses.load(Ordering::Relaxed),
4440                    )
4441                })
4442            })
4443            .unwrap_or((0, 0, 0, 0))
4444    }
4445
4446    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4447        self.graph.set_first_load_assume_new(enabled);
4448    }
4449
4450    pub fn first_load_assume_new(&self) -> bool {
4451        self.graph.first_load_assume_new()
4452    }
4453
4454    pub fn reset_ensure_touched(&mut self) {
4455        self.graph.reset_ensure_touched();
4456    }
4457
4458    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4459        self.graph.finalize_sheet_index(sheet);
4460    }
4461
4462    /// Execute a named Engine action.
4463    ///
4464    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4465    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4466    /// provide rollback/atomicity.
4467    ///
4468    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4469    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
4470    pub fn action<T>(
4471        &mut self,
4472        name: impl AsRef<str>,
4473        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4474    ) -> Result<T, crate::engine::EditorError> {
4475        if self.action_depth != 0 {
4476            return Err(crate::engine::EditorError::TransactionFailed {
4477                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4478                    .to_string(),
4479            });
4480        }
4481
4482        self.action_depth = 1;
4483        let engine_ptr: *mut Engine<R> = self;
4484        let _guard = ActionDepthGuard {
4485            engine: engine_ptr,
4486            _marker: std::marker::PhantomData,
4487        };
4488
4489        let mut tx = EngineAction {
4490            engine: self,
4491            name: name.as_ref().to_string(),
4492            log: None,
4493            arrow_undo: None,
4494            atomic_policy: false,
4495        };
4496        f(&mut tx)
4497    }
4498
4499    /// Execute a named Engine action with atomic commit/rollback semantics.
4500    ///
4501    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
4502    pub fn action_atomic<T>(
4503        &mut self,
4504        name: impl Into<String>,
4505        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4506    ) -> Result<T, crate::engine::EditorError> {
4507        let (v, _j) = self.action_atomic_journal(name, f)?;
4508        Ok(v)
4509    }
4510
4511    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
4512    pub fn action_atomic_journal<T>(
4513        &mut self,
4514        name: impl Into<String>,
4515        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4516    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4517        if self.action_depth != 0 {
4518            return Err(crate::engine::EditorError::TransactionFailed {
4519                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4520                    .to_string(),
4521            });
4522        }
4523
4524        self.action_depth = 1;
4525        let engine_ptr: *mut Engine<R> = self;
4526        let _guard = ActionDepthGuard {
4527            engine: engine_ptr,
4528            _marker: std::marker::PhantomData,
4529        };
4530
4531        let name_str = name.into();
4532        let mut log = crate::engine::ChangeLog::new();
4533        let start_len = log.len();
4534        self.action_atomic_impl(&mut log, start_len, name_str, f)
4535    }
4536
4537    fn action_atomic_impl<T>(
4538        &mut self,
4539        log: &mut crate::engine::ChangeLog,
4540        start_len: usize,
4541        name: String,
4542        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4543    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4544        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4545        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4546
4547        let log_ptr: *mut crate::engine::ChangeLog = log;
4548        let mut tx = EngineAction {
4549            engine: self,
4550            name: name.clone(),
4551            log: Some(log_ptr),
4552            arrow_undo: Some(arrow_ptr),
4553            atomic_policy: true,
4554        };
4555
4556        let res = f(&mut tx);
4557
4558        // Capture graph structural delta for this action.
4559        let graph_events: Vec<crate::engine::ChangeEvent> =
4560            unsafe { (&*log_ptr).events() }[start_len..].to_vec();
4561        let graph_batch = crate::engine::GraphUndoBatch {
4562            events: graph_events,
4563        };
4564        let affected_cells = arrow_undo.ops.len();
4565        let journal = crate::engine::ActionJournal {
4566            name,
4567            graph: graph_batch,
4568            arrow: arrow_undo,
4569            affected_cells,
4570        };
4571
4572        match res {
4573            Ok(v) => {
4574                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4575                    for event in &journal.graph.events {
4576                        self.record_formula_plane_change_for_event(event);
4577                    }
4578                    self.mark_data_edited();
4579                }
4580                Ok((v, journal))
4581            }
4582            Err(e) => {
4583                if let Err(rb) = self.rollback_from_action_journal(&journal) {
4584                    return Err(crate::engine::EditorError::TransactionFailed {
4585                        reason: format!(
4586                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
4587                        ),
4588                    });
4589                }
4590                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4591                    for event in &journal.graph.events {
4592                        self.record_formula_plane_change_for_event(event);
4593                    }
4594                }
4595                Err(e)
4596            }
4597        }
4598    }
4599
4600    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
4601    ///
4602    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
4603    /// - Dependency graph structural edits (via inverse ChangeEvents)
4604    /// - Arrow-truth overlay writes mirrored from ChangeEvents
4605    /// - ChangeLog entries (truncated back to the pre-action length)
4606    pub fn action_with_logger<T>(
4607        &mut self,
4608        log: &mut crate::engine::ChangeLog,
4609        name: impl AsRef<str>,
4610        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4611    ) -> Result<T, crate::engine::EditorError> {
4612        if self.action_depth != 0 {
4613            return Err(crate::engine::EditorError::TransactionFailed {
4614                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4615                    .to_string(),
4616            });
4617        }
4618
4619        self.action_depth = 1;
4620        let engine_ptr: *mut Engine<R> = self;
4621        let _guard = ActionDepthGuard {
4622            engine: engine_ptr,
4623            _marker: std::marker::PhantomData,
4624        };
4625
4626        let start_len = log.len();
4627        let name_str = name.as_ref().to_string();
4628        log.begin_compound(name_str.clone());
4629
4630        // Use the provided ChangeLog as an observability sink.
4631        // Correctness is provided by the internal `ActionJournal` returned from the atomic impl.
4632        let res = self.action_atomic_impl(log, start_len, name_str, f);
4633
4634        match res {
4635            Ok((v, _journal)) => {
4636                log.end_compound();
4637                Ok(v)
4638            }
4639            Err(e) => {
4640                // Close compound and truncate log as cleanup only.
4641                log.end_compound();
4642                log.truncate(start_len);
4643                Err(e)
4644            }
4645        }
4646    }
4647
4648    fn rollback_from_action_journal(
4649        &mut self,
4650        journal: &crate::engine::ActionJournal,
4651    ) -> Result<(), crate::engine::EditorError> {
4652        // 1) Roll back the dependency graph structure.
4653        journal.graph.undo(&mut self.graph)?;
4654        // 2) Roll back engine row-visibility sidecar events.
4655        self.apply_inverse_row_visibility_events(&journal.graph.events);
4656        // 3) Roll back Arrow-truth overlays.
4657        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
4658        Ok(())
4659    }
4660
4661    fn rollback_from_change_events(
4662        &mut self,
4663        events: &[crate::engine::ChangeEvent],
4664    ) -> Result<(), crate::engine::EditorError> {
4665        use crate::engine::ChangeEvent;
4666
4667        // 1) Roll back the dependency graph.
4668        {
4669            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4670            let mut compound_stack: Vec<usize> = Vec::new();
4671            for ev in events.iter().rev() {
4672                match ev {
4673                    ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
4674                    ChangeEvent::CompoundStart { depth, .. } => {
4675                        if compound_stack.last() == Some(depth) {
4676                            compound_stack.pop();
4677                        }
4678                    }
4679                    ChangeEvent::SetRowVisibility { .. } => {
4680                        // Engine-side metadata handled after dropping graph editor borrow.
4681                    }
4682                    _ => {
4683                        editor.apply_inverse(ev.clone())?;
4684                    }
4685                }
4686            }
4687        }
4688
4689        // 2) Roll back engine row-visibility metadata.
4690        for ev in events.iter().rev() {
4691            self.apply_inverse_row_visibility_event(ev);
4692        }
4693
4694        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
4695        for ev in events.iter().rev() {
4696            self.mirror_inverse_change_to_arrow(ev);
4697        }
4698
4699        Ok(())
4700    }
4701
4702    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
4703        let sheet_id = self.graph.sheet_id(sheet)?;
4704        let coord = Coord::from_excel(row, col, true, true);
4705        let cell = CellRef::new(sheet_id, coord);
4706        let vid = self.graph.get_vertex_for_cell(&cell)?;
4707        let ast_id = self.graph.get_formula_id(vid)?;
4708        self.graph
4709            .data_store()
4710            .retrieve_ast(ast_id, self.graph.sheet_reg())
4711    }
4712
4713    pub fn define_name_with_logger(
4714        &mut self,
4715        log: &mut crate::engine::ChangeLog,
4716        name: &str,
4717        definition: NamedDefinition,
4718        scope: NameScope,
4719    ) -> Result<(), crate::engine::EditorError> {
4720        self.graph
4721            .validate_define_name(name, scope)
4722            .map_err(crate::engine::EditorError::Excel)?;
4723        let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4724        if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4725            return Err(crate::engine::EditorError::TransactionUnsupported {
4726                reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
4727                    .to_string(),
4728            });
4729        }
4730        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4731        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4732        {
4733            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4734            editor.define_name(name, definition, scope)?;
4735        }
4736        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4737        if !demoted {
4738            self.mark_topology_edited();
4739        }
4740        Ok(())
4741    }
4742
4743    pub fn update_name_with_logger(
4744        &mut self,
4745        log: &mut crate::engine::ChangeLog,
4746        name: &str,
4747        definition: NamedDefinition,
4748        scope: NameScope,
4749    ) -> Result<(), crate::engine::EditorError> {
4750        self.graph
4751            .validate_existing_name(name, scope)
4752            .map_err(crate::engine::EditorError::Excel)?;
4753        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4754        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4755        {
4756            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4757            editor.update_name(name, definition, scope)?;
4758        }
4759        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4760        if !demoted {
4761            self.mark_topology_edited();
4762        }
4763        Ok(())
4764    }
4765
4766    pub fn delete_name_with_logger(
4767        &mut self,
4768        log: &mut crate::engine::ChangeLog,
4769        name: &str,
4770        scope: NameScope,
4771    ) -> Result<(), crate::engine::EditorError> {
4772        self.graph
4773            .validate_existing_name(name, scope)
4774            .map_err(crate::engine::EditorError::Excel)?;
4775        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4776        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4777        {
4778            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4779            editor.delete_name(name, scope)?;
4780        }
4781        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4782        if !demoted {
4783            self.mark_topology_edited();
4784        }
4785        Ok(())
4786    }
4787
4788    pub fn edit_with_logger<T>(
4789        &mut self,
4790        log: &mut crate::engine::ChangeLog,
4791        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
4792    ) -> Result<T, crate::engine::EditorError> {
4793        // Record starting log length so we can mirror only newly-recorded events.
4794        let start_len = log.len();
4795
4796        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
4797        // (graph value cache is intentionally empty in canonical mode).
4798        struct ArrowSpillReader<'a> {
4799            sheets: &'a crate::arrow_store::SheetStore,
4800        }
4801        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
4802            fn read_cell_value(
4803                &self,
4804                sheet: &str,
4805                row: u32,
4806                col: u32,
4807            ) -> Option<formualizer_common::LiteralValue> {
4808                use formualizer_common::LiteralValue;
4809                let asheet = self.sheets.sheet(sheet)?;
4810                let r0 = row.saturating_sub(1) as usize;
4811                let c0 = col.saturating_sub(1) as usize;
4812                let v = asheet.get_cell_value(r0, c0);
4813                if matches!(v, LiteralValue::Empty) {
4814                    None
4815                } else {
4816                    Some(v)
4817                }
4818            }
4819        }
4820
4821        let ret = {
4822            let spill_reader = ArrowSpillReader {
4823                sheets: &self.arrow_sheets,
4824            };
4825            let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
4826                &mut self.graph,
4827                log,
4828                &spill_reader,
4829            );
4830            f(&mut editor)
4831        };
4832
4833        let new_events = log.events()[start_len..].to_vec();
4834        if new_events.iter().any(|event| {
4835            matches!(
4836                event,
4837                ChangeEvent::DefineName { .. }
4838                    | ChangeEvent::UpdateName { .. }
4839                    | ChangeEvent::DeleteName { .. }
4840            )
4841        }) {
4842            self.rollback_from_change_events(&new_events)?;
4843            log.truncate(start_len);
4844            return Err(crate::engine::EditorError::TransactionUnsupported {
4845                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
4846            });
4847        }
4848
4849        // Mirror value-impacting graph events to Arrow for forward edits.
4850        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
4851        for ev in &new_events {
4852            self.mirror_forward_change_to_arrow(ev);
4853        }
4854        for ev in &new_events {
4855            self.record_formula_plane_change_for_event(ev);
4856        }
4857
4858        Ok(ret)
4859    }
4860
4861    pub(crate) fn preflight_replay_admission(
4862        &mut self,
4863        events: &[ChangeEvent],
4864        forward: bool,
4865    ) -> Result<(), crate::engine::EditorError> {
4866        if !self.graph_admission_enabled() {
4867            return Ok(());
4868        }
4869        let mut vertex_delta = 0i128;
4870        let mut edge_delta = 0i128;
4871        let mut added_vertices = 0usize;
4872        let mut added_edges = 0usize;
4873        let mut formula_cells = BTreeSet::new();
4874        for event in events {
4875            match event {
4876                ChangeEvent::AddVertex {
4877                    formula,
4878                    coord,
4879                    sheet_id,
4880                    ..
4881                } => {
4882                    let delta = if forward { 1 } else { -1 };
4883                    vertex_delta += delta;
4884                    if forward {
4885                        added_vertices = added_vertices.saturating_add(1);
4886                        if formula.is_some() {
4887                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4888                        }
4889                    }
4890                }
4891                ChangeEvent::RemoveVertex {
4892                    old_formula,
4893                    coord,
4894                    sheet_id,
4895                    ..
4896                } => {
4897                    let delta = if forward { -1 } else { 1 };
4898                    vertex_delta += delta;
4899                    if !forward {
4900                        added_vertices = added_vertices.saturating_add(1);
4901                        if old_formula.is_some()
4902                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
4903                        {
4904                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4905                        }
4906                    }
4907                }
4908                ChangeEvent::EdgeAdded { .. } => {
4909                    let delta = if forward { 1 } else { -1 };
4910                    edge_delta += delta;
4911                    if forward {
4912                        added_edges = added_edges.saturating_add(1);
4913                    }
4914                }
4915                ChangeEvent::EdgeRemoved { .. } => {
4916                    let delta = if forward { -1 } else { 1 };
4917                    edge_delta += delta;
4918                    if !forward {
4919                        added_edges = added_edges.saturating_add(1);
4920                    }
4921                }
4922                ChangeEvent::SetFormula {
4923                    addr, old_formula, ..
4924                } => {
4925                    if forward || old_formula.is_some() {
4926                        formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
4927                    }
4928                }
4929                _ => {}
4930            }
4931        }
4932        let stats = self.graph.baseline_stats();
4933        let final_vertices = i128::try_from(stats.graph_vertex_count)
4934            .ok()
4935            .and_then(|count| count.checked_add(vertex_delta))
4936            .and_then(|count| usize::try_from(count).ok())
4937            .ok_or_else(|| {
4938                crate::engine::EditorError::Excel(
4939                    ExcelError::new(ExcelErrorKind::NImpl)
4940                        .with_message("replay vertex count overflow"),
4941                )
4942            })?;
4943        let final_edges = i128::try_from(stats.graph_edge_count)
4944            .ok()
4945            .and_then(|count| count.checked_add(edge_delta))
4946            .and_then(|count| usize::try_from(count).ok())
4947            .ok_or_else(|| {
4948                crate::engine::EditorError::Excel(
4949                    ExcelError::new(ExcelErrorKind::NImpl)
4950                        .with_message("replay edge count overflow"),
4951                )
4952            })?;
4953        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
4954            final_vertices,
4955            final_edges,
4956            materialization_cells: formula_cells.len() as u64,
4957            added_vertices,
4958            added_edges,
4959        })
4960        .map_err(crate::engine::EditorError::Excel)
4961    }
4962
4963    pub fn undo_logged(
4964        &mut self,
4965        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
4966        log: &mut crate::engine::ChangeLog,
4967    ) -> Result<(), crate::engine::EditorError> {
4968        let pending_events = log
4969            .last_group_indices()
4970            .into_iter()
4971            .map(|index| log.events()[index].clone())
4972            .collect::<Vec<_>>();
4973        self.preflight_replay_admission(&pending_events, false)?;
4974        let names = Self::name_event_names(&pending_events);
4975        let prepared =
4976            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
4977        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4978        let batch = undo.undo(&mut self.graph, log)?;
4979        for item in batch.iter().rev() {
4980            self.apply_inverse_row_visibility_event(&item.event);
4981            self.apply_inverse_staged_formula_event(&item.event);
4982        }
4983        self.mirror_undo_batch_to_arrow(&batch);
4984        if !batch.is_empty() {
4985            for item in &batch {
4986                self.record_formula_plane_change_for_event(&item.event);
4987            }
4988            if !demoted && !names.is_empty() {
4989                self.mark_topology_edited();
4990            }
4991        }
4992        Ok(())
4993    }
4994
4995    pub fn redo_logged(
4996        &mut self,
4997        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
4998        log: &mut crate::engine::ChangeLog,
4999    ) -> Result<(), crate::engine::EditorError> {
5000        let pending_events = undo.pending_redo_events();
5001        self.preflight_replay_admission(&pending_events, true)?;
5002        let names = Self::name_event_names(&pending_events);
5003        let prepared =
5004            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5005        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5006        let batch = undo.redo(&mut self.graph, log)?;
5007        for item in &batch {
5008            self.apply_forward_row_visibility_event(&item.event);
5009            self.apply_forward_staged_formula_event(&item.event);
5010        }
5011        self.mirror_redo_batch_to_arrow(&batch);
5012        if !batch.is_empty() {
5013            for item in &batch {
5014                self.record_formula_plane_change_for_event(&item.event);
5015            }
5016            if !demoted && !names.is_empty() {
5017                self.mark_topology_edited();
5018            }
5019        }
5020        Ok(())
5021    }
5022
5023    /// Undo the last committed atomic action using the journal stack.
5024    ///
5025    /// This path does not require a `ChangeLog`.
5026    pub fn undo_action(
5027        &mut self,
5028        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5029    ) -> Result<(), crate::engine::EditorError> {
5030        let Some(journal) = undo.pop_undo_action() else {
5031            return Ok(());
5032        };
5033        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5034            undo.push_done_action(journal);
5035            return Err(error);
5036        }
5037        let names = Self::name_event_names(&journal.graph.events);
5038        let prepared =
5039            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5040                Ok(prepared) => prepared,
5041                Err(error) => {
5042                    undo.push_done_action(journal);
5043                    return Err(error);
5044                }
5045            };
5046        let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5047            Ok(demoted) => demoted,
5048            Err(error) => {
5049                undo.push_done_action(journal);
5050                return Err(error);
5051            }
5052        };
5053
5054        journal.graph.undo(&mut self.graph)?;
5055        self.apply_inverse_row_visibility_events(&journal.graph.events);
5056        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5057        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5058            for event in &journal.graph.events {
5059                self.record_formula_plane_change_for_event(event);
5060            }
5061            if !demoted && !names.is_empty() {
5062                self.mark_topology_edited();
5063            } else {
5064                self.mark_data_edited();
5065            }
5066        }
5067
5068        undo.push_redo_action(journal);
5069        Ok(())
5070    }
5071
5072    /// Redo the last undone atomic action using the journal stack.
5073    ///
5074    /// This path does not require a `ChangeLog`.
5075    pub fn redo_action(
5076        &mut self,
5077        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5078    ) -> Result<(), crate::engine::EditorError> {
5079        let Some(journal) = undo.pop_redo_action() else {
5080            return Ok(());
5081        };
5082        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5083            undo.push_redo_action(journal);
5084            return Err(error);
5085        }
5086        let names = Self::name_event_names(&journal.graph.events);
5087        let prepared =
5088            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5089                Ok(prepared) => prepared,
5090                Err(error) => {
5091                    undo.push_redo_action(journal);
5092                    return Err(error);
5093                }
5094            };
5095        let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5096            Ok(demoted) => demoted,
5097            Err(error) => {
5098                undo.push_redo_action(journal);
5099                return Err(error);
5100            }
5101        };
5102
5103        journal.graph.redo(&mut self.graph)?;
5104        self.apply_forward_row_visibility_events(&journal.graph.events);
5105        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5106        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5107            for event in &journal.graph.events {
5108                self.record_formula_plane_change_for_event(event);
5109            }
5110            if !demoted && !names.is_empty() {
5111                self.mark_topology_edited();
5112            } else {
5113                self.mark_data_edited();
5114            }
5115        }
5116
5117        undo.push_done_action(journal);
5118        Ok(())
5119    }
5120
5121    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5122        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5123        // Coord stores 0-based indices.
5124        let row = addr.coord.row() + 1;
5125        let col = addr.coord.col() + 1;
5126        (sheet, row, col)
5127    }
5128
5129    fn mirror_undo_batch_to_arrow(
5130        &mut self,
5131        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5132    ) {
5133        // Undo applies inverses in reverse order.
5134        for item in batch.iter().rev() {
5135            self.mirror_inverse_change_to_arrow(&item.event);
5136        }
5137    }
5138
5139    fn mirror_redo_batch_to_arrow(
5140        &mut self,
5141        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5142    ) {
5143        // Redo applies events in forward order.
5144        for item in batch.iter() {
5145            self.mirror_forward_change_to_arrow(&item.event);
5146        }
5147    }
5148
5149    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5150        use crate::engine::ChangeEvent;
5151        use formualizer_common::LiteralValue;
5152
5153        match ev {
5154            ChangeEvent::SetValue {
5155                addr,
5156                old_value,
5157                old_formula,
5158                ..
5159            } => {
5160                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5161                if old_formula.is_some() {
5162                    self.clear_delta_overlay_cell(&sheet, row, col);
5163                } else {
5164                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5165                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5166                }
5167            }
5168            ChangeEvent::SetFormula {
5169                addr,
5170                old_value,
5171                old_formula,
5172                ..
5173            } => {
5174                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5175                if old_formula.is_some() {
5176                    self.clear_delta_overlay_cell(&sheet, row, col);
5177                } else {
5178                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5179                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5180                }
5181            }
5182            ChangeEvent::SpillCommitted { old, new, .. } => {
5183                // Inverse: restore `old` (or clear if none).
5184                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5185                if let Some(snap) = old {
5186                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5187                }
5188            }
5189            ChangeEvent::SpillCleared { old, .. } => {
5190                // Inverse: restore prior spill.
5191                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5192            }
5193            ChangeEvent::SetRowVisibility { .. } => {
5194                // Engine-side metadata only; no Arrow overlay effect.
5195            }
5196            _ => {}
5197        }
5198    }
5199
5200    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5201        use crate::engine::ChangeEvent;
5202
5203        match ev {
5204            ChangeEvent::SetValue { addr, new, .. } => {
5205                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5206                self.mirror_value_to_overlay(&sheet, row, col, new);
5207            }
5208            ChangeEvent::SetFormula { addr, .. } => {
5209                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5210                self.clear_delta_overlay_cell(&sheet, row, col);
5211                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5212            }
5213            ChangeEvent::SpillCommitted { old, new, .. } => {
5214                if let Some(snap) = old {
5215                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5216                }
5217                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5218            }
5219            ChangeEvent::SpillCleared { old, .. } => {
5220                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5221            }
5222            ChangeEvent::SetRowVisibility { .. } => {
5223                // Engine-side metadata only; no Arrow overlay effect.
5224            }
5225            _ => {
5226                // Other graph structural operations do not have direct value effects in Arrow.
5227            }
5228        }
5229    }
5230
5231    fn mirror_spill_snapshot(
5232        &mut self,
5233        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5234        clear_only: bool,
5235    ) {
5236        use formualizer_common::LiteralValue;
5237
5238        let mut i = 0usize;
5239        for row in &snap.values {
5240            for v in row {
5241                if let Some(cell) = snap.target_cells.get(i) {
5242                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5243                    let out = if clear_only {
5244                        LiteralValue::Empty
5245                    } else {
5246                        v.clone()
5247                    };
5248                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5249                }
5250                i += 1;
5251            }
5252        }
5253        // If target_cells is longer than values (should not happen), clear remaining cells.
5254        if clear_only {
5255            for cell in snap.target_cells.iter().skip(i) {
5256                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5257                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5258            }
5259        }
5260    }
5261
5262    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5263        self.graph.set_default_sheet_by_name(name);
5264    }
5265
5266    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5267        self.graph.set_default_sheet_by_id(id);
5268    }
5269
5270    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5271        self.graph.set_sheet_index_mode(mode);
5272    }
5273
5274    fn clear_cached_static_schedule(&mut self) {
5275        self.cached_static_schedule = None;
5276    }
5277
5278    /// Mark data edited: bump snapshot and set edited flag.
5279    /// Value-only edits keep the stable-topology schedule cache alive.
5280    pub fn mark_data_edited(&mut self) {
5281        self.snapshot_id
5282            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5283        self.has_edited = true;
5284    }
5285
5286    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
5287    pub fn mark_topology_edited(&mut self) {
5288        self.snapshot_id
5289            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5290        self.topology_epoch = self.topology_epoch.wrapping_add(1);
5291        self.graph.bump_topology_revision();
5292        self.clear_cached_static_schedule();
5293        self.cached_mixed_topology = None;
5294        if let Some(ledger) = self.active_resource_ledger.as_mut() {
5295            let released = ledger.account_mixed_cache(0);
5296            debug_assert!(released.is_ok());
5297        }
5298        self.has_edited = true;
5299    }
5300
5301    fn mark_all_formula_vertices_dirty(&mut self) {
5302        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5303        for vertex in vertices {
5304            self.graph.mark_vertex_dirty(vertex);
5305        }
5306    }
5307
5308    fn mark_moved_formula_vertices_dirty(
5309        &mut self,
5310        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5311    ) {
5312        for vertex in &summary.vertices_moved {
5313            if self.graph.get_formula_id(*vertex).is_some() {
5314                self.graph.mark_vertex_dirty(*vertex);
5315            }
5316        }
5317    }
5318
5319    /// Access Arrow sheet store (read-only)
5320    pub fn sheet_store(&self) -> &SheetStore {
5321        &self.arrow_sheets
5322    }
5323
5324    /// Access Arrow sheet store (mutable)
5325    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5326        &mut self.arrow_sheets
5327    }
5328
5329    pub fn has_staged_formulas(&self) -> bool {
5330        !self.staged_formulas.is_empty()
5331    }
5332
5333    pub fn staged_formula_count(&self) -> usize {
5334        self.staged_formulas.values().map(StagedSheet::len).sum()
5335    }
5336
5337    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
5338    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5339        self.staged_formulas
5340            .entry(sheet.to_string())
5341            .or_default()
5342            .stage(row, col, text);
5343        self.staged_formula_index.stage(sheet, row, col);
5344    }
5345
5346    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5347        for (row, col, _) in &staged.entries {
5348            self.staged_formula_index.remove(sheet, *row, *col);
5349        }
5350        if staged.deferred_package.is_some() {
5351            self.staged_formula_index.set_package(sheet, None);
5352        }
5353    }
5354
5355    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5356        self.staged_formulas.insert(sheet, staged);
5357    }
5358
5359    #[doc(hidden)]
5360    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5361        SourceFormulaIngress { engine: self }
5362    }
5363
5364    #[doc(hidden)]
5365    /// Test-only fault-injection seam. Not part of the supported API; it exists so
5366    /// integration tests in sibling crates can fail a commit at an exact point.
5367    #[doc(hidden)]
5368    #[cfg(any(test, feature = "test-support"))]
5369    pub fn set_before_prepared_span_commit_hook(
5370        &mut self,
5371        hook: impl FnOnce() + Send + Sync + 'static,
5372    ) {
5373        self.before_prepared_span_commit_hook = Some(Box::new(hook));
5374    }
5375
5376    #[doc(hidden)]
5377    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
5378    #[cfg(test)]
5379    pub(crate) fn set_before_target_preparation_commit_hook(
5380        &mut self,
5381        hook: impl FnOnce() + Send + Sync + 'static,
5382    ) {
5383        self.before_target_preparation_commit_hook = Some(Box::new(hook));
5384    }
5385
5386    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5387        let sheet = package.sheet_name.clone();
5388        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5389        debug_assert!(staged.deferred_package.is_none());
5390        staged.deferred_package = Some(package);
5391        staged.reconcile_attached_deferred_package();
5392        self.staged_formula_index
5393            .set_package(&sheet, staged.deferred_package.as_ref());
5394    }
5395
5396    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
5397        let mut removed = None;
5398        let mut remove_sheet = false;
5399        let mut had_package = false;
5400        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
5401            had_package = entries.deferred_package.is_some();
5402            removed = entries.remove(row, col);
5403            remove_sheet = entries.is_empty();
5404        }
5405        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
5406        if !ordinary_removed && had_package {
5407            self.staged_formula_index.touch_package(sheet);
5408        }
5409        if remove_sheet {
5410            self.staged_formulas.remove(sheet);
5411            self.staged_formula_index.set_package(sheet, None);
5412        }
5413        removed
5414    }
5415
5416    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
5417        if self.staged_formulas.remove(sheet).is_some() {
5418            self.staged_formula_index.clear_sheet(sheet);
5419        }
5420    }
5421
5422    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
5423        let Some(entries) = self.staged_formulas.remove(old) else {
5424            return;
5425        };
5426        self.staged_formula_index.clear_sheet(old);
5427        let (formulas, mut package) = entries.into_parts();
5428        for (row, col, text) in formulas {
5429            self.stage_formula_text(new, row, col, text);
5430        }
5431        if let Some(package) = package.as_mut() {
5432            package.sheet_name = new.to_string();
5433        }
5434        if let Some(package) = package {
5435            self.stage_deferred_formula_package(package);
5436        }
5437    }
5438
5439    /// Get a staged formula text for a given cell if present (cloned).
5440    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
5441        self.staged_formulas
5442            .get(sheet)
5443            .and_then(|v| v.get(row, col))
5444    }
5445
5446    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
5447        &self.formula_parse_diagnostics
5448    }
5449
5450    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
5451        std::mem::take(&mut self.formula_parse_diagnostics)
5452    }
5453
5454    pub fn clear_formula_parse_diagnostics(&mut self) {
5455        self.formula_parse_diagnostics.clear();
5456    }
5457
5458    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
5459        self.last_formula_ingest_report.as_ref()
5460    }
5461
5462    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
5463        &self.formula_ingest_report_total
5464    }
5465
5466    #[cfg(test)]
5467    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
5468        &mut self,
5469        hook: impl FnOnce() + Send + Sync + 'static,
5470    ) {
5471        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
5472    }
5473
5474    #[cfg(test)]
5475    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
5476        self.inject_target_semantic_stale_once_for_test = true;
5477    }
5478
5479    #[cfg(test)]
5480    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
5481        self.force_virtual_dep_changes_remaining_for_test = rounds;
5482    }
5483
5484    #[cfg(test)]
5485    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
5486        self.fail_evaluation_commit_preflight_once_for_test = true;
5487    }
5488
5489    #[cfg(test)]
5490    pub(crate) fn set_target_preparation_fault_for_test(
5491        &mut self,
5492        fault: crate::engine::target_preparation::TargetPreparationFault,
5493    ) {
5494        self.target_preparation_fault_for_test = Some(fault);
5495    }
5496
5497    #[cfg(test)]
5498    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
5499        self.staged_formula_index.revision()
5500    }
5501
5502    #[cfg(test)]
5503    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
5504        let ordinary_storage = self
5505            .staged_formulas
5506            .values()
5507            .map(|sheet| sheet.entries.len())
5508            .sum::<usize>();
5509        let package_storage = self
5510            .staged_formulas
5511            .values()
5512            .filter(|sheet| sheet.deferred_package.is_some())
5513            .count();
5514        ordinary_storage == self.staged_formula_index.ordinary_count()
5515            && package_storage == self.staged_formula_index.package_count()
5516            && self.staged_formulas.iter().all(|(name, sheet)| {
5517                sheet.entries.iter().all(|(row, col, _)| {
5518                    let leases = self
5519                        .staged_formula_index
5520                        .leases_in_region(name, *row, *col, *row, *col);
5521                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
5522                })
5523            })
5524    }
5525
5526    #[cfg(test)]
5527    pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
5528        self.last_formula_plane_span_eval_report.as_ref()
5529    }
5530
5531    #[cfg(test)]
5532    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
5533        self.evaluation_request_begin_count_for_test
5534    }
5535
5536    #[cfg(test)]
5537    pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
5538        self.force_source_family_fallback = force;
5539    }
5540
5541    #[cfg(test)]
5542    pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
5543        self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
5544    }
5545
5546    #[cfg(test)]
5547    pub(crate) fn set_fragmented_commit_fault_for_test(
5548        &mut self,
5549        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5550    ) {
5551        self.fragmented_commit_fault_for_test = Some(fault);
5552    }
5553
5554    #[cfg(test)]
5555    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
5556        &mut self,
5557        hook: impl FnOnce() + Send + Sync + 'static,
5558    ) {
5559        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
5560    }
5561
5562    #[cfg(test)]
5563    pub(crate) fn set_after_eager_proposal_commit_hook(
5564        &mut self,
5565        hook: impl FnOnce() + Send + Sync + 'static,
5566    ) {
5567        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
5568    }
5569
5570    #[cfg(test)]
5571    pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
5572        self.graph.formula_authority().indexes_epoch()
5573    }
5574
5575    #[cfg(test)]
5576    pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
5577        self.formula_plane_capacity_bailouts
5578    }
5579
5580    #[cfg(test)]
5581    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
5582        self.topology_epoch
5583    }
5584
5585    #[cfg(test)]
5586    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
5587        self.graph.topology_revision()
5588    }
5589
5590    #[cfg(test)]
5591    pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
5592        self.cached_mixed_topology.is_some()
5593    }
5594
5595    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
5596        self.formula_ingest_report_total.mode = report.mode;
5597        self.formula_ingest_report_total.accumulate(&report);
5598        self.last_formula_ingest_report = Some(report);
5599    }
5600
5601    fn prepare_source_formula_family(
5602        &mut self,
5603        sheet_id: SheetId,
5604        family: &crate::engine::SourceFormulaFamily,
5605        allow_function_closure: bool,
5606        planning_provider: Option<&dyn crate::traits::FunctionProvider>,
5607    ) -> Result<
5608        (
5609            crate::formula_plane::placement::PreparedAnchorOncePlacement,
5610            bool,
5611        ),
5612        SourceFamilyPreparationError,
5613    > {
5614        let transport = family
5615            .validated_complete_domain(&self.workbook_load_limits)
5616            .map_err(SourceFamilyPreparationError::contract)?;
5617        let domain = match transport {
5618            crate::engine::PlacementDomainTransport::RowRun {
5619                row_start,
5620                row_end,
5621                col,
5622            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5623            crate::engine::PlacementDomainTransport::ColRun {
5624                row,
5625                col_start,
5626                col_end,
5627            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5628            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5629                sheet_id,
5630                rect.start.row,
5631                rect.end.row,
5632                rect.start.col,
5633                rect.end.col,
5634            ),
5635        };
5636        let formula = if family.anchor_text.starts_with('=') {
5637            family.anchor_text.to_string()
5638        } else {
5639            format!("={}", family.anchor_text)
5640        };
5641        let ast = formualizer_parse::parser::parse(&formula)
5642            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5643        let relocation = if allow_function_closure {
5644            validate_anchor_once_shadow_relocation(
5645                &ast,
5646                family.anchor_coord0.row,
5647                family.anchor_coord0.col,
5648                &domain,
5649                planning_provider.unwrap_or(&self.resolver),
5650            )
5651        } else {
5652            validate_anchor_once_syntax(
5653                &ast,
5654                family.anchor_coord0.row,
5655                family.anchor_coord0.col,
5656                &domain,
5657            )
5658        };
5659        relocation.map_err(SourceFamilyPreparationError::ast)?;
5660        let ast_id = self.intern_formula_ast(&ast);
5661        let placement = CellRef::new(
5662            sheet_id,
5663            Coord::from_excel(
5664                family.anchor_coord0.row + 1,
5665                family.anchor_coord0.col + 1,
5666                true,
5667                true,
5668            ),
5669        );
5670        let ingested = {
5671            let mut pipeline = match planning_provider {
5672                Some(provider) => self.graph.ingest_pipeline(provider),
5673                None => self.ingest_pipeline(),
5674            };
5675            if allow_function_closure {
5676                pipeline = pipeline.enable_function_semantics();
5677            }
5678            pipeline.ingest_formula(
5679                FormulaAstInput::RawArena(ast_id),
5680                placement,
5681                Some(family.anchor_text.clone()),
5682            )
5683        }
5684        .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5685        let function_semantics_used = ingested
5686            .labels
5687            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5688        let candidate = FormulaPlacementCandidate::new(
5689            sheet_id,
5690            family.anchor_coord0.row,
5691            family.anchor_coord0.col,
5692            ingested.ast_id,
5693            Some(family.anchor_text.clone()),
5694        );
5695        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5696            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
5697        prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
5698            .map(|prepared| (prepared, function_semantics_used))
5699            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
5700    }
5701
5702    fn placement_domain_from_transport(
5703        sheet_id: SheetId,
5704        transport: crate::engine::PlacementDomainTransport,
5705    ) -> PlacementDomain {
5706        match transport {
5707            crate::engine::PlacementDomainTransport::RowRun {
5708                row_start,
5709                row_end,
5710                col,
5711            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5712            crate::engine::PlacementDomainTransport::ColRun {
5713                row,
5714                col_start,
5715                col_end,
5716            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5717            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5718                sheet_id,
5719                rect.start.row,
5720                rect.end.row,
5721                rect.start.col,
5722                rect.end.col,
5723            ),
5724        }
5725    }
5726
5727    #[cfg(test)]
5728    pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
5729        &mut self,
5730        sheet_name: &str,
5731        source_id: crate::engine::SourceFamilyId,
5732        expected: crate::engine::PartitionLegacyMember,
5733        replay: crate::engine::DeferredReplayFormula,
5734    ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
5735    {
5736        let sheet_id = self.graph.sheet_id_mut(sheet_name);
5737        self.graph
5738            .analyze_fragmented_exact_replay_record(
5739                &self.resolver,
5740                sheet_id,
5741                source_id,
5742                expected,
5743                replay,
5744            )
5745            .map_err(|error| format!("{error:?}"))
5746    }
5747
5748    pub(crate) fn prepare_fragmented_source_transaction(
5749        &mut self,
5750        source: &crate::engine::PartitionedSourceFormulaFamily,
5751        disposition: &crate::engine::FormulaReplayDisposition,
5752        prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5753        legacy_formulas: Vec<
5754            crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
5755        >,
5756    ) -> Result<
5757        crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5758        crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
5759    > {
5760        let transaction = self.graph.prepare_fragmented_source_transaction(
5761            &self.source_formula_token,
5762            source,
5763            disposition,
5764            prepared,
5765            legacy_formulas,
5766        )?;
5767        self.prepared_legacy_admission(
5768            transaction.legacy_graph(),
5769            transaction.materialization_cells(),
5770        )
5771        .map_err(
5772            crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
5773        )?;
5774        Ok(transaction)
5775    }
5776
5777    pub(crate) fn commit_fragmented_source_transaction(
5778        &mut self,
5779        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5780        disposition: &crate::engine::FormulaReplayDisposition,
5781    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5782        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5783        let provider_revision = self.resolver.planning_semantic_revision();
5784        let epoch = registry_guard.epoch();
5785        let resolver = &self.resolver;
5786        let decision = self.graph.commit_fragmented_source_transaction(
5787            prepared,
5788            &self.source_formula_token,
5789            disposition,
5790            epoch,
5791            provider_revision,
5792            || resolver.planning_semantic_revision(),
5793            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5794        );
5795        drop(registry_guard);
5796        decision
5797    }
5798
5799    #[cfg(test)]
5800    pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
5801        &mut self,
5802        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5803        disposition: &crate::engine::FormulaReplayDisposition,
5804        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5805    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5806        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5807        let provider_revision = self.resolver.planning_semantic_revision();
5808        let epoch = registry_guard.epoch();
5809        let resolver = &self.resolver;
5810        let decision = self.graph.commit_fragmented_source_transaction(
5811            prepared,
5812            &self.source_formula_token,
5813            disposition,
5814            epoch,
5815            provider_revision,
5816            || resolver.planning_semantic_revision(),
5817            fault,
5818        );
5819        drop(registry_guard);
5820        decision
5821    }
5822
5823    #[cfg(test)]
5824    pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
5825        &mut self,
5826        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5827        disposition: &crate::engine::FormulaReplayDisposition,
5828        before_second_sample: impl FnOnce(),
5829    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5830        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5831        let provider_revision = self.resolver.planning_semantic_revision();
5832        let epoch = registry_guard.epoch();
5833        let resolver = &self.resolver;
5834        let decision = self.graph.commit_fragmented_source_transaction(
5835            prepared,
5836            &self.source_formula_token,
5837            disposition,
5838            epoch,
5839            provider_revision,
5840            || {
5841                before_second_sample();
5842                resolver.planning_semantic_revision()
5843            },
5844            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5845        );
5846        drop(registry_guard);
5847        decision
5848    }
5849
5850    #[cfg(test)]
5851    pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
5852        &mut self,
5853        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5854        disposition: &crate::engine::FormulaReplayDisposition,
5855        function_semantic_epoch: u64,
5856        function_provider_revision: Option<u64>,
5857    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5858        self.graph.commit_fragmented_source_transaction(
5859            prepared,
5860            &self.source_formula_token,
5861            disposition,
5862            function_semantic_epoch,
5863            function_provider_revision,
5864            || function_provider_revision,
5865            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5866        )
5867    }
5868
5869    pub(crate) fn analyze_partitioned_source_family_for_transaction(
5870        &mut self,
5871        sheet_name: &str,
5872        family: &crate::engine::PartitionedSourceFormulaFamily,
5873    ) -> Result<
5874        crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5875        SourceFamilyPreparationError,
5876    > {
5877        family
5878            .validate(&self.workbook_load_limits)
5879            .map_err(SourceFamilyPreparationError::contract)?;
5880        let formula = if family.template_text.starts_with('=') {
5881            family.template_text.to_string()
5882        } else {
5883            format!("={}", family.template_text)
5884        };
5885        let ast = formualizer_parse::parser::parse(&formula)
5886            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5887        let mut requests = Vec::new();
5888        Self::collect_planning_function_requests(&ast, &mut requests);
5889        let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
5890            &self.resolver,
5891            requests,
5892        )
5893        .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5894        let sheet_id = self.graph.sheet_id_mut(sheet_name);
5895        let domains: Vec<_> = family
5896            .fragments
5897            .iter()
5898            .copied()
5899            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
5900            .collect();
5901        for domain in &domains {
5902            validate_anchor_once_shadow_relocation(
5903                &ast,
5904                family.template_origin0.row,
5905                family.template_origin0.col,
5906                domain,
5907                &snapshot,
5908            )
5909            .map_err(SourceFamilyPreparationError::ast)?;
5910        }
5911        let ast_id = self.intern_formula_ast(&ast);
5912        let placement = CellRef::new(
5913            sheet_id,
5914            Coord::from_excel(
5915                family.template_origin0.row + 1,
5916                family.template_origin0.col + 1,
5917                true,
5918                true,
5919            ),
5920        );
5921        let ingested = self
5922            .graph
5923            .ingest_pipeline(&snapshot)
5924            .enable_function_semantics()
5925            .ingest_formula(
5926                FormulaAstInput::RawArena(ast_id),
5927                placement,
5928                Some(family.template_text.clone()),
5929            )
5930            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5931        let function_semantics_used = ingested
5932            .labels
5933            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5934        let name_assumptions = self
5935            .graph
5936            .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
5937            .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5938        let candidate = FormulaPlacementCandidate::new(
5939            sheet_id,
5940            family.template_origin0.row,
5941            family.template_origin0.col,
5942            ingested.ast_id,
5943            Some(family.template_text.clone()),
5944        );
5945        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5946            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
5947        let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
5948            total
5949                .checked_add(domain.cell_count())
5950                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
5951        })?;
5952        let mut placements = Vec::with_capacity(domains.len());
5953        for domain in domains {
5954            let member_count = domain.cell_count();
5955            placements.push(
5956                prepare_anchor_once_fragment(
5957                    candidate.clone(),
5958                    analysis.clone(),
5959                    domain,
5960                    member_count,
5961                    direct_cells,
5962                )
5963                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
5964            );
5965        }
5966        let fallback_cells = family.legacy_members.shared_member_count() as u64;
5967        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
5968            return Err(SourceFamilyPreparationError::analysis(
5969                "PartitionMemberCountMismatch",
5970            ));
5971        }
5972        Ok(
5973            crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
5974                engine_token: Arc::clone(&self.source_formula_token),
5975                sheet_id,
5976                sheet_name: Arc::from(sheet_name),
5977                source: family.clone(),
5978                placements,
5979                function_semantic_epoch: snapshot.epoch(),
5980                function_provider_revision: snapshot.provider_revision(),
5981                function_semantics_used,
5982                name_assumptions,
5983                direct_cells,
5984            },
5985        )
5986    }
5987
5988    fn prepare_partitioned_source_formula_family_shadow(
5989        &mut self,
5990        sheet_id: SheetId,
5991        family: &crate::engine::PartitionedSourceFormulaFamily,
5992    ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
5993        family
5994            .validate(&self.workbook_load_limits)
5995            .map_err(SourceFamilyPreparationError::contract)?;
5996        let formula = if family.template_text.starts_with('=') {
5997            family.template_text.to_string()
5998        } else {
5999            format!("={}", family.template_text)
6000        };
6001        let ast = formualizer_parse::parser::parse(&formula)
6002            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6003        let domains: Vec<_> = family
6004            .fragments
6005            .iter()
6006            .copied()
6007            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6008            .collect();
6009        for domain in &domains {
6010            validate_anchor_once_shadow_relocation(
6011                &ast,
6012                family.template_origin0.row,
6013                family.template_origin0.col,
6014                domain,
6015                &self.resolver,
6016            )
6017            .map_err(SourceFamilyPreparationError::ast)?;
6018        }
6019        let ast_id = self.intern_formula_ast(&ast);
6020        let placement = CellRef::new(
6021            sheet_id,
6022            Coord::from_excel(
6023                family.template_origin0.row + 1,
6024                family.template_origin0.col + 1,
6025                true,
6026                true,
6027            ),
6028        );
6029        let ingested = self
6030            .ingest_pipeline()
6031            .enable_function_semantics()
6032            .ingest_formula(
6033                FormulaAstInput::RawArena(ast_id),
6034                placement,
6035                Some(family.template_text.clone()),
6036            )
6037            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6038        let function_semantics_used = ingested
6039            .labels
6040            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6041        let candidate = FormulaPlacementCandidate::new(
6042            sheet_id,
6043            family.template_origin0.row,
6044            family.template_origin0.col,
6045            ingested.ast_id,
6046            Some(family.template_text.clone()),
6047        );
6048        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6049            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6050        let fragment_count = domains.len() as u64;
6051        let mut direct_cells = 0u64;
6052        for domain in domains {
6053            let member_count = domain.cell_count();
6054            validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6055                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6056            direct_cells = direct_cells
6057                .checked_add(member_count)
6058                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6059        }
6060        let fallback_cells = family.legacy_members.shared_member_count() as u64;
6061        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6062            return Err(SourceFamilyPreparationError::analysis(
6063                "PartitionMemberCountMismatch",
6064            ));
6065        }
6066        Ok(PreparedPartitionShadow {
6067            fragment_count,
6068            direct_cells,
6069            fallback_cells,
6070            function_semantics_used,
6071        })
6072    }
6073
6074    fn analyze_partitioned_formula_plane_shadow(
6075        &mut self,
6076        batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6077    ) -> FormulaIngestReport {
6078        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6079        for (sheet_name, families) in batches {
6080            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6081            for family in families {
6082                report.shadow_candidate_cells = report
6083                    .shadow_candidate_cells
6084                    .saturating_add(family.surviving_member_count);
6085                match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6086                    Ok(prepared) => {
6087                        let fragment_count = prepared.fragment_count;
6088                        let descendants = family.surviving_member_count.saturating_sub(1);
6089                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6090                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6091                        report.source_anchor_analyses =
6092                            report.source_anchor_analyses.saturating_add(1);
6093                        report.source_partition_analyses_reused = report
6094                            .source_partition_analyses_reused
6095                            .saturating_add(fragment_count.saturating_sub(1));
6096                        report.source_partitioned_families_prepared = report
6097                            .source_partitioned_families_prepared
6098                            .saturating_add(1);
6099                        report.source_partition_fragments_prepared = report
6100                            .source_partition_fragments_prepared
6101                            .saturating_add(fragment_count);
6102                        report.source_partition_span_cells_prepared = report
6103                            .source_partition_span_cells_prepared
6104                            .saturating_add(prepared.direct_cells);
6105                        report.source_partition_fallback_cells = report
6106                            .source_partition_fallback_cells
6107                            .saturating_add(prepared.fallback_cells);
6108                        report.shadow_accepted_span_cells = report
6109                            .shadow_accepted_span_cells
6110                            .saturating_add(prepared.direct_cells);
6111                        report.shadow_fallback_cells = report
6112                            .shadow_fallback_cells
6113                            .saturating_add(prepared.fallback_cells);
6114                        report.source_descendant_strings_avoided = report
6115                            .source_descendant_strings_avoided
6116                            .saturating_add(descendants);
6117                        report.source_descendant_events_avoided = report
6118                            .source_descendant_events_avoided
6119                            .saturating_add(descendants);
6120                        report.source_descendant_analyses_avoided = report
6121                            .source_descendant_analyses_avoided
6122                            .saturating_add(descendants);
6123                        report.graph_formula_vertices_avoided_shadow = report
6124                            .graph_formula_vertices_avoided_shadow
6125                            .saturating_add(prepared.direct_cells);
6126                        report.ast_roots_avoided_shadow = report
6127                            .ast_roots_avoided_shadow
6128                            .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6129                        report.edge_rows_avoided_shadow = report
6130                            .edge_rows_avoided_shadow
6131                            .saturating_add(prepared.direct_cells);
6132                        if prepared.function_semantics_used {
6133                            report.source_partition_function_semantics =
6134                                report.source_partition_function_semantics.saturating_add(1);
6135                        }
6136                    }
6137                    Err(error) => {
6138                        if error.parse_attempted {
6139                            report.source_anchor_parses =
6140                                report.source_anchor_parses.saturating_add(1);
6141                        }
6142                        if error.ast_created {
6143                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6144                        }
6145                        if error.analysis_created {
6146                            report.source_anchor_analyses =
6147                                report.source_anchor_analyses.saturating_add(1);
6148                        }
6149                        report.shadow_fallback_cells = report
6150                            .shadow_fallback_cells
6151                            .saturating_add(family.surviving_member_count);
6152                        report.source_partitioned_families_rejected = report
6153                            .source_partitioned_families_rejected
6154                            .saturating_add(1);
6155                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
6156                    }
6157                }
6158            }
6159        }
6160        report
6161    }
6162
6163    fn analyze_compressed_formula_plane_shadow(
6164        &mut self,
6165        batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6166    ) -> FormulaIngestReport {
6167        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6168        let _active_epoch = self.graph.formula_authority().plane.epoch();
6169
6170        for (sheet_name, families) in batches {
6171            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6172            for family in families {
6173                let descendants = family.member_count.saturating_sub(1);
6174                report.shadow_candidate_cells = report
6175                    .shadow_candidate_cells
6176                    .saturating_add(family.member_count);
6177
6178                match self.prepare_source_formula_family(sheet_id, family, true, None) {
6179                    Ok(_) => {
6180                        report.source_descendant_strings_avoided = report
6181                            .source_descendant_strings_avoided
6182                            .saturating_add(descendants);
6183                        report.source_descendant_events_avoided = report
6184                            .source_descendant_events_avoided
6185                            .saturating_add(descendants);
6186                        report.source_descendant_analyses_avoided = report
6187                            .source_descendant_analyses_avoided
6188                            .saturating_add(descendants);
6189                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6190                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6191                        report.source_anchor_analyses =
6192                            report.source_anchor_analyses.saturating_add(1);
6193                        report.shadow_accepted_span_cells = report
6194                            .shadow_accepted_span_cells
6195                            .saturating_add(family.member_count);
6196                        report.source_compressed_families_prepared =
6197                            report.source_compressed_families_prepared.saturating_add(1);
6198                        report.source_compressed_cells_prepared = report
6199                            .source_compressed_cells_prepared
6200                            .saturating_add(family.member_count);
6201                        report.graph_formula_vertices_avoided_shadow = report
6202                            .graph_formula_vertices_avoided_shadow
6203                            .saturating_add(family.member_count);
6204                        report.ast_roots_avoided_shadow =
6205                            report.ast_roots_avoided_shadow.saturating_add(descendants);
6206                        report.edge_rows_avoided_shadow = report
6207                            .edge_rows_avoided_shadow
6208                            .saturating_add(family.member_count);
6209                    }
6210                    Err(error) => {
6211                        if error.parse_attempted {
6212                            report.source_anchor_parses =
6213                                report.source_anchor_parses.saturating_add(1);
6214                        }
6215                        if error.ast_created {
6216                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6217                        }
6218                        if error.analysis_created {
6219                            report.source_anchor_analyses =
6220                                report.source_anchor_analyses.saturating_add(1);
6221                        }
6222                        report.shadow_fallback_cells = report
6223                            .shadow_fallback_cells
6224                            .saturating_add(family.member_count);
6225                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
6226                    }
6227                }
6228            }
6229        }
6230        report
6231    }
6232
6233    fn analyze_formula_plane_shadow_candidates(
6234        &mut self,
6235        batches: &[FormulaIngestBatch],
6236    ) -> FormulaIngestReport {
6237        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6238        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6239
6240        // Touch graph-owned authority deliberately: Tranche 3 shadow analysis uses
6241        // scratch state, but FormulaPlane ownership now lives on DependencyGraph.
6242        let _active_epoch = self.graph.formula_authority().plane.epoch();
6243
6244        let batch_sheet_ids: Vec<SheetId> = batches
6245            .iter()
6246            .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
6247            .collect();
6248        let mut groups: BTreeMap<
6249            (SheetId, u64, u32),
6250            Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
6251        > = BTreeMap::new();
6252        {
6253            let mut pipeline = self.ingest_pipeline();
6254            for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
6255                for record in &batch.formulas {
6256                    if record.row == 0 || record.col == 0 {
6257                        report.shadow_candidate_cells =
6258                            report.shadow_candidate_cells.saturating_add(1);
6259                        report.shadow_fallback_cells =
6260                            report.shadow_fallback_cells.saturating_add(1);
6261                        Self::record_shadow_fallback_reason(
6262                            &mut report,
6263                            PlacementFallbackReason::UnsupportedShapeOrGaps,
6264                            1,
6265                        );
6266                        continue;
6267                    }
6268
6269                    let placement = CellRef::new(
6270                        sheet_id,
6271                        Coord::from_excel(record.row, record.col, true, true),
6272                    );
6273                    let ingested = match pipeline.ingest_formula(
6274                        FormulaAstInput::RawArena(record.ast_id),
6275                        placement,
6276                        record.formula_text.clone(),
6277                    ) {
6278                        Ok(ingested) => ingested,
6279                        Err(_) => {
6280                            report.shadow_candidate_cells =
6281                                report.shadow_candidate_cells.saturating_add(1);
6282                            report.shadow_fallback_cells =
6283                                report.shadow_fallback_cells.saturating_add(1);
6284                            Self::record_shadow_fallback_reason(
6285                                &mut report,
6286                                PlacementFallbackReason::UnsupportedCanonicalTemplate,
6287                                1,
6288                            );
6289                            continue;
6290                        }
6291                    };
6292                    let candidate = FormulaPlacementCandidate::new(
6293                        sheet_id,
6294                        record.row - 1,
6295                        record.col - 1,
6296                        ingested.ast_id,
6297                        record.formula_text.clone(),
6298                    );
6299                    let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
6300                        Ok(analysis) => analysis,
6301                        Err(reason) => {
6302                            report.shadow_candidate_cells =
6303                                report.shadow_candidate_cells.saturating_add(1);
6304                            report.shadow_fallback_cells =
6305                                report.shadow_fallback_cells.saturating_add(1);
6306                            Self::record_shadow_fallback_reason(&mut report, reason, 1);
6307                            continue;
6308                        }
6309                    };
6310                    groups
6311                        .entry((
6312                            sheet_id,
6313                            ingested.parameterized_canonical_hash,
6314                            candidate.col,
6315                        ))
6316                        .or_default()
6317                        .push((candidate, analysis));
6318                }
6319            }
6320        }
6321
6322        let mut scratch_plane = FormulaPlane::default();
6323        for entries in groups.into_values() {
6324            let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
6325            for (component, component_analyses) in
6326                Self::split_candidate_components_with_analyses(candidates, analyses)
6327            {
6328                let placement_report = place_candidate_family_with_analyses(
6329                    &mut scratch_plane,
6330                    component,
6331                    component_analyses,
6332                );
6333                let counters = placement_report.counters;
6334                report.shadow_candidate_cells = report
6335                    .shadow_candidate_cells
6336                    .saturating_add(counters.formula_cells_seen);
6337                report.shadow_accepted_span_cells = report
6338                    .shadow_accepted_span_cells
6339                    .saturating_add(counters.accepted_span_cells);
6340                report.shadow_fallback_cells = report
6341                    .shadow_fallback_cells
6342                    .saturating_add(counters.legacy_cells);
6343                report.shadow_templates_interned = report
6344                    .shadow_templates_interned
6345                    .saturating_add(counters.templates_interned);
6346                report.shadow_spans_created = report
6347                    .shadow_spans_created
6348                    .saturating_add(counters.spans_created);
6349                report.graph_formula_vertices_avoided_shadow = report
6350                    .graph_formula_vertices_avoided_shadow
6351                    .saturating_add(counters.formula_vertices_avoided);
6352                report.ast_roots_avoided_shadow = report
6353                    .ast_roots_avoided_shadow
6354                    .saturating_add(counters.ast_roots_avoided);
6355                report.edge_rows_avoided_shadow = report
6356                    .edge_rows_avoided_shadow
6357                    .saturating_add(counters.edge_rows_avoided);
6358                for (reason, count) in counters.fallback_reasons {
6359                    Self::record_shadow_fallback_reason(&mut report, reason, count);
6360                }
6361            }
6362        }
6363        report
6364    }
6365
6366    fn record_shadow_fallback_reason(
6367        report: &mut FormulaIngestReport,
6368        reason: PlacementFallbackReason,
6369        count: u64,
6370    ) {
6371        *report
6372            .fallback_reasons
6373            .entry(format!("{reason:?}"))
6374            .or_default() += count;
6375    }
6376
6377    fn analyze_formula_plane_authoritative_ingest(
6378        &mut self,
6379        batches: &[FormulaIngestBatch],
6380    ) -> (
6381        FormulaIngestReport,
6382        Vec<FormulaIngestBatch>,
6383        PlannedFormulaMaterialize,
6384    ) {
6385        let existing_span_refs = self
6386            .graph
6387            .formula_authority()
6388            .active_span_refs()
6389            .into_iter()
6390            .collect::<rustc_hash::FxHashSet<_>>();
6391        let mut report =
6392            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6393        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6394
6395        let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
6396        let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
6397        let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
6398
6399        for batch in batches {
6400            let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6401            for record in &batch.formulas {
6402                if record.row == 0 || record.col == 0 {
6403                    report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
6404                    report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
6405                    Self::record_shadow_fallback_reason(
6406                        &mut report,
6407                        PlacementFallbackReason::UnsupportedShapeOrGaps,
6408                        1,
6409                    );
6410                    fallback
6411                        .entry(batch.sheet_name.clone())
6412                        .or_default()
6413                        .push(record.clone());
6414                    continue;
6415                }
6416
6417                pending_candidates.push((
6418                    batch.sheet_name.clone(),
6419                    FormulaPlacementCandidate::new(
6420                        sheet_id,
6421                        record.row - 1,
6422                        record.col - 1,
6423                        record.ast_id,
6424                        record.formula_text.clone(),
6425                    ),
6426                ));
6427            }
6428        }
6429
6430        let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
6431        let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
6432            (0..pending_candidates.len()).map(|_| None).collect();
6433        let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
6434            (0..pending_candidates.len()).map(|_| None).collect();
6435        {
6436            let mut pipeline = self.ingest_pipeline();
6437            for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
6438                let placement = CellRef::new(
6439                    candidate.sheet_id,
6440                    Coord::from_excel(
6441                        candidate.row.saturating_add(1),
6442                        candidate.col.saturating_add(1),
6443                        true,
6444                        true,
6445                    ),
6446                );
6447                let ingested = pipeline.ingest_formula(
6448                    FormulaAstInput::RawArena(candidate.ast_id),
6449                    placement,
6450                    candidate.formula_text.clone(),
6451                );
6452                match ingested {
6453                    Ok(ingested) => {
6454                        candidate.ast_id = ingested.ast_id;
6455                        let dep_plan = ingested.dep_plan.clone();
6456                        match CandidateAnalysis::from_ingested(candidate, &ingested) {
6457                            Ok(analysis) => {
6458                                analyses_by_index[idx] = Some(analysis);
6459                                plans_by_index[idx] = Some(dep_plan);
6460                            }
6461                            Err(reason) => {
6462                                report.shadow_candidate_cells =
6463                                    report.shadow_candidate_cells.saturating_add(1);
6464                                report.shadow_fallback_cells =
6465                                    report.shadow_fallback_cells.saturating_add(1);
6466                                Self::record_shadow_fallback_reason(&mut report, reason, 1);
6467                                planned_fallback
6468                                    .entry(sheet_name.clone())
6469                                    .or_default()
6470                                    .push((
6471                                        candidate.row.saturating_add(1),
6472                                        candidate.col.saturating_add(1),
6473                                        candidate.ast_id,
6474                                        dep_plan,
6475                                    ));
6476                            }
6477                        }
6478                    }
6479                    Err(_) => {
6480                        let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
6481                        report.shadow_candidate_cells =
6482                            report.shadow_candidate_cells.saturating_add(1);
6483                        report.shadow_fallback_cells =
6484                            report.shadow_fallback_cells.saturating_add(1);
6485                        Self::record_shadow_fallback_reason(&mut report, reason, 1);
6486                        fallback.entry(sheet_name.clone()).or_default().push(
6487                            FormulaIngestRecord::new(
6488                                candidate.row.saturating_add(1),
6489                                candidate.col.saturating_add(1),
6490                                candidate.ast_id,
6491                                candidate.formula_text.clone(),
6492                            ),
6493                        );
6494                    }
6495                }
6496            }
6497        }
6498
6499        for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
6500            if analyses_by_index[idx].is_none() {
6501                continue;
6502            }
6503            let canonical_hash = analyses_by_index[idx]
6504                .as_ref()
6505                .expect("remaining candidate has an analysis")
6506                .parameterized_canonical_hash();
6507            groups
6508                .entry((candidate.sheet_id, canonical_hash, candidate.col))
6509                .or_default()
6510                .push(idx);
6511        }
6512
6513        for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
6514            let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
6515            let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
6516            for idx in &candidate_indices {
6517                // Each candidate index belongs to exactly one group, so the
6518                // plan row can be moved out instead of deep-cloned.
6519                if let Some(plan) = plans_by_index[*idx].take() {
6520                    let candidate = &pending_candidates[*idx].1;
6521                    plans_by_coord
6522                        .entry((candidate.row, candidate.col))
6523                        .or_default()
6524                        .push(plan);
6525                }
6526            }
6527            let candidates: Vec<_> = candidate_indices
6528                .iter()
6529                .map(|idx| pending_candidates[*idx].1.clone())
6530                .collect();
6531            let components = Self::split_shadow_candidate_components(candidates);
6532            let analyzed_components =
6533                if components.len() == 1 && components[0].len() == candidate_indices.len() {
6534                    let component = components.into_iter().next().expect("one component");
6535                    let component_analyses = candidate_indices
6536                        .iter()
6537                        .map(|idx| {
6538                            analyses_by_index[*idx]
6539                                .take()
6540                                .expect("candidate analysis must be used once")
6541                        })
6542                        .collect();
6543                    vec![(component, component_analyses)]
6544                } else {
6545                    let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6546                    for idx in candidate_indices.iter().rev() {
6547                        let candidate = &pending_candidates[*idx].1;
6548                        indices_by_coord
6549                            .entry((candidate.row, candidate.col))
6550                            .or_default()
6551                            .push(*idx);
6552                    }
6553
6554                    components
6555                        .into_iter()
6556                        .map(|component| {
6557                            let mut component_analyses = Vec::with_capacity(component.len());
6558                            for candidate in &component {
6559                                let idx = indices_by_coord
6560                                    .get_mut(&(candidate.row, candidate.col))
6561                                    .and_then(Vec::pop)
6562                                    .expect("component candidate must have a precomputed analysis");
6563                                component_analyses.push(
6564                                    analyses_by_index[idx]
6565                                        .take()
6566                                        .expect("candidate analysis must be used once"),
6567                                );
6568                            }
6569                            (component, component_analyses)
6570                        })
6571                        .collect()
6572                };
6573
6574            for (component, component_analyses) in analyzed_components {
6575                for (component, component_analyses) in
6576                    split_candidate_affine_literal_runs(component, component_analyses)
6577                {
6578                    let placement_report = {
6579                        let authority = self.graph.formula_authority_mut();
6580                        place_candidate_family_with_analyses(
6581                            &mut authority.plane,
6582                            component.clone(),
6583                            component_analyses,
6584                        )
6585                    };
6586                    Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
6587
6588                    // Index candidates by placement once per component. The
6589                    // previous per-result linear `find` made this fallback
6590                    // mapping O(N²) for rejected families (e.g. an N-cell
6591                    // chain rejected with `InternalDependency`), dominating
6592                    // first-eval ingest cost on large rejected families.
6593                    // First insert wins, matching the old `Iterator::find`
6594                    // semantics for duplicate placements.
6595                    let mut candidate_by_placement: FxHashMap<
6596                        crate::formula_plane::runtime::PlacementCoord,
6597                        &FormulaPlacementCandidate,
6598                    > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
6599                    for candidate in &component {
6600                        candidate_by_placement
6601                            .entry(candidate.placement())
6602                            .or_insert(candidate);
6603                    }
6604                    for result in &placement_report.results {
6605                        let FormulaPlacementResult::Legacy { placement, .. } = result else {
6606                            continue;
6607                        };
6608                        if let Some(&candidate) = candidate_by_placement.get(placement) {
6609                            let plan = plans_by_coord
6610                                .get_mut(&(candidate.row, candidate.col))
6611                                .and_then(Vec::pop);
6612                            if let Some(plan) = plan {
6613                                planned_fallback
6614                                    .entry(sheet_name.clone())
6615                                    .or_default()
6616                                    .push((
6617                                        candidate.row.saturating_add(1),
6618                                        candidate.col.saturating_add(1),
6619                                        candidate.ast_id,
6620                                        plan,
6621                                    ));
6622                            } else {
6623                                fallback.entry(sheet_name.clone()).or_default().push(
6624                                    FormulaIngestRecord::new(
6625                                        candidate.row.saturating_add(1),
6626                                        candidate.col.saturating_add(1),
6627                                        candidate.ast_id,
6628                                        candidate.formula_text.clone(),
6629                                    ),
6630                                );
6631                            }
6632                        }
6633                    }
6634                }
6635            }
6636        }
6637
6638        let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
6639        let new_span_refs = self
6640            .graph
6641            .formula_authority()
6642            .active_span_refs()
6643            .into_iter()
6644            .filter(|span_ref| !existing_span_refs.contains(span_ref))
6645            .collect::<Vec<_>>();
6646        self.graph
6647            .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
6648
6649        let fallback_batches = fallback
6650            .into_iter()
6651            .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
6652            .collect();
6653        (report, fallback_batches, planned_fallback)
6654    }
6655
6656    fn accumulate_formula_plane_placement_report(
6657        report: &mut FormulaIngestReport,
6658        placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
6659    ) {
6660        let counters = &placement_report.counters;
6661        report.shadow_candidate_cells = report
6662            .shadow_candidate_cells
6663            .saturating_add(counters.formula_cells_seen);
6664        report.shadow_accepted_span_cells = report
6665            .shadow_accepted_span_cells
6666            .saturating_add(counters.accepted_span_cells);
6667        report.shadow_fallback_cells = report
6668            .shadow_fallback_cells
6669            .saturating_add(counters.legacy_cells);
6670        report.shadow_templates_interned = report
6671            .shadow_templates_interned
6672            .saturating_add(counters.templates_interned);
6673        report.shadow_spans_created = report
6674            .shadow_spans_created
6675            .saturating_add(counters.spans_created);
6676        report.graph_formula_vertices_avoided_shadow = report
6677            .graph_formula_vertices_avoided_shadow
6678            .saturating_add(counters.formula_vertices_avoided);
6679        report.ast_roots_avoided_shadow = report
6680            .ast_roots_avoided_shadow
6681            .saturating_add(counters.ast_roots_avoided);
6682        report.edge_rows_avoided_shadow = report
6683            .edge_rows_avoided_shadow
6684            .saturating_add(counters.edge_rows_avoided);
6685        for (reason, count) in &counters.fallback_reasons {
6686            Self::record_shadow_fallback_reason(report, *reason, *count);
6687        }
6688    }
6689
6690    fn split_candidate_components_with_analyses(
6691        candidates: Vec<FormulaPlacementCandidate>,
6692        mut analyses: Vec<CandidateAnalysis>,
6693    ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
6694        let components = Self::split_shadow_candidate_components(candidates.clone());
6695        let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
6696        for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
6697            analysis_by_coord
6698                .entry((candidate.row, candidate.col))
6699                .or_default()
6700                .push(analysis);
6701        }
6702        components
6703            .into_iter()
6704            .flat_map(|component| {
6705                let mut component_analyses = Vec::with_capacity(component.len());
6706                for candidate in &component {
6707                    let analysis = analysis_by_coord
6708                        .get_mut(&(candidate.row, candidate.col))
6709                        .and_then(Vec::pop)
6710                        .expect("component candidate must have a precomputed analysis");
6711                    component_analyses.push(analysis);
6712                }
6713                split_candidate_affine_literal_runs(component, component_analyses)
6714            })
6715            .collect()
6716    }
6717
6718    fn split_shadow_candidate_components(
6719        candidates: Vec<FormulaPlacementCandidate>,
6720    ) -> Vec<Vec<FormulaPlacementCandidate>> {
6721        if candidates.len() <= 1 {
6722            return vec![candidates];
6723        }
6724
6725        // Fast path: candidates already ordered as one contiguous
6726        // single-column (or single-row) run form exactly one 4-connected
6727        // component in their existing (row, col) order; skip the BFS.
6728        let is_row_run = candidates.windows(2).all(|w| {
6729            w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
6730        });
6731        let is_col_run = candidates.windows(2).all(|w| {
6732            w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
6733        });
6734        if is_row_run || is_col_run {
6735            return vec![candidates];
6736        }
6737
6738        let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6739        for (idx, candidate) in candidates.iter().enumerate() {
6740            coord_to_indices
6741                .entry((candidate.row, candidate.col))
6742                .or_default()
6743                .push(idx);
6744        }
6745
6746        let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
6747        let mut components = Vec::new();
6748        while let Some(&start) = remaining.iter().next() {
6749            remaining.remove(&start);
6750            let mut queue = VecDeque::from([start]);
6751            let mut component_indices = Vec::new();
6752
6753            while let Some(idx) = queue.pop_front() {
6754                component_indices.push(idx);
6755                let candidate = &candidates[idx];
6756                let mut neighbor_coords = Vec::with_capacity(5);
6757                neighbor_coords.push((candidate.row, candidate.col));
6758                if let Some(row) = candidate.row.checked_sub(1) {
6759                    neighbor_coords.push((row, candidate.col));
6760                }
6761                neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
6762                if let Some(col) = candidate.col.checked_sub(1) {
6763                    neighbor_coords.push((candidate.row, col));
6764                }
6765                neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
6766
6767                for coord in neighbor_coords {
6768                    if let Some(indices) = coord_to_indices.get(&coord) {
6769                        for &neighbor in indices {
6770                            if remaining.remove(&neighbor) {
6771                                queue.push_back(neighbor);
6772                            }
6773                        }
6774                    }
6775                }
6776            }
6777
6778            component_indices.sort_by_key(|idx| {
6779                let candidate = &candidates[*idx];
6780                (candidate.row, candidate.col, *idx)
6781            });
6782            components.push(
6783                component_indices
6784                    .into_iter()
6785                    .map(|idx| candidates[idx].clone())
6786                    .collect(),
6787            );
6788        }
6789
6790        components
6791    }
6792
6793    fn collect_planning_function_requests(
6794        ast: &ASTNode,
6795        requests: &mut Vec<(String, String, usize)>,
6796    ) {
6797        match &ast.node_type {
6798            ASTNodeType::Function { name, args } => {
6799                requests.push((String::new(), name.clone(), args.len()));
6800                for arg in args {
6801                    Self::collect_planning_function_requests(arg, requests);
6802                }
6803            }
6804            ASTNodeType::BinaryOp { left, right, .. } => {
6805                Self::collect_planning_function_requests(left, requests);
6806                Self::collect_planning_function_requests(right, requests);
6807            }
6808            ASTNodeType::UnaryOp { expr, .. } => {
6809                Self::collect_planning_function_requests(expr, requests);
6810            }
6811            ASTNodeType::Call { callee, args } => {
6812                Self::collect_planning_function_requests(callee, requests);
6813                for arg in args {
6814                    Self::collect_planning_function_requests(arg, requests);
6815                }
6816            }
6817            ASTNodeType::Array(rows) => {
6818                for cell in rows.iter().flatten() {
6819                    Self::collect_planning_function_requests(cell, requests);
6820                }
6821            }
6822            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6823        }
6824    }
6825
6826    fn prepared_placement_function_semantics_changed(
6827        &self,
6828        semantic_epoch: u64,
6829        prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
6830        guard: &crate::function_registry::SemanticEpochReadGuard,
6831    ) -> bool {
6832        if semantic_epoch == guard.epoch() {
6833            return false;
6834        }
6835        let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6836        let Some(ast) = self
6837            .graph
6838            .data_store()
6839            .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6840        else {
6841            // Missing planning evidence must never turn a semantic change into reuse.
6842            return true;
6843        };
6844        let mut requests = Vec::new();
6845        Self::collect_planning_function_requests(&ast, &mut requests);
6846        guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
6847    }
6848
6849    fn prepared_function_semantics_changed(
6850        &self,
6851        preparation: &crate::engine::FormulaCompressedPreparation,
6852        guard: &crate::function_registry::SemanticEpochReadGuard,
6853    ) -> bool {
6854        if preparation.function_semantic_epoch == guard.epoch() {
6855            return false;
6856        }
6857
6858        let mut requests = Vec::new();
6859        for (_, _, prepared) in &preparation.prepared {
6860            let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6861            let Some(ast) = self
6862                .graph
6863                .data_store()
6864                .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6865            else {
6866                return true;
6867            };
6868            Self::collect_planning_function_requests(&ast, &mut requests);
6869        }
6870        guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
6871    }
6872
6873    pub(crate) fn prepare_source_formula_families(
6874        &mut self,
6875        sheet_name: &str,
6876        families: &[crate::engine::SourceFormulaFamily],
6877    ) -> crate::engine::FormulaCompressedPreparation {
6878        let mut preparation = crate::engine::FormulaCompressedPreparation {
6879            engine_token: Arc::clone(&self.source_formula_token),
6880            function_semantic_epoch: crate::function_registry::semantic_epoch(),
6881            function_provider_revision: None,
6882            function_semantics_used: false,
6883            sheet_name: Arc::from(sheet_name),
6884            prepared: Vec::new(),
6885            rejected: BTreeMap::new(),
6886            fragmented: Vec::new(),
6887            fragmented_sources: BTreeMap::new(),
6888            eager_replay: Vec::new(),
6889            preparation_spool_replays: 0,
6890            clean_rejected_anchor_counts: [0; 3],
6891            fragmented_rejected_anchor_counts: [0; 3],
6892            exact_replay: None,
6893            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6894        };
6895        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6896            return preparation;
6897        }
6898
6899        let mut requests = Vec::new();
6900        for family in families {
6901            let formula = if family.anchor_text.starts_with('=') {
6902                family.anchor_text.to_string()
6903            } else {
6904                format!("={}", family.anchor_text)
6905            };
6906            if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
6907                Self::collect_planning_function_requests(&ast, &mut requests);
6908            }
6909        }
6910        requests.sort();
6911        requests.dedup();
6912        let snapshot =
6913            match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6914                &self.resolver,
6915                requests,
6916            ) {
6917                Ok(snapshot) => snapshot,
6918                Err(error) => {
6919                    let reason = match error {
6920                        crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
6921                            "FunctionSemanticSnapshotUnavailable"
6922                        }
6923                        crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
6924                            "FunctionProviderRevisionUnavailable"
6925                        }
6926                    };
6927                    for family in families {
6928                        preparation
6929                            .rejected
6930                            .insert(family.source_id, reason.to_string());
6931                    }
6932                    return preparation;
6933                }
6934            };
6935        preparation.function_semantic_epoch = snapshot.epoch();
6936        preparation.function_provider_revision = snapshot.provider_revision();
6937
6938        let sheet_id = self.graph.sheet_id_mut(sheet_name);
6939        for family in families {
6940            #[cfg(test)]
6941            let force_replay = self.force_source_family_fallback
6942                || cfg!(feature = "benchmark_internal")
6943                    && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
6944            #[cfg(all(feature = "benchmark_internal", not(test)))]
6945            let force_replay =
6946                std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
6947            #[cfg(any(test, feature = "benchmark_internal"))]
6948            if force_replay {
6949                preparation
6950                    .rejected
6951                    .insert(family.source_id, "ForcedReplay".to_string());
6952                continue;
6953            }
6954            match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
6955                Ok((prepared, function_semantics_used)) => {
6956                    preparation.function_semantics_used |= function_semantics_used;
6957                    preparation
6958                        .replay_disposition
6959                        .set_family_direct(family.source_id);
6960                    preparation
6961                        .prepared
6962                        .push((family.source_id, family.source_order, prepared));
6963                }
6964                Err(error) => {
6965                    preparation.clean_rejected_anchor_counts[0] = preparation
6966                        .clean_rejected_anchor_counts[0]
6967                        .saturating_add(u64::from(error.parse_attempted));
6968                    preparation.clean_rejected_anchor_counts[1] = preparation
6969                        .clean_rejected_anchor_counts[1]
6970                        .saturating_add(u64::from(error.ast_created));
6971                    preparation.clean_rejected_anchor_counts[2] = preparation
6972                        .clean_rejected_anchor_counts[2]
6973                        .saturating_add(u64::from(error.analysis_created));
6974                    preparation.rejected.insert(family.source_id, error.reason);
6975                }
6976            }
6977        }
6978        preparation
6979    }
6980
6981    fn prepare_source_formula_proposals(
6982        &mut self,
6983        sheet_name: &str,
6984        families: &[crate::engine::SourceFormulaFamily],
6985        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6986        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
6987        formula_record_count: u64,
6988        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
6989        suppressed: &BTreeSet<(u32, u32)>,
6990    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
6991        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
6992        preparation.exact_replay = Some(Arc::clone(&replay));
6993        preparation
6994            .replay_disposition
6995            .extend_suppressed_excel_coords(suppressed.iter().copied());
6996        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6997            return Ok(preparation);
6998        }
6999        #[cfg(feature = "benchmark_internal")]
7000        let benchmark_forced_replay =
7001            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7002        #[cfg(not(feature = "benchmark_internal"))]
7003        let benchmark_forced_replay = false;
7004        let authority_partitions = if benchmark_forced_replay {
7005            for partition in authority_partitions {
7006                preparation
7007                    .rejected
7008                    .insert(partition.source_id, "ForcedReplay".to_string());
7009            }
7010            &[][..]
7011        } else {
7012            authority_partitions
7013        };
7014        let authority_ids: BTreeSet<_> = authority_partitions
7015            .iter()
7016            .map(|partition| partition.source_id)
7017            .collect();
7018        for partition in replay_partitions {
7019            if !authority_ids.contains(&partition.source_id) {
7020                preparation
7021                    .replay_disposition
7022                    .register_partition(partition, false)
7023                    .map_err(|reason| {
7024                        ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7025                    })?;
7026            }
7027        }
7028        if replay_partitions.is_empty()
7029            && preparation.rejected.is_empty()
7030            && preparation.direct_cell_count() == formula_record_count
7031        {
7032            return Ok(preparation);
7033        }
7034
7035        let mut analyzed = Vec::new();
7036        for source in authority_partitions {
7037            preparation
7038                .fragmented_sources
7039                .insert(source.source_id, source.clone());
7040            match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7041                Ok(prepared) => {
7042                    let mut candidate = preparation.replay_disposition.clone();
7043                    match candidate.register_partition(source, true) {
7044                        Ok(()) => {
7045                            preparation.replay_disposition = candidate;
7046                            analyzed.push((source.clone(), prepared));
7047                        }
7048                        Err(reason) => {
7049                            preparation
7050                                .rejected
7051                                .insert(source.source_id, reason.to_string());
7052                            preparation
7053                                .replay_disposition
7054                                .register_partition(source, false)
7055                                .map_err(|reason| {
7056                                    ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7057                                })?;
7058                        }
7059                    }
7060                }
7061                Err(error) => {
7062                    preparation.fragmented_rejected_anchor_counts[0] = preparation
7063                        .fragmented_rejected_anchor_counts[0]
7064                        .saturating_add(u64::from(error.parse_attempted));
7065                    preparation.fragmented_rejected_anchor_counts[1] = preparation
7066                        .fragmented_rejected_anchor_counts[1]
7067                        .saturating_add(u64::from(error.ast_created));
7068                    preparation.fragmented_rejected_anchor_counts[2] = preparation
7069                        .fragmented_rejected_anchor_counts[2]
7070                        .saturating_add(u64::from(error.analysis_created));
7071                    preparation.rejected.insert(source.source_id, error.reason);
7072                    preparation
7073                        .replay_disposition
7074                        .register_partition(source, false)
7075                        .map_err(|reason| {
7076                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7077                        })?;
7078                }
7079            }
7080        }
7081
7082        let initial_replay = replay
7083            .lock()
7084            .map_err(|_| {
7085                ExcelError::new(ExcelErrorKind::Value)
7086                    .with_message("compressed formula exact replay lock poisoned")
7087            })?
7088            .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7089            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7090        preparation.preparation_spool_replays = 1;
7091        let sheet_id = self.graph.sheet_id_mut(sheet_name);
7092        let mut preparation_failed = false;
7093        for (source, prepared_family) in analyzed {
7094            let result = (|| {
7095                let expected: BTreeMap<_, _> = source
7096                    .legacy_members
7097                    .as_slice()
7098                    .iter()
7099                    .map(|member| (member.coord, member.kind))
7100                    .collect();
7101                let mut actual = BTreeMap::new();
7102                for record in initial_replay
7103                    .iter()
7104                    .filter(|record| record.partition_owner == Some(source.source_id))
7105                {
7106                    let coord = record
7107                        .row
7108                        .checked_sub(1)
7109                        .zip(record.col.checked_sub(1))
7110                        .map(|(row, col)| crate::engine::SourceCoord { row, col })
7111                        .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7112                    let kind = match record.family {
7113                        Some(family) if family == source.source_id => {
7114                            crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7115                        }
7116                        None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7117                        Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7118                    };
7119                    if actual.insert(coord, (kind, record.clone())).is_some() {
7120                        return Err("ExactReplayRecordDuplicate".to_string());
7121                    }
7122                }
7123                let actual_kinds: BTreeMap<_, _> = actual
7124                    .iter()
7125                    .map(|(coord, (kind, _))| (*coord, *kind))
7126                    .collect();
7127                if actual_kinds != expected {
7128                    return Err("ExactReplayLegacySetMismatch".to_string());
7129                }
7130                let mut legacy = Vec::with_capacity(source.legacy_members.len());
7131                for expected in source.legacy_members.as_slice() {
7132                    let record = actual
7133                        .remove(&expected.coord)
7134                        .map(|(_, record)| record)
7135                        .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7136                    legacy.push(
7137                        self.graph
7138                            .analyze_fragmented_exact_replay_record(
7139                                &self.resolver,
7140                                sheet_id,
7141                                source.source_id,
7142                                *expected,
7143                                record,
7144                            )
7145                            .map_err(|error| format!("{error:?}"))?,
7146                    );
7147                }
7148                Ok::<_, String>((prepared_family, legacy))
7149            })();
7150            match result {
7151                Ok((prepared_family, legacy)) => preparation.fragmented.push(
7152                    crate::engine::formula_source::PreparedFragmentedSourceProposal {
7153                        source: source.clone(),
7154                        prepared: prepared_family,
7155                        legacy,
7156                        replay: Arc::clone(&replay),
7157                    },
7158                ),
7159                Err(reason) => {
7160                    preparation_failed = true;
7161                    for count in &mut preparation.fragmented_rejected_anchor_counts {
7162                        *count = count.saturating_add(1);
7163                    }
7164                    preparation.rejected.insert(source.source_id, reason);
7165                    preparation
7166                        .replay_disposition
7167                        .force_family_legacy(source.source_id);
7168                }
7169            }
7170        }
7171
7172        preparation.eager_replay = if preparation_failed {
7173            preparation.preparation_spool_replays = 2;
7174            replay
7175                .lock()
7176                .map_err(|_| {
7177                    ExcelError::new(ExcelErrorKind::Value)
7178                        .with_message("compressed formula exact replay lock poisoned")
7179                })?
7180                .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7181                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7182        } else {
7183            initial_replay
7184        };
7185        Ok(preparation)
7186    }
7187
7188    fn formula_batch_from_exact_replay(
7189        &mut self,
7190        sheet_name: &str,
7191        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7192    ) -> Result<FormulaIngestBatch, ExcelError> {
7193        let mut cache = rustc_hash::FxHashMap::default();
7194        let mut formulas = Vec::new();
7195        for record in replayed {
7196            let key = if record.text.starts_with('=') {
7197                record.text
7198            } else {
7199                format!("={}", record.text)
7200            };
7201            let ast_id = if let Some(cached) = cache.get(&key) {
7202                *cached
7203            } else {
7204                let parsed = match formualizer_parse::parser::parse(&key) {
7205                    Ok(parsed) => parsed,
7206                    Err(error) => {
7207                        let Some(parsed) = self.handle_formula_parse_error(
7208                            sheet_name,
7209                            record.row,
7210                            record.col,
7211                            &key,
7212                            error.to_string(),
7213                        )?
7214                        else {
7215                            continue;
7216                        };
7217                        parsed
7218                    }
7219                };
7220                let ast_id = self.intern_formula_ast(&parsed);
7221                cache.insert(key.clone(), ast_id);
7222                ast_id
7223            };
7224            formulas.push(
7225                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
7226                    .with_source_proof(record.source_order, record.family, record.partition_owner),
7227            );
7228        }
7229        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
7230    }
7231
7232    fn replay_prepared_families_exact_records(
7233        &mut self,
7234        preparation: &crate::engine::FormulaCompressedPreparation,
7235    ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
7236        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7237            ExcelError::new(ExcelErrorKind::Value)
7238                .with_message("stale compressed preparation has no exact replay owner")
7239        })?;
7240        let direct: BTreeSet<_> = preparation
7241            .prepared
7242            .iter()
7243            .map(|(family, _, _)| *family)
7244            .collect();
7245        let mut replay_disposition = preparation.replay_disposition.clone();
7246        for family in &direct {
7247            replay_disposition.force_family_legacy(*family);
7248        }
7249        let replayed = replay
7250            .lock()
7251            .map_err(|_| {
7252                ExcelError::new(ExcelErrorKind::Value)
7253                    .with_message("compressed formula exact replay lock poisoned")
7254            })?
7255            .replay(&replay_disposition)
7256            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7257        let mut records: Vec<_> = replayed
7258            .into_iter()
7259            .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
7260            .collect();
7261        records.sort_by_key(|record| record.source_order);
7262        let expected: u64 = preparation
7263            .prepared
7264            .iter()
7265            .map(|(_, _, prepared)| prepared.member_count)
7266            .sum();
7267        if records.len() as u64 != expected {
7268            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7269                "compressed formula exact replay incomplete: expected {expected}, got {}",
7270                records.len()
7271            )));
7272        }
7273        Ok(records)
7274    }
7275
7276    fn replay_one_prepared_family_exact(
7277        &mut self,
7278        preparation: &crate::engine::FormulaCompressedPreparation,
7279        family: crate::engine::SourceFamilyId,
7280        member_count: u64,
7281    ) -> Result<FormulaIngestBatch, ExcelError> {
7282        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7283            ExcelError::new(ExcelErrorKind::Value)
7284                .with_message("stale compressed preparation has no exact replay owner")
7285        })?;
7286        let mut replay_disposition = preparation.replay_disposition.clone();
7287        replay_disposition.force_family_legacy(family);
7288        let replayed = replay
7289            .lock()
7290            .map_err(|_| {
7291                ExcelError::new(ExcelErrorKind::Value)
7292                    .with_message("compressed formula exact replay lock poisoned")
7293            })?
7294            .replay(&replay_disposition)
7295            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7296            .into_iter()
7297            .filter(|record| record.family == Some(family));
7298        let batch =
7299            self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
7300        if batch.formulas.len() as u64 != member_count {
7301            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7302                "compressed formula exact replay incomplete: expected {member_count}, got {}",
7303                batch.formulas.len()
7304            )));
7305        }
7306        Ok(batch)
7307    }
7308
7309    fn exact_replay_record_is_prepared_partition_legacy(
7310        source: &crate::engine::PartitionedSourceFormulaFamily,
7311        record: &crate::engine::DeferredReplayFormula,
7312    ) -> bool {
7313        if record.partition_owner != Some(source.source_id) {
7314            return false;
7315        }
7316        let Some(coord) = record
7317            .row
7318            .checked_sub(1)
7319            .zip(record.col.checked_sub(1))
7320            .map(|(row, col)| crate::engine::SourceCoord { row, col })
7321        else {
7322            return false;
7323        };
7324        source.legacy_members.as_slice().iter().any(|member| {
7325            member.coord == coord
7326                && match member.kind {
7327                    crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
7328                        record.family == Some(source.source_id)
7329                    }
7330                    crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
7331                        record.family.is_none()
7332                    }
7333                }
7334        })
7335    }
7336
7337    fn validate_whole_partition_replay(
7338        source: &crate::engine::PartitionedSourceFormulaFamily,
7339        records: &[crate::engine::DeferredReplayFormula],
7340    ) -> Result<(), ExcelError> {
7341        let mut coordinates = BTreeSet::new();
7342        let mut shared = 0u64;
7343        let mut ordinary = 0u64;
7344        for record in records {
7345            let coord = record
7346                .row
7347                .checked_sub(1)
7348                .zip(record.col.checked_sub(1))
7349                .map(|(row, col)| crate::engine::SourceCoord { row, col })
7350                .ok_or_else(|| {
7351                    ExcelError::new(ExcelErrorKind::Value)
7352                        .with_message("whole-family replay contains an invalid coordinate")
7353                })?;
7354            if !coordinates.insert(coord) {
7355                return Err(ExcelError::new(ExcelErrorKind::Value)
7356                    .with_message("whole-family replay contains a duplicate coordinate"));
7357            }
7358            match (record.family, record.partition_owner) {
7359                (Some(family), Some(owner))
7360                    if family == source.source_id && owner == source.source_id =>
7361                {
7362                    let direct = source.fragments.iter().any(|fragment| {
7363                        let rect = fragment.rect();
7364                        coord.row >= rect.start.row
7365                            && coord.row <= rect.end.row
7366                            && coord.col >= rect.start.col
7367                            && coord.col <= rect.end.col
7368                    });
7369                    let legacy = source.legacy_members.as_slice().iter().any(|member| {
7370                        member.coord == coord
7371                            && member.kind
7372                                == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7373                    });
7374                    if !direct && !legacy {
7375                        return Err(ExcelError::new(ExcelErrorKind::Value)
7376                            .with_message("whole-family replay contains an extra shared member"));
7377                    }
7378                    shared = shared.saturating_add(1);
7379                }
7380                (None, Some(owner)) if owner == source.source_id => {
7381                    if !source.legacy_members.as_slice().iter().any(|member| {
7382                        member.coord == coord
7383                            && member.kind
7384                                == crate::engine::PartitionLegacyMemberKind::OrdinaryException
7385                    }) {
7386                        return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7387                            "whole-family replay contains an extra ordinary exception",
7388                        ));
7389                    }
7390                    ordinary = ordinary.saturating_add(1);
7391                }
7392                _ => {
7393                    return Err(ExcelError::new(ExcelErrorKind::Value)
7394                        .with_message("whole-family replay ownership mismatch"));
7395                }
7396            }
7397        }
7398        if shared != source.surviving_member_count
7399            || ordinary != source.reconciliation.ordinary_exceptions
7400        {
7401            return Err(ExcelError::new(ExcelErrorKind::Value)
7402                .with_message("whole-family replay is incomplete"));
7403        }
7404        Ok(())
7405    }
7406
7407    fn prepare_target_combined_legacy_graph(
7408        &self,
7409        packages: &[PreparedTargetSourcePackage],
7410        ordinary: &[PreparedOrdinaryStagedFormula],
7411    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
7412        let mut planned_by_coord = BTreeMap::new();
7413        for package in packages {
7414            for (row, col, ast_id, plan) in &package.legacy {
7415                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
7416            }
7417        }
7418        for formula in ordinary {
7419            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
7420                planned_by_coord.insert(
7421                    (formula.sheet_id, formula.lease.row, formula.lease.col),
7422                    (ast_id, plan),
7423                );
7424            }
7425        }
7426        let planned = planned_by_coord
7427            .into_iter()
7428            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
7429            .collect::<Vec<_>>();
7430        let formula_count = planned.len();
7431        let graph = self
7432            .graph
7433            .prepare_legacy_graph_plan_multi_sheet(planned)
7434            .map_err(|error| {
7435                ExcelError::new(ExcelErrorKind::Value)
7436                    .with_message(format!("target graph preparation failed: {error}"))
7437            })?;
7438        Ok((graph, formula_count))
7439    }
7440
7441    fn materialize_target_package_direct_records(
7442        &mut self,
7443        package: &mut PreparedTargetSourcePackage,
7444        assumptions: &crate::engine::PreparationRevision,
7445        planning_requests: &mut BTreeSet<(String, String, usize)>,
7446        deadline: Option<std::time::Instant>,
7447    ) -> Result<(), ExcelError> {
7448        let existing = package
7449            .legacy
7450            .iter()
7451            .map(|(row, col, _, _)| (*row, *col))
7452            .collect::<BTreeSet<_>>();
7453        let mut missing = BTreeMap::new();
7454        for record in &package.replay_records {
7455            if !existing.contains(&(record.row, record.col)) {
7456                missing.insert((record.row, record.col), record.clone());
7457            }
7458        }
7459        let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
7460        for record in batch.formulas {
7461            self.target_preparation_checkpoint(deadline, 1)?;
7462            let ast = self
7463                .graph
7464                .data_store()
7465                .retrieve_ast(record.ast_id, self.graph.sheet_reg())
7466                .ok_or_else(|| {
7467                    ExcelError::new(ExcelErrorKind::Value)
7468                        .with_message("target fallback AST is unavailable")
7469                })?;
7470            let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
7471            if let Some(reason) =
7472                Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
7473            {
7474                return Err(Self::preparation_stale(
7475                    reason,
7476                    "target fallback planning snapshot became stale during materialization",
7477                ));
7478            }
7479            let placement = CellRef::new(
7480                package.sheet_id,
7481                Coord::from_excel(record.row, record.col, true, true),
7482            );
7483            let ingested = self
7484                .graph
7485                .ingest_pipeline(&snapshot)
7486                .enable_function_semantics()
7487                .ingest_formula(
7488                    FormulaAstInput::RawArena(record.ast_id),
7489                    placement,
7490                    record.formula_text,
7491                )?;
7492            package
7493                .legacy
7494                .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7495        }
7496        package.direct_families = 0;
7497        package.direct_cells = 0;
7498        package.direct_fragments = 0;
7499        package.direct_complete_families = 0;
7500        package.direct_complete_cells = 0;
7501        package.direct_partition_families = 0;
7502        package.direct_partition_cells = 0;
7503        Ok(())
7504    }
7505
7506    fn prepare_target_source_package(
7507        &mut self,
7508        sheet: &str,
7509        lease: StagedPackageLease,
7510        deadline: Option<std::time::Instant>,
7511    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
7512        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
7513            ExcelError::new(ExcelErrorKind::Ref)
7514                .with_message(format!("deferred source sheet not found: {sheet}"))
7515        })?;
7516        let (
7517            mut source_report,
7518            families,
7519            partitions,
7520            replay,
7521            invalidated,
7522            suppressed,
7523            reconciliation_replay,
7524        ) = {
7525            let package = self
7526                .staged_formulas
7527                .get(sheet)
7528                .and_then(|staged| staged.deferred_package.as_ref())
7529                .ok_or_else(|| {
7530                    ExcelError::new(ExcelErrorKind::Value)
7531                        .with_message("staged deferred source package is unavailable")
7532                })?;
7533            if package.sheet_name != sheet {
7534                return Err(ExcelError::new(ExcelErrorKind::Value)
7535                    .with_message("deferred formula package sheet mismatch"));
7536            }
7537            (
7538                package.report.clone(),
7539                package.families.clone(),
7540                package.partitioned_families.clone(),
7541                Arc::clone(&package.replay),
7542                package.invalidated.clone(),
7543                package.suppressed.clone(),
7544                package.reconciliation_replay.clone(),
7545            )
7546        };
7547
7548        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
7549        for partition in &partitions {
7550            replay_disposition
7551                .register_partition(partition, false)
7552                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
7553        }
7554        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
7555        self.target_preparation_checkpoint(deadline, 1)?;
7556        let mut replay_records = if let Some(mut records) = reconciliation_replay {
7557            records.retain(|record| {
7558                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
7559                else {
7560                    return true;
7561                };
7562                let coord = crate::engine::SourceCoord { row, col };
7563                let disposition = record.family.map_or_else(
7564                    || replay_disposition.ordinary_disposition(coord).0,
7565                    |family| replay_disposition.shared_disposition(family, coord),
7566                );
7567                !matches!(
7568                    disposition,
7569                    crate::engine::FormulaReplayCoordinateDisposition::Direct
7570                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
7571                )
7572            });
7573            records
7574        } else {
7575            replay
7576                .lock()
7577                .map_err(|_| {
7578                    ExcelError::new(ExcelErrorKind::Value)
7579                        .with_message("deferred formula spool lock poisoned")
7580                })?
7581                .replay_partitioned(&replay_disposition, &partitions)
7582                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7583        };
7584        replay_records.sort_by_key(|record| record.source_order);
7585        for chunk in replay_records.chunks(256) {
7586            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
7587        }
7588        if replay_records
7589            .windows(2)
7590            .any(|records| records[0].source_order == records[1].source_order)
7591        {
7592            return Err(ExcelError::new(ExcelErrorKind::Value)
7593                .with_message("duplicate deferred source-order proof"));
7594        }
7595
7596        let mut disposition = replay_disposition;
7597        let mut ordered_placements = Vec::new();
7598        let mut direct_families = 0usize;
7599        let mut direct_cells = 0u64;
7600        let mut direct_fragments = 0u64;
7601        let mut direct_complete_families = 0u64;
7602        let mut direct_complete_cells = 0u64;
7603        let mut direct_partition_families = 0u64;
7604        let mut direct_partition_cells = 0u64;
7605        let mut anchor_parses = 0u64;
7606        let mut anchor_asts = 0u64;
7607        let mut anchor_analyses = 0u64;
7608        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
7609            for family in &families {
7610                self.target_preparation_checkpoint(deadline, 1)?;
7611                if invalidated.contains(&family.source_id) {
7612                    continue;
7613                }
7614                match self.prepare_source_formula_family(sheet_id, family, true, None) {
7615                    Ok((prepared, _)) => {
7616                        anchor_parses = anchor_parses.saturating_add(1);
7617                        anchor_asts = anchor_asts.saturating_add(1);
7618                        anchor_analyses = anchor_analyses.saturating_add(1);
7619                        direct_families = direct_families.saturating_add(1);
7620                        direct_cells = direct_cells.saturating_add(prepared.member_count);
7621                        direct_complete_families = direct_complete_families.saturating_add(1);
7622                        direct_complete_cells =
7623                            direct_complete_cells.saturating_add(prepared.member_count);
7624                        if self.config.formula_plane_mode
7625                            == FormulaPlaneMode::AuthoritativeExperimental
7626                        {
7627                            disposition.set_family_direct(family.source_id);
7628                        }
7629                        ordered_placements.push((family.source_order, vec![prepared]));
7630                    }
7631                    Err(error) => {
7632                        anchor_parses =
7633                            anchor_parses.saturating_add(u64::from(error.parse_attempted));
7634                        anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7635                        anchor_analyses =
7636                            anchor_analyses.saturating_add(u64::from(error.analysis_created));
7637                        *source_report
7638                            .fallback_reasons
7639                            .entry(error.reason)
7640                            .or_default() += 1;
7641                    }
7642                }
7643            }
7644
7645            for source in &partitions {
7646                self.target_preparation_checkpoint(deadline, 1)?;
7647                if invalidated.contains(&source.source_id) {
7648                    continue;
7649                }
7650                let prepared =
7651                    match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
7652                        Ok(prepared) => {
7653                            anchor_parses = anchor_parses.saturating_add(1);
7654                            anchor_asts = anchor_asts.saturating_add(1);
7655                            anchor_analyses = anchor_analyses.saturating_add(1);
7656                            prepared
7657                        }
7658                        Err(error) => {
7659                            anchor_parses =
7660                                anchor_parses.saturating_add(u64::from(error.parse_attempted));
7661                            anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7662                            anchor_analyses =
7663                                anchor_analyses.saturating_add(u64::from(error.analysis_created));
7664                            *source_report
7665                                .fallback_reasons
7666                                .entry(error.reason)
7667                                .or_default() += 1;
7668                            continue;
7669                        }
7670                    };
7671                let fragment_regions: Vec<_> = prepared
7672                    .placements
7673                    .iter()
7674                    .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
7675                    .collect();
7676                let crosses_fragment =
7677                    prepared
7678                        .placements
7679                        .iter()
7680                        .enumerate()
7681                        .any(|(fragment_index, placement)| {
7682                            placement
7683                                .fragment_dependency_proof()
7684                                .1
7685                                .dependencies
7686                                .iter()
7687                                .any(|dependency| {
7688                                    fragment_regions.iter().enumerate().any(
7689                                        |(candidate_index, candidate)| {
7690                                            candidate_index != fragment_index
7691                                                && dependency.read_region.intersects(candidate)
7692                                        },
7693                                    )
7694                                })
7695                        });
7696                let family_records = replay_records
7697                    .iter()
7698                    .filter(|record| {
7699                        record.family == Some(source.source_id)
7700                            || record.partition_owner == Some(source.source_id)
7701                    })
7702                    .cloned()
7703                    .collect::<Vec<_>>();
7704                if crosses_fragment {
7705                    *source_report
7706                        .fallback_reasons
7707                        .entry("CrossFragmentDependency".to_string())
7708                        .or_default() += 1;
7709                    continue;
7710                }
7711                if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
7712                    *source_report
7713                        .fallback_reasons
7714                        .entry(format!("TargetPartitionReplay:{error}"))
7715                        .or_default() += 1;
7716                    continue;
7717                }
7718                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
7719                    let mut candidate = disposition.clone();
7720                    if let Err(reason) = candidate.register_partition(source, true) {
7721                        *source_report
7722                            .fallback_reasons
7723                            .entry(reason.to_string())
7724                            .or_default() += 1;
7725                        continue;
7726                    }
7727                    disposition = candidate;
7728                }
7729                direct_families = direct_families.saturating_add(1);
7730                direct_cells = direct_cells.saturating_add(prepared.direct_cells);
7731                direct_partition_families = direct_partition_families.saturating_add(1);
7732                direct_partition_cells =
7733                    direct_partition_cells.saturating_add(prepared.direct_cells);
7734                direct_fragments =
7735                    direct_fragments.saturating_add(prepared.placements.len() as u64);
7736                ordered_placements.push((source.source_order, prepared.placements));
7737            }
7738        }
7739        ordered_placements.sort_by_key(|(source_order, _)| *source_order);
7740        let placements = ordered_placements
7741            .into_iter()
7742            .flat_map(|(_, placements)| placements)
7743            .collect();
7744
7745        Ok(PreparedTargetSourcePackage {
7746            sheet: sheet.to_string(),
7747            sheet_id,
7748            lease,
7749            source_report,
7750            replay_records,
7751            disposition,
7752            placements,
7753            legacy: Vec::new(),
7754            direct_families,
7755            direct_cells,
7756            direct_fragments,
7757            direct_complete_families,
7758            direct_complete_cells,
7759            direct_partition_families,
7760            direct_partition_cells,
7761            anchor_parses,
7762            anchor_asts,
7763            anchor_analyses,
7764        })
7765    }
7766
7767    fn fallback_planning_snapshot(
7768        &self,
7769        batch: &FormulaIngestBatch,
7770    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7771        let mut requests = Vec::new();
7772        for formula in &batch.formulas {
7773            let ast = self
7774                .graph
7775                .data_store()
7776                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
7777                .ok_or_else(|| {
7778                    ExcelError::new(ExcelErrorKind::Value)
7779                        .with_message("ordered fallback AST is unavailable")
7780                })?;
7781            Self::collect_planning_function_requests(&ast, &mut requests);
7782        }
7783        requests.sort();
7784        requests.dedup();
7785        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7786            &self.resolver,
7787            requests,
7788        )
7789        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
7790    }
7791
7792    fn prepare_legacy_batch_fallback(
7793        &mut self,
7794        batch: FormulaIngestBatch,
7795        function_provider: &dyn crate::traits::FunctionProvider,
7796    ) -> Result<
7797        (
7798            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
7799            u64,
7800        ),
7801        ExcelError,
7802    > {
7803        let formula_count = batch.formulas.len() as u64;
7804        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7805        let mut planned = Vec::with_capacity(batch.formulas.len());
7806        for record in batch.formulas {
7807            let placement = CellRef::new(
7808                sheet_id,
7809                Coord::from_excel(record.row, record.col, true, true),
7810            );
7811            let ingested = self
7812                .graph
7813                .ingest_pipeline(function_provider)
7814                .enable_function_semantics()
7815                .ingest_formula(
7816                    FormulaAstInput::RawArena(record.ast_id),
7817                    placement,
7818                    record.formula_text,
7819                )
7820                .map_err(|error| {
7821                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
7822                })?;
7823            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7824        }
7825        let plan = self
7826            .graph
7827            .prepare_legacy_graph_plan(sheet_id, planned)
7828            .map_err(|error| {
7829                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
7830            })?;
7831        Ok((plan, formula_count))
7832    }
7833
7834    fn publish_compressed_partial_report(
7835        &mut self,
7836        report: &FormulaIngestReport,
7837        direct_report: &FormulaIngestReport,
7838    ) {
7839        if direct_report.source_family_promoted == 0
7840            && direct_report.graph_formula_cells_materialized == 0
7841        {
7842            return;
7843        }
7844        let mut published = report.clone();
7845        published.accumulate(direct_report);
7846        self.record_formula_ingest_report(published);
7847    }
7848
7849    fn finish_compressed_formula_sources(
7850        &mut self,
7851        batches: Vec<(
7852            FormulaIngestBatch,
7853            crate::engine::FormulaCompressedSourceReport,
7854            crate::engine::FormulaCompressedPreparation,
7855        )>,
7856    ) -> Result<FormulaIngestReport, ExcelError> {
7857        self.observe_function_semantic_epoch()?;
7858        if batches.iter().any(|(_, _, preparation)| {
7859            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7860        }) {
7861            return Err(ExcelError::new(ExcelErrorKind::Value)
7862                .with_message("compressed source preparation belongs to another engine"));
7863        }
7864        if batches.iter().any(|(fallback, _, preparation)| {
7865            preparation.sheet_name.as_ref() != fallback.sheet_name
7866        }) {
7867            return Err(ExcelError::new(ExcelErrorKind::Value)
7868                .with_message("compressed source preparation sheet mismatch"));
7869        }
7870        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7871        let initial_provider_revision = self.resolver.planning_semantic_revision();
7872        let mut fallback_batches = Vec::with_capacity(batches.len());
7873        let mut stale_fallback_batches = Vec::new();
7874        let mut pending_preparations = Vec::new();
7875        for (mut fallback, mut source, mut preparation) in batches {
7876            for formula in fallback.formulas.drain(..) {
7877                let source_order = formula.source_order.ok_or_else(|| {
7878                    ExcelError::new(ExcelErrorKind::Value).with_message(
7879                        "compressed source supplied formulas without source-order proof",
7880                    )
7881                })?;
7882                let text = formula.formula_text.ok_or_else(|| {
7883                    ExcelError::new(ExcelErrorKind::Value).with_message(
7884                        "ordered compressed fallback formula has no exact source text",
7885                    )
7886                })?;
7887                preparation
7888                    .eager_replay
7889                    .push(crate::engine::DeferredReplayFormula {
7890                        source_order,
7891                        row: formula.row,
7892                        col: formula.col,
7893                        text: text.to_string(),
7894                        family: formula.source_family,
7895                        partition_owner: formula.partition_owner,
7896                    });
7897            }
7898            preparation
7899                .eager_replay
7900                .sort_by_key(|record| record.source_order);
7901            source.source_spool_replays = source
7902                .source_spool_replays
7903                .saturating_add(preparation.preparation_spool_replays);
7904            let stale_reason = preparation
7905                .function_semantics_used
7906                .then(|| {
7907                    if preparation.function_provider_revision != initial_provider_revision {
7908                        Some("FunctionProviderRevisionChanged")
7909                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7910                    {
7911                        Some("FunctionSemanticEpochChanged")
7912                    } else {
7913                        None
7914                    }
7915                })
7916                .flatten();
7917            let stale_semantics = stale_reason.is_some();
7918            if let Some(reason) = stale_reason {
7919                for (family, _, _) in &preparation.prepared {
7920                    preparation.rejected.insert(*family, reason.to_string());
7921                }
7922            }
7923            for reason in preparation.rejected.values() {
7924                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7925            }
7926            let clean_direct_cells: u64 = preparation
7927                .prepared
7928                .iter()
7929                .map(|(_, _, prepared)| prepared.member_count)
7930                .sum();
7931            let fragmented_direct_cells: u64 = preparation
7932                .fragmented
7933                .iter()
7934                .map(|fragment| fragment.source.surviving_member_count)
7935                .sum();
7936            let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
7937            source.replay_families = source.families_seen.saturating_sub(
7938                preparation
7939                    .prepared
7940                    .len()
7941                    .saturating_add(preparation.fragmented.len()) as u64,
7942            );
7943            source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
7944            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
7945                fallback.sheet_name.clone(),
7946                source,
7947            );
7948            if stale_semantics {
7949                stale_fallback_batches.push((fallback, compressed));
7950            } else {
7951                fallback_batches.push((fallback, compressed));
7952            }
7953            pending_preparations.push((preparation, stale_semantics));
7954        }
7955        drop(initial_guard);
7956
7957        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
7958        let configured_mode = self.config.formula_plane_mode;
7959        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7960        let stale_result =
7961            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
7962        self.config.formula_plane_mode = configured_mode;
7963        let mut report = stale_result?;
7964        report.mode = configured_mode;
7965
7966        self.config.formula_plane_mode = FormulaPlaneMode::Off;
7967        let fallback_result =
7968            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
7969        self.config.formula_plane_mode = configured_mode;
7970        match fallback_result {
7971            Ok(fallback_report) => report.accumulate(&fallback_report),
7972            Err(error) => {
7973                self.record_formula_ingest_report(report);
7974                return Err(error);
7975            }
7976        }
7977
7978        let mut direct_report =
7979            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7980        for (preparation, _) in &pending_preparations {
7981            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
7982                preparation.clean_rejected_anchor_counts[0]
7983                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
7984            );
7985            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
7986                preparation.clean_rejected_anchor_counts[1]
7987                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
7988            );
7989            direct_report.source_anchor_analyses =
7990                direct_report.source_anchor_analyses.saturating_add(
7991                    preparation.clean_rejected_anchor_counts[2]
7992                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
7993                );
7994            let rejected_partitions: Vec<_> = preparation
7995                .rejected
7996                .keys()
7997                .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
7998                .collect();
7999            direct_report.source_partitioned_families_rejected = direct_report
8000                .source_partitioned_families_rejected
8001                .saturating_add(rejected_partitions.len() as u64);
8002            direct_report.source_partition_failures = direct_report
8003                .source_partition_failures
8004                .saturating_add(rejected_partitions.len() as u64);
8005            direct_report.source_partition_fallback_cells = direct_report
8006                .source_partition_fallback_cells
8007                .saturating_add(
8008                    rejected_partitions
8009                        .iter()
8010                        .map(|source| source.surviving_member_count)
8011                        .sum::<u64>(),
8012                );
8013        }
8014        loop {
8015            #[cfg(any(test, feature = "test-support"))]
8016            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8017                hook();
8018            }
8019            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8020            let commit_provider_revision = self.resolver.planning_semantic_revision();
8021            let mut newly_stale = Vec::new();
8022            let mut current = Vec::new();
8023            for pending in pending_preparations.drain(..) {
8024                let stale_reason = pending
8025                    .0
8026                    .function_semantics_used
8027                    .then(|| {
8028                        if pending.0.function_provider_revision != commit_provider_revision {
8029                            Some("FunctionProviderRevisionChanged")
8030                        } else if self
8031                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
8032                        {
8033                            Some("FunctionSemanticEpochChanged")
8034                        } else {
8035                            None
8036                        }
8037                    })
8038                    .flatten();
8039                if let Some(reason) = stale_reason {
8040                    newly_stale.push((pending, reason));
8041                } else {
8042                    current.push(pending);
8043                }
8044            }
8045            if !newly_stale.is_empty() {
8046                drop(commit_guard);
8047                for ((mut preparation, was_initially_stale), reason) in newly_stale {
8048                    for (family, _, _) in &preparation.prepared {
8049                        preparation.rejected.insert(*family, reason.to_string());
8050                    }
8051                    let exact = match self.replay_prepared_families_exact_records(&preparation) {
8052                        Ok(exact) => exact,
8053                        Err(error) => {
8054                            self.publish_compressed_partial_report(&report, &direct_report);
8055                            return Err(error);
8056                        }
8057                    };
8058                    preparation.eager_replay.extend(exact);
8059                    preparation
8060                        .eager_replay
8061                        .sort_by_key(|record| record.source_order);
8062                    direct_report.source_spool_replays =
8063                        direct_report.source_spool_replays.saturating_add(1);
8064                    if !was_initially_stale {
8065                        *direct_report
8066                            .fallback_reasons
8067                            .entry(reason.to_string())
8068                            .or_default() += preparation.prepared.len() as u64;
8069                    }
8070                    direct_report.source_family_fallback = direct_report
8071                        .source_family_fallback
8072                        .saturating_add(preparation.prepared.len() as u64);
8073                    direct_report.source_family_fallback_cells =
8074                        direct_report.source_family_fallback_cells.saturating_add(
8075                            preparation
8076                                .prepared
8077                                .iter()
8078                                .map(|(_, _, prepared)| prepared.member_count)
8079                                .sum(),
8080                        );
8081                    preparation.prepared.clear();
8082                    preparation.function_semantics_used = false;
8083                    if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8084                        current.push((preparation, false));
8085                    }
8086                }
8087                pending_preparations = current;
8088                continue;
8089            }
8090            drop(commit_guard);
8091
8092            // Preserve source order across clean and fragmented families. Every clean
8093            // placement uses E3's incremental append, so no global index rebuild is needed.
8094            enum SourceProposal {
8095                KnownFallback {
8096                    source_order: crate::engine::SourceFormulaOrder,
8097                    records: Vec<crate::engine::DeferredReplayFormula>,
8098                },
8099                Clean(
8100                    crate::engine::SourceFamilyId,
8101                    crate::engine::SourceFormulaOrder,
8102                    crate::formula_plane::placement::PreparedAnchorOncePlacement,
8103                ),
8104                Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8105            }
8106
8107            impl SourceProposal {
8108                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8109                    match self {
8110                        Self::KnownFallback { source_order, .. } => *source_order,
8111                        Self::Clean(_, source_order, _) => *source_order,
8112                        Self::Fragment(fragment) => fragment.source.source_order,
8113                    }
8114                }
8115            }
8116
8117            for (mut preparation, _) in current {
8118                let mut proposals = Vec::with_capacity(
8119                    preparation
8120                        .prepared
8121                        .len()
8122                        .saturating_add(preparation.fragmented.len())
8123                        .saturating_add(preparation.eager_replay.len()),
8124                );
8125                proposals.extend(preparation.prepared.drain(..).map(
8126                    |(source_id, source_order, prepared)| {
8127                        SourceProposal::Clean(source_id, source_order, prepared)
8128                    },
8129                ));
8130
8131                let accepted_fragments: BTreeMap<_, _> = preparation
8132                    .fragmented
8133                    .iter()
8134                    .map(|fragment| (fragment.source.source_id, &fragment.source))
8135                    .collect();
8136                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8137                for record in preparation.eager_replay.drain(..).filter(|record| {
8138                    !record.partition_owner.is_some_and(|owner| {
8139                        accepted_fragments.get(&owner).is_some_and(|source| {
8140                            Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8141                        })
8142                    })
8143                }) {
8144                    if let Some(owner) = record.partition_owner.or(record.family) {
8145                        family_fallbacks.entry(owner).or_default().push(record);
8146                    } else {
8147                        proposals.push(SourceProposal::KnownFallback {
8148                            source_order: record.source_order,
8149                            records: vec![record],
8150                        });
8151                    }
8152                }
8153                for (_, mut records) in family_fallbacks {
8154                    records.sort_by_key(|record| record.source_order);
8155                    if records
8156                        .windows(2)
8157                        .any(|window| window[0].source_order == window[1].source_order)
8158                    {
8159                        self.publish_compressed_partial_report(&report, &direct_report);
8160                        return Err(ExcelError::new(ExcelErrorKind::Value)
8161                            .with_message("duplicate exact-replay source-order proof"));
8162                    }
8163                    let Some(source_order) = records.first().map(|record| record.source_order)
8164                    else {
8165                        self.publish_compressed_partial_report(&report, &direct_report);
8166                        return Err(ExcelError::new(ExcelErrorKind::Value)
8167                            .with_message("empty exact-replay fallback family"));
8168                    };
8169                    proposals.push(SourceProposal::KnownFallback {
8170                        source_order,
8171                        records,
8172                    });
8173                }
8174                proposals.extend(
8175                    preparation
8176                        .fragmented
8177                        .drain(..)
8178                        .map(SourceProposal::Fragment),
8179                );
8180                proposals.sort_by_key(SourceProposal::source_order);
8181                if proposals
8182                    .windows(2)
8183                    .any(|window| window[0].source_order() == window[1].source_order())
8184                {
8185                    self.publish_compressed_partial_report(&report, &direct_report);
8186                    return Err(ExcelError::new(ExcelErrorKind::Value)
8187                        .with_message("ambiguous compressed source-order proof"));
8188                }
8189
8190                for proposal in proposals {
8191                    match proposal {
8192                        SourceProposal::KnownFallback { records, .. } => {
8193                            let batch = match self.formula_batch_from_exact_replay(
8194                                preparation.sheet_name.as_ref(),
8195                                records,
8196                            ) {
8197                                Ok(batch) => batch,
8198                                Err(error) => {
8199                                    self.publish_compressed_partial_report(&report, &direct_report);
8200                                    return Err(error);
8201                                }
8202                            };
8203                            if batch.is_empty() {
8204                                continue;
8205                            }
8206                            let snapshot = match self.fallback_planning_snapshot(&batch) {
8207                                Ok(snapshot) => snapshot,
8208                                Err(error) => {
8209                                    self.publish_compressed_partial_report(&report, &direct_report);
8210                                    return Err(error);
8211                                }
8212                            };
8213                            let commit_guard =
8214                                crate::function_registry::semantic_epoch_read_guard();
8215                            let provider_revision_initial =
8216                                self.resolver.planning_semantic_revision();
8217                            if (commit_guard.epoch() != snapshot.epoch()
8218                                && snapshot.semantic_changes_affect_requests_since_guarded(
8219                                    &commit_guard,
8220                                    snapshot.epoch(),
8221                                ))
8222                                || snapshot.provider_revision().is_some_and(|revision| {
8223                                    Some(revision) != provider_revision_initial
8224                                })
8225                            {
8226                                drop(commit_guard);
8227                                self.publish_compressed_partial_report(&report, &direct_report);
8228                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8229                                    "ordered fallback planning snapshot became stale",
8230                                ));
8231                            }
8232                            let (plan, formula_count) = match self
8233                                .prepare_legacy_batch_fallback(batch, &snapshot)
8234                            {
8235                                Ok(plan) => plan,
8236                                Err(error) => {
8237                                    drop(commit_guard);
8238                                    self.publish_compressed_partial_report(&report, &direct_report);
8239                                    return Err(error);
8240                                }
8241                            };
8242                            let provider_revision_after =
8243                                self.resolver.planning_semantic_revision();
8244                            if provider_revision_after != provider_revision_initial {
8245                                drop(commit_guard);
8246                                self.publish_compressed_partial_report(&report, &direct_report);
8247                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8248                                    "function provider changed while preparing ordered fallback",
8249                                ));
8250                            }
8251                            let graph_vertices = plan.new_vertex_count();
8252                            let Some(graph_edges) = plan.planned_edge_count() else {
8253                                drop(commit_guard);
8254                                self.publish_compressed_partial_report(&report, &direct_report);
8255                                return Err(ExcelError::new(ExcelErrorKind::Value)
8256                                    .with_message("prepared ordered fallback size overflow"));
8257                            };
8258                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
8259                            {
8260                                drop(commit_guard);
8261                                self.publish_compressed_partial_report(&report, &direct_report);
8262                                return Err(error);
8263                            }
8264                            if let Err(error) =
8265                                self.graph.validate_prepared_legacy_graph_plan(&plan)
8266                            {
8267                                drop(commit_guard);
8268                                self.publish_compressed_partial_report(&report, &direct_report);
8269                                return Err(ExcelError::new(ExcelErrorKind::Value)
8270                                    .with_message(error.to_string()));
8271                            }
8272                            #[cfg(test)]
8273                            if let Some(hook) = self
8274                                .before_legacy_fallback_final_provider_sample_hook
8275                                .take()
8276                            {
8277                                hook();
8278                            }
8279                            let provider_revision_final =
8280                                self.resolver.planning_semantic_revision();
8281                            if provider_revision_final != provider_revision_initial {
8282                                drop(commit_guard);
8283                                self.publish_compressed_partial_report(&report, &direct_report);
8284                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8285                                    "function provider changed after ordered fallback validation",
8286                                ));
8287                            }
8288                            let graph_formulas =
8289                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
8290                            direct_report.formula_cells_seen = direct_report
8291                                .formula_cells_seen
8292                                .saturating_add(formula_count);
8293                            direct_report.graph_formula_cells_materialized = direct_report
8294                                .graph_formula_cells_materialized
8295                                .saturating_add(graph_formulas as u64);
8296                            direct_report.graph_vertices_created = direct_report
8297                                .graph_vertices_created
8298                                .saturating_add(graph_vertices as u64);
8299                            direct_report.graph_edges_created = direct_report
8300                                .graph_edges_created
8301                                .saturating_add(graph_edges as u64);
8302                        }
8303                        SourceProposal::Clean(source_id, _, prepared) => {
8304                            let commit_guard =
8305                                crate::function_registry::semantic_epoch_read_guard();
8306                            let commit_provider_revision =
8307                                self.resolver.planning_semantic_revision();
8308                            let semantic_changed = preparation.function_semantics_used
8309                                && self.prepared_placement_function_semantics_changed(
8310                                    preparation.function_semantic_epoch,
8311                                    &prepared,
8312                                    &commit_guard,
8313                                );
8314                            let cells = prepared.member_count;
8315                            let stats = self.graph.baseline_stats();
8316                            self.preflight_graph_admission(
8317                                crate::engine::resource_ledger::GraphAdmission {
8318                                    final_vertices: stats.graph_vertex_count,
8319                                    final_edges: stats.graph_edge_count,
8320                                    materialization_cells: 0,
8321                                    added_vertices: 0,
8322                                    added_edges: 0,
8323                                },
8324                            )?;
8325                            let append = self
8326                                .graph
8327                                .formula_authority()
8328                                .prepare_formula_plane_append(
8329                                    vec![prepared],
8330                                    self.graph.data_store(),
8331                                    self.graph.sheet_reg(),
8332                                )
8333                                .map_err(|error| error.to_string())
8334                                .and_then(|append| {
8335                                    if preparation.function_semantics_used
8336                                        && preparation.function_provider_revision
8337                                            != commit_provider_revision
8338                                    {
8339                                        return Err("FunctionProviderRevisionChanged".to_string());
8340                                    }
8341                                    if preparation.function_semantics_used
8342                                        && self.resolver.planning_semantic_revision()
8343                                            != commit_provider_revision
8344                                    {
8345                                        return Err("FunctionProviderRevisionChanged".to_string());
8346                                    }
8347                                    if semantic_changed {
8348                                        return Err("FunctionSemanticEpochChanged".to_string());
8349                                    }
8350                                    self.graph
8351                                        .formula_authority()
8352                                        .validate_prepared_formula_plane_append(
8353                                            &append,
8354                                            self.graph.data_store(),
8355                                            self.graph.sheet_reg(),
8356                                        )
8357                                        .map_err(|error| error.to_string())?;
8358                                    Ok(append)
8359                                });
8360                            match append {
8361                                Ok(append) => {
8362                                    let placement_report = self
8363                                        .graph
8364                                        .formula_authority_mut()
8365                                        .apply_prevalidated_formula_plane_append(append);
8366                                    self.graph.mark_formula_spans_dirty(
8367                                        placement_report.spans.iter().copied(),
8368                                        WholeSpanDirtyReason::NewSpan,
8369                                    );
8370                                    direct_report.formula_cells_seen =
8371                                        direct_report.formula_cells_seen.saturating_add(cells);
8372                                    direct_report.shadow_candidate_cells =
8373                                        direct_report.shadow_candidate_cells.saturating_add(cells);
8374                                    direct_report.shadow_accepted_span_cells = direct_report
8375                                        .shadow_accepted_span_cells
8376                                        .saturating_add(cells);
8377                                    direct_report.shadow_templates_interned =
8378                                        direct_report.shadow_templates_interned.saturating_add(
8379                                            placement_report.work.templates_to_append as u64,
8380                                        );
8381                                    direct_report.shadow_spans_created = direct_report
8382                                        .shadow_spans_created
8383                                        .saturating_add(placement_report.spans.len() as u64);
8384                                    direct_report.graph_formula_vertices_avoided_shadow =
8385                                        direct_report
8386                                            .graph_formula_vertices_avoided_shadow
8387                                            .saturating_add(cells);
8388                                    direct_report.ast_roots_avoided_shadow = direct_report
8389                                        .ast_roots_avoided_shadow
8390                                        .saturating_add(cells.saturating_sub(1));
8391                                    direct_report.edge_rows_avoided_shadow = direct_report
8392                                        .edge_rows_avoided_shadow
8393                                        .saturating_add(cells);
8394                                    direct_report.source_family_promoted =
8395                                        direct_report.source_family_promoted.saturating_add(1);
8396                                    direct_report.source_family_promoted_cells = direct_report
8397                                        .source_family_promoted_cells
8398                                        .saturating_add(cells);
8399                                    direct_report.source_anchor_parses =
8400                                        direct_report.source_anchor_parses.saturating_add(1);
8401                                    direct_report.source_anchor_asts =
8402                                        direct_report.source_anchor_asts.saturating_add(1);
8403                                    direct_report.source_anchor_analyses =
8404                                        direct_report.source_anchor_analyses.saturating_add(1);
8405                                    let descendants = cells.saturating_sub(1);
8406                                    direct_report.source_descendant_strings_avoided = direct_report
8407                                        .source_descendant_strings_avoided
8408                                        .saturating_add(descendants);
8409                                    direct_report.source_descendant_events_avoided = direct_report
8410                                        .source_descendant_events_avoided
8411                                        .saturating_add(descendants);
8412                                    direct_report.source_descendant_analyses_avoided =
8413                                        direct_report
8414                                            .source_descendant_analyses_avoided
8415                                            .saturating_add(descendants);
8416                                    direct_report.source_compressed_families_prepared =
8417                                        direct_report
8418                                            .source_compressed_families_prepared
8419                                            .saturating_add(1);
8420                                    direct_report.source_compressed_cells_prepared = direct_report
8421                                        .source_compressed_cells_prepared
8422                                        .saturating_add(cells);
8423                                }
8424                                Err(error) => {
8425                                    drop(commit_guard);
8426                                    let reason = format!("CleanFormulaPlaneAppend:{error}");
8427                                    let batch = match self.replay_one_prepared_family_exact(
8428                                        &preparation,
8429                                        source_id,
8430                                        cells,
8431                                    ) {
8432                                        Ok(batch) => batch,
8433                                        Err(error) => {
8434                                            self.publish_compressed_partial_report(
8435                                                &report,
8436                                                &direct_report,
8437                                            );
8438                                            return Err(error);
8439                                        }
8440                                    };
8441                                    let snapshot = match self.fallback_planning_snapshot(&batch) {
8442                                        Ok(snapshot) => snapshot,
8443                                        Err(error) => {
8444                                            self.publish_compressed_partial_report(
8445                                                &report,
8446                                                &direct_report,
8447                                            );
8448                                            return Err(error);
8449                                        }
8450                                    };
8451                                    let commit_guard =
8452                                        crate::function_registry::semantic_epoch_read_guard();
8453                                    let provider_revision =
8454                                        self.resolver.planning_semantic_revision();
8455                                    if (commit_guard.epoch() != snapshot.epoch()
8456                                        && snapshot.semantic_changes_affect_requests_since_guarded(
8457                                            &commit_guard,
8458                                            snapshot.epoch(),
8459                                        ))
8460                                        || snapshot.provider_revision().is_some_and(|revision| {
8461                                            Some(revision) != provider_revision
8462                                        })
8463                                    {
8464                                        drop(commit_guard);
8465                                        self.publish_compressed_partial_report(
8466                                            &report,
8467                                            &direct_report,
8468                                        );
8469                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8470                                            .with_message(
8471                                                "clean-family fallback planning snapshot became stale",
8472                                            ));
8473                                    }
8474                                    let (plan, formula_count) = match self
8475                                        .prepare_legacy_batch_fallback(batch, &snapshot)
8476                                    {
8477                                        Ok(plan) => plan,
8478                                        Err(error) => {
8479                                            drop(commit_guard);
8480                                            self.publish_compressed_partial_report(
8481                                                &report,
8482                                                &direct_report,
8483                                            );
8484                                            return Err(error);
8485                                        }
8486                                    };
8487                                    let provider_revision_after =
8488                                        self.resolver.planning_semantic_revision();
8489                                    if provider_revision_after != provider_revision {
8490                                        drop(commit_guard);
8491                                        self.publish_compressed_partial_report(
8492                                            &report,
8493                                            &direct_report,
8494                                        );
8495                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8496                                            .with_message(
8497                                                "function provider changed while preparing clean-family fallback",
8498                                            ));
8499                                    }
8500                                    let graph_vertices = plan.new_vertex_count();
8501                                    let Some(graph_edges) = plan.planned_edge_count() else {
8502                                        drop(commit_guard);
8503                                        self.publish_compressed_partial_report(
8504                                            &report,
8505                                            &direct_report,
8506                                        );
8507                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8508                                            .with_message(
8509                                                "prepared clean-family fallback size overflow",
8510                                            ));
8511                                    };
8512                                    if let Err(error) =
8513                                        self.prepared_legacy_admission(&plan, formula_count)
8514                                    {
8515                                        drop(commit_guard);
8516                                        self.publish_compressed_partial_report(
8517                                            &report,
8518                                            &direct_report,
8519                                        );
8520                                        return Err(error);
8521                                    }
8522                                    if let Err(error) =
8523                                        self.graph.validate_prepared_legacy_graph_plan(&plan)
8524                                    {
8525                                        drop(commit_guard);
8526                                        self.publish_compressed_partial_report(
8527                                            &report,
8528                                            &direct_report,
8529                                        );
8530                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8531                                            .with_message(error.to_string()));
8532                                    }
8533                                    #[cfg(test)]
8534                                    if let Some(hook) = self
8535                                        .before_legacy_fallback_final_provider_sample_hook
8536                                        .take()
8537                                    {
8538                                        hook();
8539                                    }
8540                                    let provider_revision_final =
8541                                        self.resolver.planning_semantic_revision();
8542                                    if provider_revision_final != provider_revision {
8543                                        drop(commit_guard);
8544                                        self.publish_compressed_partial_report(
8545                                            &report,
8546                                            &direct_report,
8547                                        );
8548                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8549                                            .with_message(
8550                                                "function provider changed after clean-family fallback validation",
8551                                            ));
8552                                    }
8553                                    let graph_formulas =
8554                                        self.graph.apply_prevalidated_legacy_graph_plan(plan);
8555                                    direct_report.formula_cells_seen = direct_report
8556                                        .formula_cells_seen
8557                                        .saturating_add(formula_count);
8558                                    direct_report.graph_formula_cells_materialized = direct_report
8559                                        .graph_formula_cells_materialized
8560                                        .saturating_add(graph_formulas as u64);
8561                                    direct_report.graph_vertices_created = direct_report
8562                                        .graph_vertices_created
8563                                        .saturating_add(graph_vertices as u64);
8564                                    direct_report.graph_edges_created = direct_report
8565                                        .graph_edges_created
8566                                        .saturating_add(graph_edges as u64);
8567                                    direct_report.source_spool_replays =
8568                                        direct_report.source_spool_replays.saturating_add(1);
8569                                    direct_report.source_family_fallback =
8570                                        direct_report.source_family_fallback.saturating_add(1);
8571                                    direct_report.source_family_fallback_cells = direct_report
8572                                        .source_family_fallback_cells
8573                                        .saturating_add(cells);
8574                                    direct_report.source_anchor_parses =
8575                                        direct_report.source_anchor_parses.saturating_add(1);
8576                                    direct_report.source_anchor_asts =
8577                                        direct_report.source_anchor_asts.saturating_add(1);
8578                                    direct_report.source_anchor_analyses =
8579                                        direct_report.source_anchor_analyses.saturating_add(1);
8580                                    *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8581                                }
8582                            }
8583                        }
8584                        SourceProposal::Fragment(fragment) => {
8585                            let commit_guard =
8586                                crate::function_registry::semantic_epoch_read_guard();
8587                            let commit_epoch = commit_guard.epoch();
8588                            let commit_provider_revision =
8589                                self.resolver.planning_semantic_revision();
8590                            let source = fragment.source;
8591                            let source_id = source.source_id;
8592                            #[cfg(test)]
8593                            let commit_fault = self
8594                                .fragmented_commit_fault_for_test
8595                                .take()
8596                                .unwrap_or(
8597                                crate::engine::fragmented_transaction::FragmentedCommitFault::None,
8598                            );
8599                            #[cfg(not(test))]
8600                            let commit_fault =
8601                                crate::engine::fragmented_transaction::FragmentedCommitFault::None;
8602                            let outcome = self
8603                                .prepare_fragmented_source_transaction(
8604                                    &source,
8605                                    &preparation.replay_disposition,
8606                                    fragment.prepared,
8607                                    fragment.legacy,
8608                                )
8609                                .map(|transaction| {
8610                                    let resolver = &self.resolver;
8611                                    self.graph.commit_fragmented_source_transaction(
8612                                        transaction,
8613                                        &self.source_formula_token,
8614                                        &preparation.replay_disposition,
8615                                        commit_epoch,
8616                                        commit_provider_revision,
8617                                        || resolver.planning_semantic_revision(),
8618                                        commit_fault,
8619                                    )
8620                                });
8621                            let fallback_reason = match outcome {
8622                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
8623                                    let mut delta = success.report_delta;
8624                                    delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
8625                                    delta.source_family_promoted = 1;
8626                                    delta.source_family_promoted_cells =
8627                                        source.surviving_member_count;
8628                                    delta.source_partitioned_families_seen = 0;
8629                                    delta.source_partition_holes = 0;
8630                                    delta.source_partition_ordinary_exceptions = 0;
8631                                    delta.source_partition_surviving_cells = 0;
8632                                    delta.shadow_templates_interned =
8633                                        success.plane.work.templates_to_append as u64;
8634                                    delta.shadow_spans_created = success.plane.spans.len() as u64;
8635                                    let descendants =
8636                                        source.surviving_member_count.saturating_sub(1);
8637                                    delta.source_descendant_strings_avoided = descendants;
8638                                    delta.source_descendant_events_avoided = descendants;
8639                                    delta.source_descendant_analyses_avoided = descendants;
8640                                    direct_report.accumulate(&delta);
8641                                    None
8642                                }
8643                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
8644                                    Some(format!("{reason:?}"))
8645                                }
8646                                Err(error) => Some(format!("{error:?}")),
8647                            };
8648                            if let Some(reason) = fallback_reason {
8649                                drop(commit_guard);
8650                                let mut replay_disposition = preparation.replay_disposition.clone();
8651                                replay_disposition.force_family_legacy(source_id);
8652                                let replayed = (|| {
8653                                    let records = fragment
8654                                        .replay
8655                                        .lock()
8656                                        .map_err(|_| {
8657                                            ExcelError::new(ExcelErrorKind::Value).with_message(
8658                                                "compressed formula exact replay lock poisoned",
8659                                            )
8660                                        })?
8661                                        .replay_partitioned(
8662                                            &replay_disposition,
8663                                            std::slice::from_ref(&source),
8664                                        )
8665                                        .map_err(|message| {
8666                                            ExcelError::new(ExcelErrorKind::Value)
8667                                                .with_message(message)
8668                                        })?
8669                                        .into_iter()
8670                                        .filter(|record| {
8671                                            record.family == Some(source_id)
8672                                                || (record.family.is_none()
8673                                                    && record.partition_owner == Some(source_id))
8674                                        })
8675                                        .collect::<Vec<_>>();
8676                                    Self::validate_whole_partition_replay(&source, &records)?;
8677                                    Ok::<_, ExcelError>(records)
8678                                })();
8679                                let replayed = match replayed {
8680                                    Ok(replayed) => replayed,
8681                                    Err(error) => {
8682                                        self.publish_compressed_partial_report(
8683                                            &report,
8684                                            &direct_report,
8685                                        );
8686                                        return Err(error);
8687                                    }
8688                                };
8689                                let batch = match self.formula_batch_from_exact_replay(
8690                                    preparation.sheet_name.as_ref(),
8691                                    replayed,
8692                                ) {
8693                                    Ok(batch) => batch,
8694                                    Err(error) => {
8695                                        self.publish_compressed_partial_report(
8696                                            &report,
8697                                            &direct_report,
8698                                        );
8699                                        return Err(error);
8700                                    }
8701                                };
8702                                let snapshot = match self.fallback_planning_snapshot(&batch) {
8703                                    Ok(snapshot) => snapshot,
8704                                    Err(error) => {
8705                                        self.publish_compressed_partial_report(
8706                                            &report,
8707                                            &direct_report,
8708                                        );
8709                                        return Err(error);
8710                                    }
8711                                };
8712                                let commit_guard =
8713                                    crate::function_registry::semantic_epoch_read_guard();
8714                                let provider_revision = self.resolver.planning_semantic_revision();
8715                                if (commit_guard.epoch() != snapshot.epoch()
8716                                    && snapshot.semantic_changes_affect_requests_since_guarded(
8717                                        &commit_guard,
8718                                        snapshot.epoch(),
8719                                    ))
8720                                    || snapshot
8721                                        .provider_revision()
8722                                        .is_some_and(|revision| Some(revision) != provider_revision)
8723                                {
8724                                    self.publish_compressed_partial_report(&report, &direct_report);
8725                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8726                                        format!(
8727                                            "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
8728                                            snapshot.epoch(),
8729                                            commit_guard.epoch(),
8730                                            snapshot.provider_revision(),
8731                                            provider_revision,
8732                                        ),
8733                                    ));
8734                                }
8735                                let (plan, formula_count) =
8736                                    match self.prepare_legacy_batch_fallback(batch, &snapshot) {
8737                                        Ok(plan) => plan,
8738                                        Err(error) => {
8739                                            drop(commit_guard);
8740                                            self.publish_compressed_partial_report(
8741                                                &report,
8742                                                &direct_report,
8743                                            );
8744                                            return Err(error);
8745                                        }
8746                                    };
8747                                let provider_revision_after =
8748                                    self.resolver.planning_semantic_revision();
8749                                if provider_revision_after != provider_revision {
8750                                    drop(commit_guard);
8751                                    self.publish_compressed_partial_report(&report, &direct_report);
8752                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8753                                        .with_message(
8754                                            "function provider changed while preparing whole-family fallback",
8755                                        ));
8756                                }
8757                                let graph_vertices = plan.new_vertex_count();
8758                                let Some(graph_edges) = plan.planned_edge_count() else {
8759                                    drop(commit_guard);
8760                                    self.publish_compressed_partial_report(&report, &direct_report);
8761                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8762                                        .with_message(
8763                                            "prepared whole-family fallback size overflow",
8764                                        ));
8765                                };
8766                                if let Err(error) =
8767                                    self.prepared_legacy_admission(&plan, formula_count)
8768                                {
8769                                    drop(commit_guard);
8770                                    self.publish_compressed_partial_report(&report, &direct_report);
8771                                    return Err(error);
8772                                }
8773                                if let Err(error) =
8774                                    self.graph.validate_prepared_legacy_graph_plan(&plan)
8775                                {
8776                                    drop(commit_guard);
8777                                    self.publish_compressed_partial_report(&report, &direct_report);
8778                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8779                                        .with_message(error.to_string()));
8780                                }
8781                                #[cfg(test)]
8782                                if let Some(hook) = self
8783                                    .before_legacy_fallback_final_provider_sample_hook
8784                                    .take()
8785                                {
8786                                    hook();
8787                                }
8788                                let provider_revision_final =
8789                                    self.resolver.planning_semantic_revision();
8790                                if provider_revision_final != provider_revision {
8791                                    drop(commit_guard);
8792                                    self.publish_compressed_partial_report(&report, &direct_report);
8793                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8794                                        "function provider changed after whole-family fallback validation",
8795                                    ));
8796                                }
8797                                let graph_formulas =
8798                                    self.graph.apply_prevalidated_legacy_graph_plan(plan);
8799                                direct_report.formula_cells_seen = direct_report
8800                                    .formula_cells_seen
8801                                    .saturating_add(formula_count);
8802                                direct_report.graph_formula_cells_materialized = direct_report
8803                                    .graph_formula_cells_materialized
8804                                    .saturating_add(graph_formulas as u64);
8805                                direct_report.graph_vertices_created = direct_report
8806                                    .graph_vertices_created
8807                                    .saturating_add(graph_vertices as u64);
8808                                direct_report.graph_edges_created = direct_report
8809                                    .graph_edges_created
8810                                    .saturating_add(graph_edges as u64);
8811                                direct_report.source_spool_replays =
8812                                    direct_report.source_spool_replays.saturating_add(1);
8813                                direct_report.source_family_fallback =
8814                                    direct_report.source_family_fallback.saturating_add(1);
8815                                direct_report.source_family_fallback_cells = direct_report
8816                                    .source_family_fallback_cells
8817                                    .saturating_add(source.surviving_member_count);
8818                                direct_report.source_partitioned_families_rejected = direct_report
8819                                    .source_partitioned_families_rejected
8820                                    .saturating_add(1);
8821                                direct_report.source_partition_failures =
8822                                    direct_report.source_partition_failures.saturating_add(1);
8823                                direct_report.source_partition_fallback_cells = direct_report
8824                                    .source_partition_fallback_cells
8825                                    .saturating_add(source.surviving_member_count);
8826                                direct_report.source_anchor_parses =
8827                                    direct_report.source_anchor_parses.saturating_add(1);
8828                                direct_report.source_anchor_asts =
8829                                    direct_report.source_anchor_asts.saturating_add(1);
8830                                direct_report.source_anchor_analyses =
8831                                    direct_report.source_anchor_analyses.saturating_add(1);
8832                                *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8833                            }
8834                        }
8835                    }
8836                    #[cfg(test)]
8837                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
8838                        hook();
8839                    }
8840                }
8841            }
8842            break;
8843        }
8844        report.accumulate(&direct_report);
8845        self.record_formula_ingest_report(report.clone());
8846        Ok(report)
8847    }
8848    /// Ingest replayed per-cell formulas while preserving compressed source counters.
8849    pub(crate) fn ingest_compressed_formula_source_batches(
8850        &mut self,
8851        batches: Vec<(
8852            FormulaIngestBatch,
8853            crate::engine::FormulaCompressedSourceBatch,
8854        )>,
8855    ) -> Result<FormulaIngestReport, ExcelError> {
8856        self.ingest_compressed_formula_source_batches_inner(batches, true)
8857    }
8858
8859    fn ingest_compressed_formula_source_batches_inner(
8860        &mut self,
8861        batches: Vec<(
8862            FormulaIngestBatch,
8863            crate::engine::FormulaCompressedSourceBatch,
8864        )>,
8865        publish_report: bool,
8866    ) -> Result<FormulaIngestReport, ExcelError> {
8867        let mut source_counts = [0_u64; 11];
8868        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
8869        let mut formula_batches = Vec::with_capacity(batches.len());
8870        let mut compressed_families = Vec::new();
8871        let mut partitioned_families = Vec::new();
8872        for (batch, compressed_batch) in batches {
8873            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
8874            if sheet_name.as_ref() != batch.sheet_name {
8875                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8876                    "compressed formula source sheet does not match its replay batch",
8877                ));
8878            }
8879            compressed_families.push((batch.sheet_name.clone(), families));
8880            partitioned_families.push((batch.sheet_name.clone(), partitions));
8881            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
8882            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
8883            source_counts[2] =
8884                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
8885            source_counts[3] =
8886                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
8887            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
8888            source_counts[5] =
8889                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
8890            source_counts[6] =
8891                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
8892            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
8893            source_counts[8] =
8894                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
8895            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
8896            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
8897            source_report.families_seen = source_report
8898                .families_seen
8899                .saturating_add(compressed.families_seen);
8900            source_report.family_cells_seen = source_report
8901                .family_cells_seen
8902                .saturating_add(compressed.family_cells_seen);
8903            source_report.source_clean_families = source_report
8904                .source_clean_families
8905                .saturating_add(compressed.source_clean_families);
8906            source_report.source_clean_cells = source_report
8907                .source_clean_cells
8908                .saturating_add(compressed.source_clean_cells);
8909            source_report.source_fragmentable_families = source_report
8910                .source_fragmentable_families
8911                .saturating_add(compressed.source_fragmentable_families);
8912            source_report.source_fragmentable_cells = source_report
8913                .source_fragmentable_cells
8914                .saturating_add(compressed.source_fragmentable_cells);
8915            source_report.source_fragment_count = source_report
8916                .source_fragment_count
8917                .saturating_add(compressed.source_fragment_count);
8918            source_report.source_isolated_fallback_cells = source_report
8919                .source_isolated_fallback_cells
8920                .saturating_add(compressed.source_isolated_fallback_cells);
8921            source_report.source_hole_exclusions = source_report
8922                .source_hole_exclusions
8923                .saturating_add(compressed.source_hole_exclusions);
8924            source_report.source_ordinary_exclusions = source_report
8925                .source_ordinary_exclusions
8926                .saturating_add(compressed.source_ordinary_exclusions);
8927            source_report.source_partition_failures = source_report
8928                .source_partition_failures
8929                .saturating_add(compressed.source_partition_failures);
8930            source_report.replay_families = source_report
8931                .replay_families
8932                .saturating_add(compressed.replay_families);
8933            source_report.replay_cells = source_report
8934                .replay_cells
8935                .saturating_add(compressed.replay_cells);
8936            source_report.forward_descendants = source_report
8937                .forward_descendants
8938                .saturating_add(compressed.forward_descendants);
8939            source_report.evidence_limit_fallbacks = source_report
8940                .evidence_limit_fallbacks
8941                .saturating_add(compressed.evidence_limit_fallbacks);
8942            source_report.evidence_peak_bytes = source_report
8943                .evidence_peak_bytes
8944                .max(compressed.evidence_peak_bytes);
8945            for (reason, count) in compressed.fallback_reasons {
8946                *source_report.fallback_reasons.entry(reason).or_default() += count;
8947            }
8948            formula_batches.push(batch);
8949        }
8950        self.ingest_formula_batches_inner(
8951            formula_batches,
8952            source_counts,
8953            Some(source_report),
8954            compressed_families,
8955            partitioned_families,
8956            publish_report,
8957        )
8958    }
8959
8960    pub fn ingest_formula_batches(
8961        &mut self,
8962        batches: Vec<FormulaIngestBatch>,
8963    ) -> Result<FormulaIngestReport, ExcelError> {
8964        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
8965        let report = self.ingest_formula_batches_inner(
8966            batches,
8967            [0; 11],
8968            None,
8969            Vec::new(),
8970            Vec::new(),
8971            true,
8972        )?;
8973        if has_formulas {
8974            self.mark_topology_edited();
8975        }
8976        Ok(report)
8977    }
8978
8979    fn ingest_formula_batches_unpublished(
8980        &mut self,
8981        batches: Vec<FormulaIngestBatch>,
8982    ) -> Result<FormulaIngestReport, ExcelError> {
8983        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
8984    }
8985
8986    fn ingest_formula_batches_inner(
8987        &mut self,
8988        batches: Vec<FormulaIngestBatch>,
8989        source_counts: [u64; 11],
8990        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
8991        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
8992        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
8993        publish_report: bool,
8994    ) -> Result<FormulaIngestReport, ExcelError> {
8995        self.observe_function_semantic_epoch()?;
8996        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
8997        #[cfg(feature = "benchmark_internal")]
8998        let benchmark_forced_replay =
8999            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9000        #[cfg(not(feature = "benchmark_internal"))]
9001        let benchmark_forced_replay = false;
9002        let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9003            && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9004        {
9005            let mut report =
9006                FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9007            report
9008                .fallback_reasons
9009                .insert("ForcedReplay".to_string(), formula_cells_seen);
9010            (report, batches, BTreeMap::new())
9011        } else {
9012            match self.config.formula_plane_mode {
9013                FormulaPlaneMode::Off => (
9014                    FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9015                    batches,
9016                    BTreeMap::new(),
9017                ),
9018                FormulaPlaneMode::Shadow => (
9019                    if partitioned_families
9020                        .iter()
9021                        .all(|(_, families)| families.is_empty())
9022                    {
9023                        if compressed_families.is_empty() {
9024                            self.analyze_formula_plane_shadow_candidates(&batches)
9025                        } else {
9026                            self.analyze_compressed_formula_plane_shadow(&compressed_families)
9027                        }
9028                    } else {
9029                        let mut partition_report =
9030                            self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9031                        if compressed_families
9032                            .iter()
9033                            .any(|(_, families)| !families.is_empty())
9034                        {
9035                            partition_report.accumulate(
9036                                &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9037                            );
9038                        }
9039                        partition_report
9040                    },
9041                    batches,
9042                    BTreeMap::new(),
9043                ),
9044                FormulaPlaneMode::AuthoritativeExperimental => {
9045                    self.analyze_formula_plane_authoritative_ingest(&batches)
9046                }
9047            }
9048        };
9049        report.formula_cells_seen = formula_cells_seen;
9050        report.source_formula_events = source_counts[0];
9051        report.source_ordinary_events = source_counts[1];
9052        report.source_shared_anchor_events = source_counts[2];
9053        report.source_shared_descendant_events = source_counts[3];
9054        report.source_unknown_events = source_counts[4];
9055        report.source_formula_records_spooled = source_counts[5];
9056        report.source_spool_encoded_bytes = source_counts[6];
9057        report.source_spool_peak_memory_bytes = source_counts[7];
9058        report.source_spool_spilled_bytes = source_counts[8];
9059        report.source_spool_spill_files = source_counts[9];
9060        report.source_spool_replays = source_counts[10];
9061        if let Some(source) = source_report {
9062            report.source_families_seen = source.families_seen;
9063            report.source_family_cells_seen = source.family_cells_seen;
9064            report.source_family_shadow_eligible = source.source_clean_families;
9065            report.source_family_shadow_eligible_cells = source.source_clean_cells;
9066            report.source_partitioned_families_seen = source.source_fragmentable_families;
9067            report.source_partition_holes = source.source_hole_exclusions;
9068            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9069            report.source_partition_failures = source.source_partition_failures;
9070            report.source_partition_surviving_cells = source.source_fragmentable_cells;
9071            report.source_family_fallback = report
9072                .source_family_fallback
9073                .saturating_add(source.replay_families);
9074            report.source_family_fallback_cells = report
9075                .source_family_fallback_cells
9076                .saturating_add(source.replay_cells);
9077            report.source_forward_descendants = source.forward_descendants;
9078            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9079            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9080            for (reason, count) in source.fallback_reasons {
9081                let total = report.fallback_reasons.entry(reason).or_default();
9082                *total = total.saturating_add(count);
9083            }
9084        }
9085
9086        if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9087            || !planned_materialize.is_empty()
9088        {
9089            let mut prepared_by_sheet = planned_materialize;
9090            for batch in materialize_batches {
9091                if batch.is_empty() {
9092                    continue;
9093                }
9094                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9095                    ExcelError::new(ExcelErrorKind::Ref)
9096                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9097                })?;
9098                let mut pipeline = self.ingest_pipeline();
9099                let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9100                    let placement = CellRef::new(
9101                        sheet_id,
9102                        Coord::from_excel(record.row, record.col, true, true),
9103                    );
9104                    (
9105                        FormulaAstInput::RawArena(record.ast_id),
9106                        placement,
9107                        record.formula_text,
9108                    )
9109                }))?;
9110                prepared_by_sheet
9111                    .entry(batch.sheet_name)
9112                    .or_default()
9113                    .extend(ingested.into_iter().map(|formula| {
9114                        (
9115                            formula.placement.coord.row() + 1,
9116                            formula.placement.coord.col() + 1,
9117                            formula.ast_id,
9118                            formula.dep_plan,
9119                        )
9120                    }));
9121            }
9122            let admission_preflighted = self.graph_admission_enabled();
9123            if admission_preflighted {
9124                let mut preview = Vec::new();
9125                for (sheet_name, formulas) in &prepared_by_sheet {
9126                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9127                        ExcelError::new(ExcelErrorKind::Ref)
9128                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
9129                    })?;
9130                    preview.extend(
9131                        formulas
9132                            .iter()
9133                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9134                    );
9135                }
9136                let admission = self.graph.preview_formula_mutations(&preview)?;
9137                self.preflight_graph_admission(admission)?;
9138            }
9139
9140            let mut builder = self.begin_bulk_ingest();
9141            if admission_preflighted {
9142                builder.mark_admission_preflighted();
9143            }
9144            for (sheet_name, formulas) in prepared_by_sheet {
9145                if formulas.is_empty() {
9146                    continue;
9147                }
9148                let sheet_id = builder.add_sheet(&sheet_name);
9149                builder.add_formula_plans(sheet_id, formulas);
9150            }
9151            let summary = builder.finish()?;
9152            report.graph_formula_cells_materialized = summary.formulas as u64;
9153            report.graph_vertices_created = summary.vertices as u64;
9154            report.graph_edges_created = summary.edges as u64;
9155        }
9156
9157        if publish_report {
9158            self.record_formula_ingest_report(report.clone());
9159        }
9160        Ok(report)
9161    }
9162
9163    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9164        let mut unique = Vec::new();
9165        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9166            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9167                prior.sheet == diagnostic.sheet
9168                    && prior.row == diagnostic.row
9169                    && prior.col == diagnostic.col
9170                    && prior.formula == diagnostic.formula
9171                    && prior.policy == diagnostic.policy
9172            });
9173            if !duplicate {
9174                unique.push(diagnostic);
9175            }
9176        }
9177        self.formula_parse_diagnostics.extend(unique);
9178    }
9179
9180    pub fn handle_formula_parse_error(
9181        &mut self,
9182        sheet: &str,
9183        row: u32,
9184        col: u32,
9185        formula: &str,
9186        message: String,
9187    ) -> Result<Option<ASTNode>, ExcelError> {
9188        let policy = self.config.formula_parse_policy;
9189
9190        if policy == FormulaParsePolicy::Strict {
9191            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9192            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9193                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9194            )));
9195        }
9196
9197        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9198            sheet: sheet.to_string(),
9199            row,
9200            col,
9201            formula: formula.to_string(),
9202            message: message.clone(),
9203            policy,
9204        });
9205
9206        match policy {
9207            FormulaParsePolicy::Strict => unreachable!(),
9208            FormulaParsePolicy::KeepCachedValue => Ok(None),
9209            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9210                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9211                None,
9212            ))),
9213            FormulaParsePolicy::CoerceToError => {
9214                let err = ExcelError::new(ExcelErrorKind::Error)
9215                    .with_message(format!("Malformed formula: {message}"));
9216                Ok(Some(ASTNode::new(
9217                    ASTNodeType::Literal(LiteralValue::Error(err)),
9218                    None,
9219                )))
9220            }
9221        }
9222    }
9223
9224    #[cfg(test)]
9225    fn target_preparation_fault(
9226        &mut self,
9227        seam: crate::engine::target_preparation::TargetPreparationFault,
9228    ) -> Result<(), ExcelError> {
9229        if self.target_preparation_fault_for_test == Some(seam) {
9230            self.target_preparation_fault_for_test = None;
9231            Err(ExcelError::new(ExcelErrorKind::Value)
9232                .with_message(format!("injected target preparation fault: {seam:?}")))
9233        } else {
9234            Ok(())
9235        }
9236    }
9237
9238    fn preparation_stale(
9239        reason: formualizer_common::PreparationStaleReason,
9240        message: impl Into<String>,
9241    ) -> ExcelError {
9242        ExcelError::new(ExcelErrorKind::Value)
9243            .with_message(message)
9244            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
9245    }
9246
9247    fn preparation_revision_stale_reason(
9248        assumptions: &crate::engine::PreparationRevision,
9249        current: &crate::engine::PreparationRevision,
9250        planning_requests: &BTreeSet<(String, String, usize)>,
9251        staged_leases_match: bool,
9252    ) -> Option<formualizer_common::PreparationStaleReason> {
9253        if assumptions.graph != current.graph {
9254            Some(formualizer_common::PreparationStaleReason::Graph)
9255        } else if assumptions.authority != current.authority
9256            || assumptions.authority_indexes != current.authority_indexes
9257            || assumptions.authority_indexed_plane != current.authority_indexed_plane
9258        {
9259            Some(formualizer_common::PreparationStaleReason::Authority)
9260        } else if assumptions.staged != current.staged || !staged_leases_match {
9261            Some(formualizer_common::PreparationStaleReason::Staged)
9262        } else if assumptions.symbols != current.symbols {
9263            Some(formualizer_common::PreparationStaleReason::Symbols)
9264        } else if assumptions.provider != current.provider {
9265            Some(formualizer_common::PreparationStaleReason::Provider)
9266        } else if assumptions.semantic != current.semantic
9267            && crate::function_registry::semantic_changes_affect_requests_since(
9268                assumptions.semantic,
9269                planning_requests.iter().cloned(),
9270            )
9271        {
9272            Some(formualizer_common::PreparationStaleReason::Semantic)
9273        } else {
9274            None
9275        }
9276    }
9277
9278    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
9279        let (authority, authority_indexes, authority_indexed_plane) =
9280            self.graph.authority_revisions();
9281        crate::engine::PreparationRevision {
9282            graph: self.graph.topology_revision(),
9283            authority,
9284            authority_indexes,
9285            authority_indexed_plane,
9286            staged: self.staged_formula_index.revision(),
9287            symbols: self.graph.symbol_revision(),
9288            semantic: crate::function_registry::semantic_epoch(),
9289            provider: self.resolver.planning_semantic_revision(),
9290        }
9291    }
9292
9293    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
9294        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
9295        let provider = self.resolver.planning_semantic_revision();
9296        let semantic = registry_guard.epoch();
9297        let (authority, authority_indexes, authority_indexed_plane) =
9298            self.graph.authority_revisions();
9299        let mut span_refs = self.graph.formula_authority().active_span_refs();
9300        span_refs.sort_unstable_by_key(|span_ref| {
9301            (span_ref.id.0, span_ref.generation, span_ref.version)
9302        });
9303        PlanningRevisionSnapshot {
9304            engine_topology_epoch: self.topology_epoch,
9305            graph_topology_revision: self.graph.topology_revision(),
9306            authority,
9307            authority_indexes,
9308            authority_indexed_plane,
9309            staged: self.staged_formula_index.revision(),
9310            symbols: self.graph.symbol_revision(),
9311            semantic,
9312            provider,
9313            formula_plane_mode: self.config.formula_plane_mode,
9314            deterministic_mode: self.config.deterministic_mode.clone(),
9315            budgets: self.evaluation_resource_budgets.clone(),
9316            span_refs,
9317        }
9318    }
9319
9320    fn recalc_plan_key(&self) -> RecalcPlanKey {
9321        RecalcPlanKey {
9322            engine_token: Arc::clone(&self.recalc_plan_token),
9323            revisions: self.planning_revision_snapshot(),
9324        }
9325    }
9326
9327    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
9328        ExcelError::new(ExcelErrorKind::Value)
9329            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
9330            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
9331    }
9332
9333    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
9334        use formualizer_common::PlanStaleReason;
9335
9336        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
9337            return Err(Self::plan_stale(PlanStaleReason::Engine));
9338        }
9339
9340        let current = self.planning_revision_snapshot();
9341        let expected = &key.revisions;
9342        let stale = if expected.provider != current.provider {
9343            Some(PlanStaleReason::Provider)
9344        } else if expected.semantic != current.semantic {
9345            Some(PlanStaleReason::Semantic)
9346        } else if expected.budgets != current.budgets
9347            || expected.deterministic_mode != current.deterministic_mode
9348        {
9349            Some(PlanStaleReason::Budget)
9350        } else if expected.staged != current.staged {
9351            Some(PlanStaleReason::Staged)
9352        } else if expected.symbols != current.symbols {
9353            Some(PlanStaleReason::Symbols)
9354        } else if expected.formula_plane_mode != current.formula_plane_mode
9355            || expected.authority != current.authority
9356            || expected.authority_indexes != current.authority_indexes
9357            || expected.authority_indexed_plane != current.authority_indexed_plane
9358        {
9359            Some(PlanStaleReason::Authority)
9360        } else if expected.span_refs != current.span_refs {
9361            Some(PlanStaleReason::SpanGeneration)
9362        } else if expected.graph_topology_revision != current.graph_topology_revision
9363            || expected.engine_topology_epoch != current.engine_topology_epoch
9364        {
9365            Some(PlanStaleReason::Graph)
9366        } else {
9367            None
9368        };
9369        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
9370    }
9371
9372    fn target_preparation_checkpoint(
9373        &mut self,
9374        deadline: Option<std::time::Instant>,
9375        work: u64,
9376    ) -> Result<(), ExcelError> {
9377        if self
9378            .active_cancel_flag
9379            .as_ref()
9380            .is_some_and(|cancel| cancel.is_cancelled())
9381        {
9382            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
9383                .with_message("target graph preparation cancelled"));
9384        }
9385        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
9386            return Err(crate::engine::ResourceLedgerError::Exhausted(
9387                formualizer_common::ResourceExhaustionDetail {
9388                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
9389                    limit: 0,
9390                    observed: 1,
9391                    request_id: self
9392                        .active_evaluation_resource_request
9393                        .as_ref()
9394                        .map(|stats| stats.request_id),
9395                },
9396            )
9397            .into_excel_error());
9398        }
9399        self.charge_bounded_work(work)
9400    }
9401
9402    fn opaque_reason_in_ast(
9403        &self,
9404        ast: &ASTNode,
9405        provider: &dyn crate::traits::FunctionProvider,
9406    ) -> Option<crate::engine::OpaqueReason> {
9407        match &ast.node_type {
9408            ASTNodeType::Function { name, args } => {
9409                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
9410                if canonical == "INDIRECT" {
9411                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
9412                }
9413                let Some(function) = provider.get_function_for_planning("", name) else {
9414                    return Some(crate::engine::OpaqueReason::UnknownFunction);
9415                };
9416                let caps = function.caps();
9417                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9418                    || caps.contains(FnCaps::RETURNS_REFERENCE)
9419                {
9420                    return Some(crate::engine::OpaqueReason::DynamicReference);
9421                }
9422                args.iter()
9423                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
9424            }
9425            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
9426            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
9427            ASTNodeType::BinaryOp { left, right, .. } => self
9428                .opaque_reason_in_ast(left, provider)
9429                .or_else(|| self.opaque_reason_in_ast(right, provider)),
9430            ASTNodeType::Array(rows) => rows
9431                .iter()
9432                .flat_map(|row| row.iter())
9433                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
9434            ASTNodeType::Reference {
9435                reference:
9436                    ReferenceType::Cell {
9437                        sheet: Some(sheet), ..
9438                    }
9439                    | ReferenceType::Range {
9440                        sheet: Some(sheet), ..
9441                    },
9442                ..
9443            } if self.graph.sheet_id(sheet).is_none() => {
9444                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
9445            }
9446            ASTNodeType::Reference {
9447                reference:
9448                    ReferenceType::External(_)
9449                    | ReferenceType::Cell3D { .. }
9450                    | ReferenceType::Range3D { .. },
9451                ..
9452            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
9453            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
9454        }
9455    }
9456
9457    fn target_planning_snapshot(
9458        &mut self,
9459        ast: &ASTNode,
9460        planning_requests: &mut BTreeSet<(String, String, usize)>,
9461    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
9462        #[cfg(test)]
9463        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
9464            return Err(Self::preparation_stale(
9465                formualizer_common::PreparationStaleReason::Semantic,
9466                "injected target semantic preparation movement",
9467            ));
9468        }
9469        #[cfg(test)]
9470        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
9471            hook();
9472        }
9473        let mut requests = Vec::new();
9474        Self::collect_planning_function_requests(ast, &mut requests);
9475        requests.sort();
9476        requests.dedup();
9477        planning_requests.extend(requests.iter().cloned());
9478        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
9479            &self.resolver,
9480            requests,
9481        )
9482        .map_err(|error| {
9483            ExcelError::new(ExcelErrorKind::Value)
9484                .with_message(format!("target planning snapshot unavailable: {error:?}"))
9485        })
9486    }
9487
9488    fn target_planning_snapshot_stale_reason(
9489        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
9490        assumptions: &crate::engine::PreparationRevision,
9491    ) -> Option<formualizer_common::PreparationStaleReason> {
9492        if snapshot
9493            .provider_revision()
9494            .is_some_and(|revision| Some(revision) != assumptions.provider)
9495        {
9496            Some(formualizer_common::PreparationStaleReason::Provider)
9497        } else if snapshot.epoch() != assumptions.semantic
9498            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
9499        {
9500            Some(formualizer_common::PreparationStaleReason::Semantic)
9501        } else {
9502            None
9503        }
9504    }
9505
9506    fn widen_target_preparation(
9507        policy: crate::engine::OpaquePreparePolicy,
9508        scope: &mut crate::engine::PrepareScope,
9509        reasons: &mut Vec<crate::engine::OpaqueReason>,
9510        reason: crate::engine::OpaqueReason,
9511    ) -> Result<bool, ExcelError> {
9512        if policy == crate::engine::OpaquePreparePolicy::Error {
9513            return Err(ExcelError::new(ExcelErrorKind::NImpl)
9514                .with_message(format!("opaque target preparation semantics: {reason:?}")));
9515        }
9516        if !reasons.contains(&reason) {
9517            reasons.push(reason);
9518        }
9519        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
9520            *scope = crate::engine::PrepareScope::Workbook;
9521            Ok(true)
9522        } else {
9523            Ok(false)
9524        }
9525    }
9526
9527    fn widen_target_preparation_to_sheet(
9528        policy: crate::engine::OpaquePreparePolicy,
9529        scope: &mut crate::engine::PrepareScope,
9530        reasons: &mut Vec<crate::engine::OpaqueReason>,
9531        reason: crate::engine::OpaqueReason,
9532        sheet: &str,
9533    ) -> Result<bool, ExcelError> {
9534        if policy == crate::engine::OpaquePreparePolicy::Error {
9535            return Err(ExcelError::new(ExcelErrorKind::NImpl)
9536                .with_message(format!("opaque target preparation semantics: {reason:?}")));
9537        }
9538        if !reasons.contains(&reason) {
9539            reasons.push(reason);
9540        }
9541        match scope {
9542            crate::engine::PrepareScope::Exact => {
9543                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
9544                Ok(true)
9545            }
9546            crate::engine::PrepareScope::Sheets(sheets) => {
9547                if sheets.iter().any(|candidate| candidate == sheet) {
9548                    Ok(false)
9549                } else {
9550                    sheets.push(sheet.to_string());
9551                    sheets.sort();
9552                    Ok(true)
9553                }
9554            }
9555            crate::engine::PrepareScope::Workbook => Ok(false),
9556        }
9557    }
9558
9559    fn ast_has_proven_sheet_local_dynamic(
9560        ast: &ASTNode,
9561        provider: &dyn crate::traits::FunctionProvider,
9562    ) -> bool {
9563        fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
9564            match &ast.node_type {
9565                ASTNodeType::Function { name, args } => {
9566                    let Some(function) = provider.get_function_for_planning("", name) else {
9567                        return (false, false);
9568                    };
9569                    let caps = function.caps();
9570                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9571                        || caps.contains(FnCaps::RETURNS_REFERENCE);
9572                    let canonical = name.rsplit('.').next().unwrap_or(name);
9573                    if dynamic
9574                        && !canonical.eq_ignore_ascii_case("OFFSET")
9575                        && !canonical.eq_ignore_ascii_case("INDEX")
9576                    {
9577                        return (false, true);
9578                    }
9579                    let mut has_dynamic = dynamic;
9580                    for arg in args {
9581                        let (safe, child_dynamic) = classify(arg, provider);
9582                        if !safe {
9583                            return (false, has_dynamic || child_dynamic);
9584                        }
9585                        has_dynamic |= child_dynamic;
9586                    }
9587                    (true, has_dynamic)
9588                }
9589                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
9590                ASTNodeType::BinaryOp { left, right, .. } => {
9591                    let (left_safe, left_dynamic) = classify(left, provider);
9592                    let (right_safe, right_dynamic) = classify(right, provider);
9593                    (left_safe && right_safe, left_dynamic || right_dynamic)
9594                }
9595                ASTNodeType::Array(rows) => {
9596                    let mut has_dynamic = false;
9597                    for item in rows.iter().flatten() {
9598                        let (safe, child_dynamic) = classify(item, provider);
9599                        if !safe {
9600                            return (false, has_dynamic || child_dynamic);
9601                        }
9602                        has_dynamic |= child_dynamic;
9603                    }
9604                    (true, has_dynamic)
9605                }
9606                ASTNodeType::Reference {
9607                    reference:
9608                        ReferenceType::Cell { sheet: None, .. }
9609                        | ReferenceType::Range { sheet: None, .. },
9610                    ..
9611                }
9612                | ASTNodeType::Literal(_)
9613                | ASTNodeType::Omitted => (true, false),
9614                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
9615            }
9616        }
9617
9618        let (safe, dynamic) = classify(ast, provider);
9619        safe && dynamic
9620    }
9621
9622    fn table_selection_region(
9623        &self,
9624        entry: &crate::engine::graph::TableEntry,
9625        selection: &crate::engine::TableSelection,
9626    ) -> Result<PreparationRegion, ExcelError> {
9627        let mut start_row = entry.range.start.coord.row() + 1;
9628        let mut end_row = entry.range.end.coord.row() + 1;
9629        let mut start_col = entry.range.start.coord.col() + 1;
9630        let mut end_col = entry.range.end.coord.col() + 1;
9631        match selection {
9632            crate::engine::TableSelection::Whole => {}
9633            crate::engine::TableSelection::Headers => {
9634                if !entry.header_row {
9635                    return Err(ExcelError::new(ExcelErrorKind::Value)
9636                        .with_message(format!("table {} has no header row", entry.name)));
9637                }
9638                end_row = start_row;
9639            }
9640            crate::engine::TableSelection::Data => {
9641                if entry.header_row {
9642                    start_row = start_row.saturating_add(1);
9643                }
9644                if entry.totals_row {
9645                    end_row = end_row.saturating_sub(1);
9646                }
9647            }
9648            crate::engine::TableSelection::Totals => {
9649                if !entry.totals_row {
9650                    return Err(ExcelError::new(ExcelErrorKind::Value)
9651                        .with_message(format!("table {} has no totals row", entry.name)));
9652                }
9653                start_row = end_row;
9654            }
9655            crate::engine::TableSelection::Column(column) => {
9656                let index = entry.col_index(column).ok_or_else(|| {
9657                    ExcelError::new(ExcelErrorKind::Name)
9658                        .with_message(format!("unknown table column: {column}"))
9659                })?;
9660                start_col = start_col.saturating_add(index as u32);
9661                end_col = start_col;
9662            }
9663            crate::engine::TableSelection::Columns { start, end } => {
9664                let first = entry.col_index(start).ok_or_else(|| {
9665                    ExcelError::new(ExcelErrorKind::Name)
9666                        .with_message(format!("unknown table column: {start}"))
9667                })?;
9668                let last = entry.col_index(end).ok_or_else(|| {
9669                    ExcelError::new(ExcelErrorKind::Name)
9670                        .with_message(format!("unknown table column: {end}"))
9671                })?;
9672                if first > last {
9673                    return Err(ExcelError::new(ExcelErrorKind::Value)
9674                        .with_message("table column selection is reversed"));
9675                }
9676                start_col = start_col.saturating_add(first as u32);
9677                end_col = entry
9678                    .range
9679                    .start
9680                    .coord
9681                    .col()
9682                    .saturating_add(last as u32)
9683                    .saturating_add(1);
9684            }
9685        }
9686        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
9687            start_row = entry.range.start.coord.row() + 1;
9688            end_row = start_row;
9689        }
9690        if start_row > end_row || start_col > end_col {
9691            return Err(
9692                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
9693            );
9694        }
9695        Ok(PreparationRegion {
9696            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
9697            sheet_id: entry.sheet_id(),
9698            start_row,
9699            start_col,
9700            end_row,
9701            end_col,
9702        })
9703    }
9704
9705    #[cfg(test)]
9706    pub(crate) fn reset_target_root_dedup_probes_for_test() {
9707        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
9708    }
9709
9710    #[cfg(test)]
9711    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
9712        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
9713    }
9714
9715    pub(crate) fn resolve_target_producers(
9716        &mut self,
9717        targets: &[crate::engine::EvaluationTarget],
9718    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9719        use crate::engine::target_preparation::TargetProducer;
9720
9721        let request_id = self
9722            .active_evaluation_resource_request
9723            .as_ref()
9724            .map(|request| request.request_id);
9725        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
9726            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
9727        let active_span_refs = self
9728            .graph
9729            .formula_authority()
9730            .active_span_refs()
9731            .into_iter()
9732            .map(|span_ref| (span_ref.id, span_ref))
9733            .collect::<BTreeMap<_, _>>();
9734        let resolve_region = |engine: &mut Self,
9735                              region: Region,
9736                              value_only: Option<CellRef>,
9737                              roots: &mut OrderedTargetProducers|
9738         -> Result<(), ExcelError> {
9739            let before = roots.len();
9740            let authority = engine.graph.formula_authority();
9741            for matched in authority.producer_results.query(region).matches {
9742                if let FormulaProducerId::Span(span_id) = matched.value.producer
9743                    && let Some(span_ref) = active_span_refs.get(&span_id).copied()
9744                    && let Some(demanded) = matched.value.result_region.intersection(region)
9745                {
9746                    roots
9747                        .push(TargetProducer::Span { span_ref, demanded })
9748                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9749                }
9750            }
9751            for anchor in engine.graph.spill_anchors_in_region(
9752                region.sheet_id(),
9753                region.axis_ranges().0.query_bounds().0,
9754                region.axis_ranges().1.query_bounds().0,
9755                region.axis_ranges().0.query_bounds().1,
9756                region.axis_ranges().1.query_bounds().1,
9757            ) {
9758                roots
9759                    .push(TargetProducer::Legacy(anchor))
9760                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9761            }
9762            for vertex in engine.graph.vertices_in_region(
9763                region.sheet_id(),
9764                region.axis_ranges().0.query_bounds().0,
9765                region.axis_ranges().0.query_bounds().1,
9766                region.axis_ranges().1.query_bounds().0,
9767                region.axis_ranges().1.query_bounds().1,
9768            ) {
9769                let vertex = engine
9770                    .graph
9771                    .get_cell_ref(vertex)
9772                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
9773                    .unwrap_or(vertex);
9774                match engine.graph.get_vertex_kind(vertex) {
9775                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
9776                        roots.push(TargetProducer::Legacy(vertex)).map_err(|_| {
9777                            target_root_allocation_error(roots.len() + 1, request_id)
9778                        })?;
9779                    }
9780                    VertexKind::NamedScalar
9781                    | VertexKind::NamedArray
9782                    | VertexKind::Range
9783                    | VertexKind::InfiniteRange
9784                    | VertexKind::Table => {
9785                        roots.push(TargetProducer::Symbol(vertex)).map_err(|_| {
9786                            target_root_allocation_error(roots.len() + 1, request_id)
9787                        })?;
9788                    }
9789                    VertexKind::Empty | VertexKind::Cell | VertexKind::External => {}
9790                }
9791            }
9792            if roots.len() == before
9793                && let Some(cell) = value_only
9794            {
9795                roots
9796                    .push(TargetProducer::ValueOnly(cell))
9797                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9798            }
9799            Ok(())
9800        };
9801
9802        for target in targets {
9803            match target {
9804                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
9805                    if *row == 0 || *col == 0 {
9806                        return Err(ExcelError::new(ExcelErrorKind::Ref)
9807                            .with_message("target cell coordinates are one-based"));
9808                    }
9809                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
9810                        ExcelError::new(ExcelErrorKind::Ref)
9811                            .with_message(format!("target sheet not found: {sheet}"))
9812                    })?;
9813                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
9814                    resolve_region(
9815                        self,
9816                        Region::point(sheet_id, *row - 1, *col - 1),
9817                        Some(cell),
9818                        &mut roots,
9819                    )?;
9820                }
9821                crate::engine::EvaluationTarget::Range(range) => {
9822                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
9823                        ExcelError::new(ExcelErrorKind::Ref)
9824                            .with_message(format!("target sheet not found: {}", range.sheet))
9825                    })?;
9826                    resolve_region(
9827                        self,
9828                        Region::rect(
9829                            sheet_id,
9830                            range.start_row - 1,
9831                            range.end_row - 1,
9832                            range.start_col - 1,
9833                            range.end_col - 1,
9834                        ),
9835                        None,
9836                        &mut roots,
9837                    )?;
9838                }
9839                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
9840                    let sheet_id = scope_sheet
9841                        .as_deref()
9842                        .and_then(|sheet| self.graph.sheet_id(sheet))
9843                        .unwrap_or_else(|| self.graph.default_sheet_id());
9844                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
9845                        roots
9846                            .push(TargetProducer::Symbol(entry.vertex))
9847                            .map_err(|_| {
9848                                target_root_allocation_error(roots.len() + 1, request_id)
9849                            })?;
9850                    }
9851                }
9852                crate::engine::EvaluationTarget::Table { name, .. } => {
9853                    if let Some(entry) = self.graph.resolve_table_entry(name) {
9854                        roots
9855                            .push(TargetProducer::Symbol(entry.vertex))
9856                            .map_err(|_| {
9857                                target_root_allocation_error(roots.len() + 1, request_id)
9858                            })?;
9859                    }
9860                }
9861            }
9862        }
9863        Ok(roots.into_vec())
9864    }
9865
9866    fn resolve_target_producers_from_existing_roots(
9867        &mut self,
9868        roots: &[crate::engine::target_preparation::TargetProducer],
9869    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9870        use crate::engine::target_preparation::TargetProducer;
9871        let request_id = self
9872            .active_evaluation_resource_request
9873            .as_ref()
9874            .map(|request| request.request_id);
9875        let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
9876            .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
9877        for root in roots {
9878            match *root {
9879                TargetProducer::Span { demanded, .. } => {
9880                    let (rows, cols) = demanded.axis_ranges();
9881                    for vertex in self.graph.vertices_in_region(
9882                        demanded.sheet_id(),
9883                        rows.query_bounds().0,
9884                        rows.query_bounds().1,
9885                        cols.query_bounds().0,
9886                        cols.query_bounds().1,
9887                    ) {
9888                        if matches!(
9889                            self.graph.get_vertex_kind(vertex),
9890                            VertexKind::FormulaScalar | VertexKind::FormulaArray
9891                        ) {
9892                            refreshed
9893                                .push(TargetProducer::Legacy(vertex))
9894                                .map_err(|_| {
9895                                    target_root_allocation_error(refreshed.len() + 1, request_id)
9896                                })?;
9897                        }
9898                    }
9899                }
9900                other => {
9901                    refreshed.push(other).map_err(|_| {
9902                        target_root_allocation_error(refreshed.len() + 1, request_id)
9903                    })?;
9904                }
9905            }
9906        }
9907        Ok(refreshed.into_vec())
9908    }
9909
9910    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
9911    ///
9912    /// This method prepares graph topology only. It does not evaluate target values.
9913    pub fn prepare_graph_for_targets(
9914        &mut self,
9915        targets: &[crate::engine::EvaluationTarget],
9916        options: crate::engine::TargetEvalOptions<'_>,
9917    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9918        let previous_budgets = self.evaluation_resource_budgets.clone();
9919        let diagnostics_len = self.formula_parse_diagnostics.len();
9920        let previous_report = self.last_formula_ingest_report.clone();
9921        if let Some(budgets) = options.budgets {
9922            self.evaluation_resource_budgets = budgets.clone();
9923        }
9924        let previous_graph_budget_override = self
9925            .graph
9926            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
9927        // Hoist the call's cancellation onto the engine for its duration, so the
9928        // preparation checkpoints observe it. This is a standalone entry point, so
9929        // the previous value is restored rather than cleared.
9930        let previous_cancel = self.active_cancel_flag.take();
9931        self.active_cancel_flag = options.cancel.clone();
9932        let result = self.observe_evaluation_resource_request(
9933            EvaluationRequestKind::TargetPreparation,
9934            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
9935        );
9936        self.active_cancel_flag = previous_cancel;
9937        self.graph
9938            .set_admission_budget_override(previous_graph_budget_override);
9939        self.evaluation_resource_budgets = previous_budgets;
9940        if result.is_err() {
9941            self.formula_parse_diagnostics.truncate(diagnostics_len);
9942            self.last_formula_ingest_report = previous_report;
9943        }
9944        result
9945    }
9946
9947    fn prepare_graph_for_targets_unobserved(
9948        &mut self,
9949        targets: &[crate::engine::EvaluationTarget],
9950        options: &crate::engine::TargetEvalOptions<'_>,
9951    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9952        let scratch_checkpoint = self
9953            .active_resource_ledger
9954            .as_ref()
9955            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
9956        let result = self.prepare_graph_for_targets_transaction(targets, options);
9957        let release = self
9958            .active_resource_ledger
9959            .as_mut()
9960            .map_or(Ok(()), |ledger| {
9961                ledger.release_scratch_to(scratch_checkpoint)
9962            });
9963        match (result, release) {
9964            (result, Ok(())) => result,
9965            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
9966        }
9967    }
9968
9969    fn prepare_graph_for_targets_transaction(
9970        &mut self,
9971        targets: &[crate::engine::EvaluationTarget],
9972        options: &crate::engine::TargetEvalOptions<'_>,
9973    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9974        use crate::engine::{
9975            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
9976            TableSelection,
9977        };
9978
9979        self.target_preparation_checkpoint(options.deadline, 0)?;
9980        self.observe_function_semantic_epoch()?;
9981        let assumptions = self.preparation_revisions();
9982        let ledger_at_start = self
9983            .active_resource_ledger
9984            .as_ref()
9985            .map(|ledger| ledger.snapshot());
9986        let diagnostics_len = self.formula_parse_diagnostics.len();
9987        let report_len = self.last_formula_ingest_report.clone();
9988        let request_id = options.request_id.or_else(|| {
9989            self.active_evaluation_resource_request
9990                .as_ref()
9991                .map(|stats| stats.request_id)
9992        });
9993
9994        let mut scope = PrepareScope::Exact;
9995        let mut reasons = Vec::new();
9996        let mut regions = VecDeque::new();
9997        let mut normalized = Vec::with_capacity(targets.len());
9998        let mut symbol_vertices = VecDeque::new();
9999        for target in targets {
10000            self.target_preparation_checkpoint(options.deadline, 1)?;
10001            match target {
10002                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10003                    if *row == 0 || *col == 0 {
10004                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10005                            .with_message("target cell coordinates are one-based"));
10006                    }
10007                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10008                        ExcelError::new(ExcelErrorKind::Ref)
10009                            .with_message(format!("target sheet not found: {sheet}"))
10010                    })?;
10011                    regions.push_back(PreparationRegion {
10012                        sheet: sheet.clone(),
10013                        sheet_id,
10014                        start_row: *row,
10015                        start_col: *col,
10016                        end_row: *row,
10017                        end_col: *col,
10018                    });
10019                    normalized.push(target.clone());
10020                }
10021                crate::engine::EvaluationTarget::Range(range) => {
10022                    if range.start_row == 0
10023                        || range.start_col == 0
10024                        || range.end_row < range.start_row
10025                        || range.end_col < range.start_col
10026                    {
10027                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10028                            .with_message("invalid target range"));
10029                    }
10030                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10031                        ExcelError::new(ExcelErrorKind::Ref)
10032                            .with_message(format!("target sheet not found: {}", range.sheet))
10033                    })?;
10034                    regions.push_back(PreparationRegion {
10035                        sheet: range.sheet.clone(),
10036                        sheet_id,
10037                        start_row: range.start_row,
10038                        start_col: range.start_col,
10039                        end_row: range.end_row,
10040                        end_col: range.end_col,
10041                    });
10042                    normalized.push(target.clone());
10043                }
10044                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10045                    let current_sheet = match scope_sheet {
10046                        Some(sheet) => self.graph.sheet_id(sheet).ok_or_else(|| {
10047                            ExcelError::new(ExcelErrorKind::Ref)
10048                                .with_message(format!("name scope sheet not found: {sheet}"))
10049                        })?,
10050                        None => self.graph.default_sheet_id(),
10051                    };
10052                    if let Some(entry) = self.graph.resolve_name_entry(name, current_sheet) {
10053                        symbol_vertices.push_back(entry.vertex);
10054                    } else {
10055                        Self::widen_target_preparation(
10056                            options.opaque_policy,
10057                            &mut scope,
10058                            &mut reasons,
10059                            OpaqueReason::UnresolvedName,
10060                        )?;
10061                    }
10062                    normalized.push(target.clone());
10063                }
10064                crate::engine::EvaluationTarget::Table { name, selection } => {
10065                    if let Some(entry) = self.graph.resolve_table_entry(name) {
10066                        let region = self.table_selection_region(entry, selection)?;
10067                        symbol_vertices.push_back(entry.vertex);
10068                        regions.push_back(region);
10069                    } else {
10070                        Self::widen_target_preparation(
10071                            options.opaque_policy,
10072                            &mut scope,
10073                            &mut reasons,
10074                            OpaqueReason::UnresolvedTable,
10075                        )?;
10076                    }
10077                    normalized.push(target.clone());
10078                }
10079            }
10080        }
10081
10082        let mut visited_regions = FxHashSet::default();
10083        let mut visited_vertices = FxHashSet::default();
10084        let mut selected = FxHashSet::default();
10085        let mut prepared = Vec::new();
10086        let mut pending_diagnostics = Vec::new();
10087        let mut planning_requests = BTreeSet::new();
10088        let mut selected_cells = Vec::new();
10089        let mut workbook_seeded = false;
10090        let mut sheet_scope_seeded = BTreeSet::new();
10091        let mut indexed_query_sheets = FxHashSet::default();
10092        let mut discovery_scratch_reserved = 0u64;
10093        let mut package_encountered = false;
10094        let mut selected_package_sheets = FxHashSet::default();
10095        let mut prepared_packages = Vec::new();
10096        let authoritative_with_ordinary = self.config.formula_plane_mode
10097            == FormulaPlaneMode::AuthoritativeExperimental
10098            && self.staged_formula_index.ordinary_count() != 0;
10099        let has_unknown_package_sheet = self
10100            .staged_formula_index
10101            .package_sheets()
10102            .any(|sheet| self.graph.sheet_id(sheet).is_none());
10103
10104        loop {
10105            if let PrepareScope::Sheets(sheets) = &scope {
10106                for sheet in sheets.clone() {
10107                    if !sheet_scope_seeded.insert(sheet.clone()) {
10108                        continue;
10109                    }
10110                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10111                        package_encountered = true;
10112                        Self::widen_target_preparation(
10113                            options.opaque_policy,
10114                            &mut scope,
10115                            &mut reasons,
10116                            OpaqueReason::UnsupportedSourceSemantics,
10117                        )?;
10118                        continue;
10119                    };
10120                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10121                        self.target_preparation_checkpoint(options.deadline, 1)?;
10122                        regions.push_back(PreparationRegion {
10123                            sheet: sheet.clone(),
10124                            sheet_id,
10125                            start_row: lease.row,
10126                            start_col: lease.col,
10127                            end_row: lease.row,
10128                            end_col: lease.col,
10129                        });
10130                    }
10131                    if self
10132                        .staged_formula_index
10133                        .package_lease_for_sheet(&sheet)
10134                        .is_some()
10135                    {
10136                        regions.push_back(PreparationRegion {
10137                            sheet: sheet.clone(),
10138                            sheet_id,
10139                            start_row: 1,
10140                            start_col: 1,
10141                            end_row: self.workbook_load_limits.max_sheet_rows,
10142                            end_col: self.workbook_load_limits.max_sheet_cols,
10143                        });
10144                    }
10145                }
10146            }
10147            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10148                workbook_seeded = true;
10149                for (sheet, lease) in self.staged_formula_index.all_leases() {
10150                    self.target_preparation_checkpoint(options.deadline, 1)?;
10151                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10152                        package_encountered = true;
10153                        Self::widen_target_preparation(
10154                            options.opaque_policy,
10155                            &mut scope,
10156                            &mut reasons,
10157                            OpaqueReason::UnsupportedSourceSemantics,
10158                        )?;
10159                        continue;
10160                    };
10161                    regions.push_back(PreparationRegion {
10162                        sheet,
10163                        sheet_id,
10164                        start_row: lease.row,
10165                        start_col: lease.col,
10166                        end_row: lease.row,
10167                        end_col: lease.col,
10168                    });
10169                }
10170                let package_sheets = self
10171                    .staged_formula_index
10172                    .package_sheets()
10173                    .map(str::to_string)
10174                    .collect::<Vec<_>>();
10175                for sheet in package_sheets {
10176                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10177                        package_encountered = true;
10178                        Self::widen_target_preparation(
10179                            options.opaque_policy,
10180                            &mut scope,
10181                            &mut reasons,
10182                            OpaqueReason::UnsupportedSourceSemantics,
10183                        )?;
10184                        continue;
10185                    };
10186                    regions.push_back(PreparationRegion {
10187                        sheet,
10188                        sheet_id,
10189                        start_row: 1,
10190                        start_col: 1,
10191                        end_row: self.workbook_load_limits.max_sheet_rows,
10192                        end_col: self.workbook_load_limits.max_sheet_cols,
10193                    });
10194                }
10195            }
10196
10197            let Some(region) = regions.pop_front() else {
10198                if let Some(vertex) = symbol_vertices.pop_front() {
10199                    self.target_preparation_checkpoint(options.deadline, 1)?;
10200                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10201                        continue;
10202                    }
10203                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10204                    if let Some(ast) = self.graph.get_formula(vertex) {
10205                        let snapshot =
10206                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
10207                        if let Some(reason) =
10208                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10209                        {
10210                            return Err(Self::preparation_stale(
10211                                reason,
10212                                "target planning snapshot became stale during discovery",
10213                            ));
10214                        }
10215                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10216                        if let Some(reason) =
10217                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10218                        {
10219                            if reason == OpaqueReason::DynamicReference
10220                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10221                            {
10222                                let sheet = self.graph.get_vertex_sheet_id(vertex);
10223                                let sheet = self.graph.sheet_name(sheet).to_string();
10224                                Self::widen_target_preparation_to_sheet(
10225                                    options.opaque_policy,
10226                                    &mut scope,
10227                                    &mut reasons,
10228                                    reason,
10229                                    &sheet,
10230                                )?;
10231                            } else {
10232                                Self::widen_target_preparation(
10233                                    options.opaque_policy,
10234                                    &mut scope,
10235                                    &mut reasons,
10236                                    reason,
10237                                )?;
10238                            }
10239                        }
10240                    } else if vertex_is_dynamic {
10241                        Self::widen_target_preparation(
10242                            options.opaque_policy,
10243                            &mut scope,
10244                            &mut reasons,
10245                            OpaqueReason::DynamicReference,
10246                        )?;
10247                    }
10248                    if let Some(anchor) = self
10249                        .graph
10250                        .get_cell_ref(vertex)
10251                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
10252                    {
10253                        symbol_vertices.push_back(anchor);
10254                    }
10255                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
10256                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
10257                        regions.push_back(PreparationRegion {
10258                            sheet,
10259                            sheet_id: cell.sheet_id,
10260                            start_row: cell.coord.row() + 1,
10261                            start_col: cell.coord.col() + 1,
10262                            end_row: cell.coord.row() + 1,
10263                            end_col: cell.coord.col() + 1,
10264                        });
10265                    }
10266                    for dependency in self.graph.get_dependencies(vertex) {
10267                        self.target_preparation_checkpoint(options.deadline, 1)?;
10268                        symbol_vertices.push_back(dependency);
10269                    }
10270                    if let Some(range_dependencies) = self
10271                        .graph
10272                        .formula_range_dependencies(vertex)
10273                        .map(<[_]>::to_vec)
10274                    {
10275                        for range in range_dependencies {
10276                            self.target_preparation_checkpoint(options.deadline, 1)?;
10277                            let sheet_id = match range.sheet {
10278                                crate::reference::SharedSheetLocator::Id(id) => id,
10279                                crate::reference::SharedSheetLocator::Current => {
10280                                    self.graph.get_vertex_sheet_id(vertex)
10281                                }
10282                                crate::reference::SharedSheetLocator::Name(ref name) => {
10283                                    let Some(id) = self.graph.sheet_id(name) else {
10284                                        if Self::widen_target_preparation(
10285                                            options.opaque_policy,
10286                                            &mut scope,
10287                                            &mut reasons,
10288                                            OpaqueReason::UnresolvedCrossSheetBinding,
10289                                        )? {
10290                                            break;
10291                                        }
10292                                        continue;
10293                                    };
10294                                    id
10295                                }
10296                            };
10297                            let sheet = self.graph.sheet_name(sheet_id).to_string();
10298                            regions.push_back(PreparationRegion {
10299                                sheet,
10300                                sheet_id,
10301                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10302                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10303                                end_row: range
10304                                    .end_row
10305                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10306                                        bound.index + 1
10307                                    }),
10308                                end_col: range
10309                                    .end_col
10310                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10311                                        bound.index + 1
10312                                    }),
10313                            });
10314                        }
10315                    }
10316                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
10317                        match &name.definition {
10318                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
10319                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
10320                                sheet_id: cell.sheet_id,
10321                                start_row: cell.coord.row() + 1,
10322                                start_col: cell.coord.col() + 1,
10323                                end_row: cell.coord.row() + 1,
10324                                end_col: cell.coord.col() + 1,
10325                            }),
10326                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
10327                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
10328                                sheet_id: range.start.sheet_id,
10329                                start_row: range.start.coord.row() + 1,
10330                                start_col: range.start.coord.col() + 1,
10331                                end_row: range.end.coord.row() + 1,
10332                                end_col: range.end.coord.col() + 1,
10333                            }),
10334                            NamedDefinition::Formula {
10335                                ast,
10336                                dependencies,
10337                                range_deps,
10338                            } => {
10339                                let snapshot =
10340                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
10341                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
10342                                    &snapshot,
10343                                    &assumptions,
10344                                ) {
10345                                    return Err(Self::preparation_stale(
10346                                        reason,
10347                                        "target planning snapshot became stale during discovery",
10348                                    ));
10349                                }
10350                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
10351                                    Self::widen_target_preparation(
10352                                        options.opaque_policy,
10353                                        &mut scope,
10354                                        &mut reasons,
10355                                        reason,
10356                                    )?;
10357                                }
10358                                for dependency in dependencies {
10359                                    self.target_preparation_checkpoint(options.deadline, 1)?;
10360                                    symbol_vertices.push_back(*dependency);
10361                                }
10362                                for range in range_deps {
10363                                    self.target_preparation_checkpoint(options.deadline, 1)?;
10364                                    let sheet_id = match range.sheet {
10365                                        crate::reference::SharedSheetLocator::Id(id) => id,
10366                                        _ => self.graph.default_sheet_id(),
10367                                    };
10368                                    regions.push_back(PreparationRegion {
10369                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
10370                                        sheet_id,
10371                                        start_row: range
10372                                            .start_row
10373                                            .map_or(1, |bound| bound.index + 1),
10374                                        start_col: range
10375                                            .start_col
10376                                            .map_or(1, |bound| bound.index + 1),
10377                                        end_row: range.end_row.map_or(
10378                                            self.workbook_load_limits.max_sheet_rows,
10379                                            |bound| bound.index + 1,
10380                                        ),
10381                                        end_col: range.end_col.map_or(
10382                                            self.workbook_load_limits.max_sheet_cols,
10383                                            |bound| bound.index + 1,
10384                                        ),
10385                                    });
10386                                }
10387                            }
10388                            NamedDefinition::Literal(_) => {}
10389                        }
10390                    }
10391                    if let Some(table) = self.graph.table_by_vertex(vertex) {
10392                        regions
10393                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
10394                    }
10395                    continue;
10396                }
10397                break;
10398            };
10399
10400            self.target_preparation_checkpoint(options.deadline, 1)?;
10401            if !visited_regions.insert(region.clone()) {
10402                continue;
10403            }
10404            if let PrepareScope::Sheets(sheets) = &mut scope
10405                && !sheets.iter().any(|sheet| sheet == &region.sheet)
10406            {
10407                sheets.push(region.sheet.clone());
10408                sheets.sort();
10409            }
10410            let package_match = self.staged_formula_index.package_for_region(
10411                &region.sheet,
10412                region.start_row,
10413                region.start_col,
10414                region.end_row,
10415                region.end_col,
10416            );
10417            let package_lease = match package_match {
10418                Some(Ok(lease)) => Some(lease),
10419                Some(Err(()))
10420                    if region.start_row == 1
10421                        && region.start_col == 1
10422                        && region.end_row == self.workbook_load_limits.max_sheet_rows
10423                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
10424                {
10425                    self.staged_formula_index
10426                        .package_lease_for_sheet(&region.sheet)
10427                }
10428                Some(Err(())) => {
10429                    Self::widen_target_preparation(
10430                        options.opaque_policy,
10431                        &mut scope,
10432                        &mut reasons,
10433                        OpaqueReason::DeferredSourcePackage,
10434                    )?;
10435                    None
10436                }
10437                None => None,
10438            };
10439            let compatibility_before_package_replay = package_lease.is_some()
10440                && (authoritative_with_ordinary || has_unknown_package_sheet);
10441            let package_lease = if compatibility_before_package_replay {
10442                package_encountered = true;
10443                Self::widen_target_preparation(
10444                    options.opaque_policy,
10445                    &mut scope,
10446                    &mut reasons,
10447                    OpaqueReason::UnsupportedSourceSemantics,
10448                )?;
10449                None
10450            } else {
10451                package_lease
10452            };
10453            if let Some(package_lease) = package_lease
10454                && selected_package_sheets.insert(region.sheet.clone())
10455            {
10456                self.target_preparation_checkpoint(options.deadline, 1)?;
10457                let mut package = self.prepare_target_source_package(
10458                    &region.sheet,
10459                    package_lease,
10460                    options.deadline,
10461                )?;
10462                for placement in &package.placements {
10463                    self.target_preparation_checkpoint(options.deadline, 1)?;
10464                    for dependency in &placement.fragment_dependency_proof().1.dependencies {
10465                        self.target_preparation_checkpoint(options.deadline, 1)?;
10466                        let (rows, cols) = dependency.read_region.axis_ranges();
10467                        let (start_row, end_row) = rows.query_bounds();
10468                        let (start_col, end_col) = cols.query_bounds();
10469                        let dependency_sheet = dependency.read_region.sheet_id();
10470                        regions.push_back(PreparationRegion {
10471                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10472                            sheet_id: dependency_sheet,
10473                            start_row: start_row.saturating_add(1),
10474                            start_col: start_col.saturating_add(1),
10475                            end_row: end_row
10476                                .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
10477                                .saturating_add(1),
10478                            end_col: end_col
10479                                .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
10480                                .saturating_add(1),
10481                        });
10482                    }
10483                }
10484
10485                let mut final_fallback = BTreeMap::new();
10486                for record in package.fallback_records() {
10487                    final_fallback.insert((record.row, record.col), record.clone());
10488                }
10489                let batch = self
10490                    .formula_batch_from_exact_replay(&region.sheet, final_fallback.into_values())?;
10491                for record in batch.formulas {
10492                    self.target_preparation_checkpoint(options.deadline, 1)?;
10493                    let ast = self
10494                        .graph
10495                        .data_store()
10496                        .retrieve_ast(record.ast_id, self.graph.sheet_reg())
10497                        .ok_or_else(|| {
10498                            ExcelError::new(ExcelErrorKind::Value)
10499                                .with_message("target fallback AST is unavailable")
10500                        })?;
10501                    let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10502                    if let Some(reason) =
10503                        Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10504                    {
10505                        return Err(Self::preparation_stale(
10506                            reason,
10507                            "target fallback planning snapshot became stale during discovery",
10508                        ));
10509                    }
10510                    let proven_sheet_local_dynamic =
10511                        Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10512                    if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10513                        if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
10514                            Self::widen_target_preparation_to_sheet(
10515                                options.opaque_policy,
10516                                &mut scope,
10517                                &mut reasons,
10518                                reason,
10519                                &region.sheet,
10520                            )?;
10521                        } else {
10522                            Self::widen_target_preparation(
10523                                options.opaque_policy,
10524                                &mut scope,
10525                                &mut reasons,
10526                                reason,
10527                            )?;
10528                        }
10529                    }
10530                    let placement = CellRef::new(
10531                        package.sheet_id,
10532                        Coord::from_excel(record.row, record.col, true, true),
10533                    );
10534                    let ingested = self
10535                        .graph
10536                        .ingest_pipeline(&snapshot)
10537                        .enable_function_semantics()
10538                        .ingest_formula(
10539                            FormulaAstInput::RawArena(record.ast_id),
10540                            placement,
10541                            record.formula_text,
10542                        )?;
10543                    if ingested.dep_plan.dynamic {
10544                        if proven_sheet_local_dynamic {
10545                            Self::widen_target_preparation_to_sheet(
10546                                options.opaque_policy,
10547                                &mut scope,
10548                                &mut reasons,
10549                                OpaqueReason::DynamicReference,
10550                                &region.sheet,
10551                            )?;
10552                        } else {
10553                            Self::widen_target_preparation(
10554                                options.opaque_policy,
10555                                &mut scope,
10556                                &mut reasons,
10557                                OpaqueReason::DynamicReference,
10558                            )?;
10559                        }
10560                    }
10561                    for dep in &ingested.dep_plan.direct_cell_deps {
10562                        self.target_preparation_checkpoint(options.deadline, 1)?;
10563                        regions.push_back(PreparationRegion {
10564                            sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10565                            sheet_id: dep.sheet_id,
10566                            start_row: dep.coord.row().saturating_add(1),
10567                            start_col: dep.coord.col().saturating_add(1),
10568                            end_row: dep.coord.row().saturating_add(1),
10569                            end_col: dep.coord.col().saturating_add(1),
10570                        });
10571                    }
10572                    for range in &ingested.dep_plan.range_deps {
10573                        self.target_preparation_checkpoint(options.deadline, 1)?;
10574                        let dependency_sheet = match &range.sheet {
10575                            crate::reference::SharedSheetLocator::Id(id) => *id,
10576                            crate::reference::SharedSheetLocator::Current => package.sheet_id,
10577                            crate::reference::SharedSheetLocator::Name(name) => {
10578                                let Some(id) = self.graph.sheet_id(name) else {
10579                                    Self::widen_target_preparation(
10580                                        options.opaque_policy,
10581                                        &mut scope,
10582                                        &mut reasons,
10583                                        OpaqueReason::UnresolvedCrossSheetBinding,
10584                                    )?;
10585                                    continue;
10586                                };
10587                                id
10588                            }
10589                        };
10590                        regions.push_back(PreparationRegion {
10591                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10592                            sheet_id: dependency_sheet,
10593                            start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10594                            start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10595                            end_row: range
10596                                .end_row
10597                                .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10598                                    bound.index + 1
10599                                }),
10600                            end_col: range
10601                                .end_col
10602                                .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10603                                    bound.index + 1
10604                                }),
10605                        });
10606                    }
10607                    for name in ingested
10608                        .dep_plan
10609                        .resolved_named_refs
10610                        .iter()
10611                        .chain(&ingested.dep_plan.named_refs)
10612                    {
10613                        self.target_preparation_checkpoint(options.deadline, 1)?;
10614                        if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
10615                            symbol_vertices.push_back(entry.vertex);
10616                        } else if self.graph.resolve_source_scalar_entry(name).is_none()
10617                            && self.graph.resolve_source_table_entry(name).is_none()
10618                        {
10619                            Self::widen_target_preparation(
10620                                options.opaque_policy,
10621                                &mut scope,
10622                                &mut reasons,
10623                                OpaqueReason::UnresolvedName,
10624                            )?;
10625                        }
10626                    }
10627                    for table in &ingested.dep_plan.table_refs {
10628                        self.target_preparation_checkpoint(options.deadline, 1)?;
10629                        if let Some(entry) = self.graph.resolve_table_entry(table) {
10630                            symbol_vertices.push_back(entry.vertex);
10631                        } else if self.graph.resolve_source_table_entry(table).is_none() {
10632                            Self::widen_target_preparation(
10633                                options.opaque_policy,
10634                                &mut scope,
10635                                &mut reasons,
10636                                OpaqueReason::UnresolvedTable,
10637                            )?;
10638                        }
10639                    }
10640                    package.legacy.push((
10641                        record.row,
10642                        record.col,
10643                        ingested.ast_id,
10644                        ingested.dep_plan,
10645                    ));
10646                }
10647                prepared_packages.push(package);
10648            }
10649            let leases = self.staged_formula_index.leases_in_region(
10650                &region.sheet,
10651                region.start_row,
10652                region.start_col,
10653                region.end_row,
10654                region.end_col,
10655            );
10656            for lease in leases {
10657                self.target_preparation_checkpoint(options.deadline, 1)?;
10658                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
10659                    ExcelError::new(ExcelErrorKind::Ref)
10660                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
10661                })?;
10662                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
10663                if !selected.insert(key) {
10664                    continue;
10665                }
10666                let text = self
10667                    .staged_formulas
10668                    .get(&region.sheet)
10669                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
10670                    .ok_or_else(|| {
10671                        ExcelError::new(ExcelErrorKind::Value)
10672                            .with_message("staged formula index is stale")
10673                    })?
10674                    .to_string();
10675                let formula = if text.starts_with('=') {
10676                    text.clone()
10677                } else {
10678                    format!("={text}")
10679                };
10680                self.target_preparation_checkpoint(options.deadline, 1)?;
10681                let ast = match formualizer_parse::parser::parse(&formula) {
10682                    Ok(ast) => ast,
10683                    Err(error) => {
10684                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
10685                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
10686                                format!(
10687                                    "Formula parse error at {}!{}{}: {error}",
10688                                    region.sheet,
10689                                    col_letters_from_1based(lease.col)
10690                                        .unwrap_or_else(|_| "?".to_string()),
10691                                    lease.row
10692                                ),
10693                            ));
10694                        }
10695                        pending_diagnostics.push(FormulaParseDiagnostic {
10696                            sheet: region.sheet.clone(),
10697                            row: lease.row,
10698                            col: lease.col,
10699                            formula: formula.clone(),
10700                            message: error.to_string(),
10701                            policy: self.config.formula_parse_policy,
10702                        });
10703                        match self.config.formula_parse_policy {
10704                            FormulaParsePolicy::KeepCachedValue => {
10705                                selected_cells.push(
10706                                    formualizer_common::RangeAddress::new(
10707                                        region.sheet.clone(),
10708                                        lease.row,
10709                                        lease.col,
10710                                        lease.row,
10711                                        lease.col,
10712                                    )
10713                                    .expect("selected staged coordinates are valid"),
10714                                );
10715                                prepared.push(PreparedOrdinaryStagedFormula {
10716                                    sheet: region.sheet.clone(),
10717                                    sheet_id,
10718                                    lease,
10719                                    ast_id: None,
10720                                    plan: None,
10721                                });
10722                                continue;
10723                            }
10724                            FormulaParsePolicy::AsText => ASTNode::new(
10725                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
10726                                None,
10727                            ),
10728                            FormulaParsePolicy::CoerceToError => ASTNode::new(
10729                                ASTNodeType::Literal(LiteralValue::Error(
10730                                    ExcelError::new(ExcelErrorKind::Error)
10731                                        .with_message(format!("Malformed formula: {error}")),
10732                                )),
10733                                None,
10734                            ),
10735                            FormulaParsePolicy::Strict => unreachable!(),
10736                        }
10737                    }
10738                };
10739                self.target_preparation_checkpoint(options.deadline, 1)?;
10740                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10741                self.target_preparation_checkpoint(options.deadline, 1)?;
10742                if let Some(reason) =
10743                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10744                {
10745                    return Err(Self::preparation_stale(
10746                        reason,
10747                        "target planning snapshot became stale during discovery",
10748                    ));
10749                }
10750                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10751                    if reason == OpaqueReason::DynamicReference
10752                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10753                    {
10754                        Self::widen_target_preparation_to_sheet(
10755                            options.opaque_policy,
10756                            &mut scope,
10757                            &mut reasons,
10758                            reason,
10759                            &region.sheet,
10760                        )?;
10761                    } else {
10762                        Self::widen_target_preparation(
10763                            options.opaque_policy,
10764                            &mut scope,
10765                            &mut reasons,
10766                            reason,
10767                        )?;
10768                    }
10769                }
10770                let proven_sheet_local_dynamic =
10771                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10772                let placement = CellRef::new(
10773                    sheet_id,
10774                    Coord::from_excel(lease.row, lease.col, true, true),
10775                );
10776                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
10777                    FormulaAstInput::Tree(ast),
10778                    placement,
10779                    Some(Arc::from(formula)),
10780                )?;
10781                self.target_preparation_checkpoint(options.deadline, 1)?;
10782                if ingested.dep_plan.dynamic {
10783                    if proven_sheet_local_dynamic {
10784                        Self::widen_target_preparation_to_sheet(
10785                            options.opaque_policy,
10786                            &mut scope,
10787                            &mut reasons,
10788                            OpaqueReason::DynamicReference,
10789                            &region.sheet,
10790                        )?;
10791                    } else {
10792                        Self::widen_target_preparation(
10793                            options.opaque_policy,
10794                            &mut scope,
10795                            &mut reasons,
10796                            OpaqueReason::DynamicReference,
10797                        )?;
10798                    }
10799                }
10800                for dep in &ingested.dep_plan.direct_cell_deps {
10801                    self.target_preparation_checkpoint(options.deadline, 1)?;
10802                    regions.push_back(PreparationRegion {
10803                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10804                        sheet_id: dep.sheet_id,
10805                        start_row: dep.coord.row() + 1,
10806                        start_col: dep.coord.col() + 1,
10807                        end_row: dep.coord.row() + 1,
10808                        end_col: dep.coord.col() + 1,
10809                    });
10810                }
10811                for range in &ingested.dep_plan.range_deps {
10812                    self.target_preparation_checkpoint(options.deadline, 1)?;
10813                    let sheet_id = match range.sheet {
10814                        crate::reference::SharedSheetLocator::Id(id) => id,
10815                        crate::reference::SharedSheetLocator::Current => sheet_id,
10816                        crate::reference::SharedSheetLocator::Name(ref name) => {
10817                            let Some(id) = self.graph.sheet_id(name) else {
10818                                Self::widen_target_preparation(
10819                                    options.opaque_policy,
10820                                    &mut scope,
10821                                    &mut reasons,
10822                                    OpaqueReason::UnresolvedCrossSheetBinding,
10823                                )?;
10824                                continue;
10825                            };
10826                            id
10827                        }
10828                    };
10829                    regions.push_back(PreparationRegion {
10830                        sheet: self.graph.sheet_name(sheet_id).to_string(),
10831                        sheet_id,
10832                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10833                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10834                        end_row: range
10835                            .end_row
10836                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10837                                bound.index + 1
10838                            }),
10839                        end_col: range
10840                            .end_col
10841                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10842                                bound.index + 1
10843                            }),
10844                    });
10845                }
10846                for name in ingested
10847                    .dep_plan
10848                    .resolved_named_refs
10849                    .iter()
10850                    .chain(&ingested.dep_plan.named_refs)
10851                {
10852                    self.target_preparation_checkpoint(options.deadline, 1)?;
10853                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
10854                        symbol_vertices.push_back(entry.vertex);
10855                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
10856                        && self.graph.resolve_source_table_entry(name).is_none()
10857                    {
10858                        Self::widen_target_preparation(
10859                            options.opaque_policy,
10860                            &mut scope,
10861                            &mut reasons,
10862                            OpaqueReason::UnresolvedName,
10863                        )?;
10864                    }
10865                }
10866                for table in &ingested.dep_plan.table_refs {
10867                    self.target_preparation_checkpoint(options.deadline, 1)?;
10868                    if let Some(entry) = self.graph.resolve_table_entry(table) {
10869                        symbol_vertices.push_back(entry.vertex);
10870                    } else if self.graph.resolve_source_table_entry(table).is_none() {
10871                        Self::widen_target_preparation(
10872                            options.opaque_policy,
10873                            &mut scope,
10874                            &mut reasons,
10875                            OpaqueReason::UnresolvedTable,
10876                        )?;
10877                    }
10878                }
10879                selected_cells.push(
10880                    formualizer_common::RangeAddress::new(
10881                        region.sheet.clone(),
10882                        lease.row,
10883                        lease.col,
10884                        lease.row,
10885                        lease.col,
10886                    )
10887                    .expect("selected staged coordinates are valid"),
10888                );
10889                prepared.push(PreparedOrdinaryStagedFormula {
10890                    sheet: region.sheet.clone(),
10891                    sheet_id,
10892                    lease,
10893                    ast_id: Some(ingested.ast_id),
10894                    plan: Some(ingested.dep_plan),
10895                });
10896            }
10897
10898            if indexed_query_sheets.insert(region.sheet_id) {
10899                self.graph.prepare_sheet_index_for_query(region.sheet_id);
10900                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
10901                    .saturating_mul(32);
10902                self.reserve_graph_source_scratch(bytes)?;
10903                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
10904            }
10905            let spill_anchors = self.graph.spill_anchors_in_region(
10906                region.sheet_id,
10907                region.start_row - 1,
10908                region.start_col - 1,
10909                region.end_row - 1,
10910                region.end_col - 1,
10911            );
10912            for anchor in spill_anchors {
10913                self.target_preparation_checkpoint(options.deadline, 1)?;
10914                symbol_vertices.push_back(anchor);
10915            }
10916            let vertices = self.graph.vertices_in_region(
10917                region.sheet_id,
10918                region.start_row - 1,
10919                region.end_row - 1,
10920                region.start_col - 1,
10921                region.end_col - 1,
10922            );
10923            for vertex in vertices {
10924                self.target_preparation_checkpoint(options.deadline, 1)?;
10925                symbol_vertices.push_back(vertex);
10926            }
10927        }
10928
10929        #[cfg(test)]
10930        self.target_preparation_fault(
10931            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
10932        )?;
10933
10934        if package_encountered
10935            || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
10936                && !prepared.is_empty())
10937        {
10938            Self::widen_target_preparation(
10939                options.opaque_policy,
10940                &mut scope,
10941                &mut reasons,
10942                OpaqueReason::UnsupportedSourceSemantics,
10943            )?;
10944            self.target_preparation_checkpoint(options.deadline, 0)?;
10945            let selected_count = self.staged_formula_count();
10946            let selected_packages = self
10947                .staged_formulas
10948                .values()
10949                .filter_map(|staged| staged.deferred_package.as_ref())
10950                .map(|package| package.families.len() + package.partitioned_families.len())
10951                .sum();
10952            self.formula_parse_diagnostics.truncate(diagnostics_len);
10953            self.last_formula_ingest_report = report_len;
10954            #[cfg(test)]
10955            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
10956                hook();
10957            }
10958            self.target_preparation_checkpoint(options.deadline, 0)?;
10959            #[cfg(test)]
10960            self.target_preparation_fault(
10961                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
10962            )?;
10963            let current_revisions = self.preparation_revisions();
10964            if let Some(reason) = Self::preparation_revision_stale_reason(
10965                &assumptions,
10966                &current_revisions,
10967                &planning_requests,
10968                true,
10969            ) {
10970                return Err(Self::preparation_stale(
10971                    reason,
10972                    "target compatibility preparation plan is stale",
10973                ));
10974            }
10975            #[cfg(test)]
10976            self.target_preparation_fault(
10977                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
10978            )?;
10979            let commit_work_before = self
10980                .active_resource_ledger
10981                .as_ref()
10982                .map_or(0, |ledger| ledger.snapshot().work_charged);
10983            let commit_started = crate::instant::FzInstant::now();
10984            self.build_graph_all_unobserved()?;
10985            let commit_window = commit_started.elapsed();
10986            let ledger_after = self
10987                .active_resource_ledger
10988                .as_ref()
10989                .map(|ledger| ledger.snapshot());
10990            let actual_commit_work = ledger_after
10991                .map_or(0, |snapshot| snapshot.work_charged)
10992                .saturating_sub(commit_work_before);
10993            let observed_scratch_bytes = ledger_after
10994                .map_or(0, |snapshot| snapshot.scratch_peak)
10995                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
10996            let revisions = assumptions.clone();
10997            let report = PreparedTargetGraphReport {
10998                request_id: request_id.unwrap_or_default(),
10999                requested_targets: targets.len(),
11000                normalized_regions: visited_regions.len(),
11001                normalized_target_list: normalized,
11002                selected_staged_cells: selected_count,
11003                selected_source_families: selected_packages,
11004                retained_staged_cells: self.staged_formula_count(),
11005                selected_cells,
11006                retained_cells: Vec::new(),
11007                widened_scope: PrepareScope::Workbook,
11008                widening_reasons: reasons,
11009                revisions,
11010                commit_window,
11011                estimated_scratch_bytes: discovery_scratch_reserved
11012                    .saturating_add((selected_count as u64).saturating_mul(256)),
11013                observed_scratch_bytes,
11014                estimated_commit_work: selected_count as u64,
11015                actual_commit_work,
11016                outcome: PreparationOutcome::CompatibilityPrepared,
11017            };
11018            self.observe_target_preparation_report(&report);
11019            return Ok(report);
11020        }
11021
11022        prepared.sort_by_key(|formula| formula.lease.insertion_order);
11023        for package in &prepared_packages {
11024            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11025                formualizer_common::RangeAddress::new(
11026                    package.sheet.clone(),
11027                    record.row,
11028                    record.col,
11029                    record.row,
11030                    record.col,
11031                )
11032                .ok()
11033            }));
11034        }
11035        let (mut legacy_graph, mut planned_formula_count) =
11036            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11037        let placements = prepared_packages
11038            .iter_mut()
11039            .flat_map(|package| std::mem::take(&mut package.placements))
11040            .collect::<Vec<_>>();
11041        let formula_plane = if placements.is_empty() {
11042            None
11043        } else {
11044            match self.graph.formula_authority().prepare_formula_plane_append(
11045                placements,
11046                self.graph.data_store(),
11047                self.graph.sheet_reg(),
11048            ) {
11049                Ok(append) => Some(append),
11050                Err(error) => {
11051                    let reason = format!("TargetFormulaPlaneAppend:{error}");
11052                    for package in &mut prepared_packages {
11053                        *package
11054                            .source_report
11055                            .fallback_reasons
11056                            .entry(reason.clone())
11057                            .or_default() += package.direct_families as u64;
11058                        if self.config.formula_plane_mode
11059                            == FormulaPlaneMode::AuthoritativeExperimental
11060                        {
11061                            self.materialize_target_package_direct_records(
11062                                package,
11063                                &assumptions,
11064                                &mut planning_requests,
11065                                options.deadline,
11066                            )?;
11067                        } else {
11068                            package.direct_families = 0;
11069                            package.direct_cells = 0;
11070                            package.direct_fragments = 0;
11071                            package.direct_complete_families = 0;
11072                            package.direct_complete_cells = 0;
11073                            package.direct_partition_families = 0;
11074                            package.direct_partition_cells = 0;
11075                        }
11076                    }
11077                    (legacy_graph, planned_formula_count) =
11078                        self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11079                    None
11080                }
11081            }
11082        };
11083        let new_vertices = legacy_graph.new_vertex_count();
11084        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11085            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11086        })?;
11087        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11088            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11089        })?;
11090        let current = self.graph.baseline_stats();
11091        let final_vertices = current
11092            .graph_vertex_count
11093            .checked_add(new_vertices)
11094            .ok_or_else(|| {
11095                crate::engine::ResourceLedgerError::Exhausted(
11096                    formualizer_common::ResourceExhaustionDetail {
11097                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11098                        limit: u64::MAX,
11099                        observed: u64::MAX,
11100                        request_id,
11101                    },
11102                )
11103                .into_excel_error()
11104            })?;
11105        let final_edges = current
11106            .graph_edge_count
11107            .checked_sub(removed_edges)
11108            .and_then(|count| count.checked_add(new_edges))
11109            .ok_or_else(|| {
11110                crate::engine::ResourceLedgerError::Exhausted(
11111                    formualizer_common::ResourceExhaustionDetail {
11112                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11113                        limit: u64::MAX,
11114                        observed: u64::MAX,
11115                        request_id,
11116                    },
11117                )
11118                .into_excel_error()
11119            })?;
11120        #[cfg(test)]
11121        self.target_preparation_fault(
11122            crate::engine::target_preparation::TargetPreparationFault::Admission,
11123        )?;
11124        let resource = |reason, limit: u64, observed: u64| {
11125            crate::engine::ResourceLedgerError::Exhausted(
11126                formualizer_common::ResourceExhaustionDetail {
11127                    reason,
11128                    limit,
11129                    observed,
11130                    request_id,
11131                },
11132            )
11133            .into_excel_error()
11134        };
11135        let admission = crate::engine::resource_ledger::GraphAdmission {
11136            final_vertices,
11137            final_edges,
11138            materialization_cells: planned_formula_count as u64,
11139            added_vertices: new_vertices,
11140            added_edges: new_edges,
11141        };
11142        let materialized_bytes = admission
11143            .materialized_graph_bytes()
11144            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11145        if let Err(error) = self.preflight_graph_admission(admission) {
11146            if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11147                self.observe_target_admission_failure(detail.reason);
11148            }
11149            return Err(error);
11150        }
11151        let selected_package_records = prepared_packages
11152            .iter()
11153            .map(|package| package.replay_records.len() as u64)
11154            .sum::<u64>();
11155        let planned_working_bytes = (prepared.len() as u64)
11156            .saturating_add(selected_package_records)
11157            .saturating_mul(256)
11158            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11159            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11160            .saturating_add((selected.len() as u64).saturating_mul(96))
11161            .saturating_add(materialized_bytes);
11162        let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11163        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11164        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11165            self.observe_target_admission_failure(
11166                formualizer_common::ResourceExhaustionReason::ScratchMemory,
11167            );
11168            return Err(error);
11169        }
11170
11171        let estimated_commit_duration = std::time::Duration::from_nanos(
11172            (new_vertices as u64)
11173                .saturating_add(new_edges as u64)
11174                .saturating_add(prepared.len() as u64)
11175                .saturating_add(selected_package_records)
11176                .max(1)
11177                .saturating_mul(100),
11178        );
11179        if options.deadline.is_some_and(|deadline| {
11180            std::time::Instant::now()
11181                .checked_add(estimated_commit_duration)
11182                .is_none_or(|finish| finish > deadline)
11183        }) {
11184            self.observe_target_admission_failure(
11185                formualizer_common::ResourceExhaustionReason::Deadline,
11186            );
11187            return Err(resource(
11188                formualizer_common::ResourceExhaustionReason::Deadline,
11189                0,
11190                1,
11191            ));
11192        }
11193        #[cfg(test)]
11194        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11195            hook();
11196        }
11197        self.target_preparation_checkpoint(options.deadline, 0)?;
11198        #[cfg(test)]
11199        self.target_preparation_fault(
11200            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11201        )?;
11202        let current_revisions = self.preparation_revisions();
11203        let staged_leases_match = prepared.iter().all(|formula| {
11204            self.staged_formula_index
11205                .lease_matches(&formula.sheet, formula.lease)
11206        }) && prepared_packages.iter().all(|package| {
11207            self.staged_formula_index
11208                .package_lease_matches(&package.sheet, package.lease)
11209        });
11210        let stale_reason = Self::preparation_revision_stale_reason(
11211            &assumptions,
11212            &current_revisions,
11213            &planning_requests,
11214            staged_leases_match,
11215        );
11216        if let Some(reason) = stale_reason {
11217            return Err(Self::preparation_stale(
11218                reason,
11219                "target graph preparation plan is stale",
11220            ));
11221        }
11222        #[cfg(test)]
11223        self.target_preparation_fault(
11224            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11225        )?;
11226        self.graph
11227            .validate_prepared_legacy_graph_plan(&legacy_graph)
11228            .map_err(|error| {
11229                Self::preparation_stale(
11230                    formualizer_common::PreparationStaleReason::Graph,
11231                    format!("target graph preparation plan is stale: {error}"),
11232                )
11233            })?;
11234        if let Some(append) = formula_plane.as_ref() {
11235            self.graph
11236                .formula_authority()
11237                .validate_prepared_formula_plane_append(
11238                    append,
11239                    self.graph.data_store(),
11240                    self.graph.sheet_reg(),
11241                )
11242                .map_err(|error| {
11243                    Self::preparation_stale(
11244                        formualizer_common::PreparationStaleReason::Authority,
11245                        format!("target FormulaPlane preparation is stale: {error}"),
11246                    )
11247                })?;
11248        }
11249        #[cfg(test)]
11250        self.target_preparation_fault(
11251            crate::engine::target_preparation::TargetPreparationFault::Reservation,
11252        )?;
11253        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
11254        self.formula_parse_diagnostics
11255            .try_reserve(pending_diagnostics.len())
11256            .map_err(|_| {
11257                resource(
11258                    formualizer_common::ResourceExhaustionReason::Admission,
11259                    pending_diagnostics.len() as u64,
11260                    pending_diagnostics.len() as u64,
11261                )
11262            })?;
11263        self.target_preparation_checkpoint(options.deadline, 0)?;
11264        #[cfg(test)]
11265        self.target_preparation_fault(
11266            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
11267        )?;
11268
11269        let commit_started = crate::instant::FzInstant::now();
11270        let committed = self
11271            .graph
11272            .apply_prevalidated_legacy_graph_plan(legacy_graph);
11273        let plane_report =
11274            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11275                formula_plane.map(|append| {
11276                    self.graph
11277                        .formula_authority_mut()
11278                        .apply_prevalidated_formula_plane_append(append)
11279                })
11280            } else {
11281                None
11282            };
11283        if let Some(report) = plane_report.as_ref() {
11284            self.graph.mark_formula_spans_dirty(
11285                report.spans.iter().copied(),
11286                WholeSpanDirtyReason::NewSpan,
11287            );
11288        }
11289        for formula in &prepared {
11290            let removed = self
11291                .staged_formulas
11292                .get_mut(&formula.sheet)
11293                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
11294            debug_assert!(removed.is_some());
11295            let index_removed = self.staged_formula_index.remove(
11296                &formula.sheet,
11297                formula.lease.row,
11298                formula.lease.col,
11299            );
11300            debug_assert!(index_removed);
11301        }
11302        for package in &prepared_packages {
11303            let removed = self
11304                .staged_formulas
11305                .get_mut(&package.sheet)
11306                .and_then(|staged| staged.deferred_package.take());
11307            debug_assert!(removed.is_some());
11308            self.staged_formula_index.set_package(&package.sheet, None);
11309        }
11310        let empty_sheets = self
11311            .staged_formulas
11312            .iter()
11313            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
11314            .collect::<Vec<_>>();
11315        for sheet in empty_sheets {
11316            self.staged_formulas.remove(&sheet);
11317        }
11318        if committed > 0 || plane_report.is_some() {
11319            self.mark_topology_edited();
11320        }
11321        self.formula_parse_diagnostics.extend(pending_diagnostics);
11322        if !prepared.is_empty() || !prepared_packages.is_empty() {
11323            let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
11324            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
11325                prepared_packages
11326                    .iter()
11327                    .map(|package| package.replay_records.len() as u64)
11328                    .sum::<u64>(),
11329            );
11330            ingest_delta.graph_formula_cells_materialized = committed as u64;
11331            ingest_delta.graph_vertices_created = new_vertices as u64;
11332            ingest_delta.graph_edges_created = new_edges as u64;
11333            for package in &prepared_packages {
11334                let source = &package.source_report;
11335                ingest_delta.source_formula_events = ingest_delta
11336                    .source_formula_events
11337                    .saturating_add(source.source_formula_events);
11338                ingest_delta.source_formula_records_spooled = ingest_delta
11339                    .source_formula_records_spooled
11340                    .saturating_add(source.source_formula_records_spooled);
11341                ingest_delta.source_spool_encoded_bytes = ingest_delta
11342                    .source_spool_encoded_bytes
11343                    .saturating_add(source.source_spool_encoded_bytes);
11344                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
11345                    .source_spool_peak_memory_bytes
11346                    .max(source.source_spool_peak_memory_bytes);
11347                ingest_delta.source_spool_spilled_bytes = ingest_delta
11348                    .source_spool_spilled_bytes
11349                    .saturating_add(source.source_spool_spilled_bytes);
11350                ingest_delta.source_spool_spill_files = ingest_delta
11351                    .source_spool_spill_files
11352                    .saturating_add(source.source_spool_spill_files);
11353                ingest_delta.source_spool_replays = ingest_delta
11354                    .source_spool_replays
11355                    .saturating_add(source.source_spool_replays)
11356                    .saturating_add(1);
11357                ingest_delta.source_families_seen = ingest_delta
11358                    .source_families_seen
11359                    .saturating_add(source.families_seen);
11360                ingest_delta.source_family_cells_seen = ingest_delta
11361                    .source_family_cells_seen
11362                    .saturating_add(source.family_cells_seen);
11363                ingest_delta.source_family_shadow_eligible = ingest_delta
11364                    .source_family_shadow_eligible
11365                    .saturating_add(source.source_clean_families);
11366                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
11367                    .source_family_shadow_eligible_cells
11368                    .saturating_add(source.source_clean_cells);
11369                ingest_delta.source_partitioned_families_seen = ingest_delta
11370                    .source_partitioned_families_seen
11371                    .saturating_add(source.source_fragmentable_families);
11372                ingest_delta.source_partition_holes = ingest_delta
11373                    .source_partition_holes
11374                    .saturating_add(source.source_hole_exclusions);
11375                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
11376                    .source_partition_ordinary_exceptions
11377                    .saturating_add(source.source_ordinary_exclusions);
11378                ingest_delta.source_partition_surviving_cells = ingest_delta
11379                    .source_partition_surviving_cells
11380                    .saturating_add(source.source_fragmentable_cells);
11381                for (reason, count) in &source.fallback_reasons {
11382                    let total = ingest_delta
11383                        .fallback_reasons
11384                        .entry(reason.clone())
11385                        .or_default();
11386                    *total = total.saturating_add(*count);
11387                }
11388                if self.config.formula_plane_mode == FormulaPlaneMode::Off {
11389                    ingest_delta.source_family_fallback = ingest_delta
11390                        .source_family_fallback
11391                        .saturating_add(source.families_seen);
11392                    ingest_delta.source_family_fallback_cells = ingest_delta
11393                        .source_family_fallback_cells
11394                        .saturating_add(source.family_cells_seen);
11395                } else {
11396                    ingest_delta.shadow_candidate_cells = ingest_delta
11397                        .shadow_candidate_cells
11398                        .saturating_add(source.family_cells_seen);
11399                    ingest_delta.shadow_accepted_span_cells = ingest_delta
11400                        .shadow_accepted_span_cells
11401                        .saturating_add(package.direct_cells);
11402                    ingest_delta.shadow_fallback_cells =
11403                        ingest_delta.shadow_fallback_cells.saturating_add(
11404                            source
11405                                .family_cells_seen
11406                                .saturating_sub(package.direct_cells),
11407                        );
11408                    ingest_delta.source_anchor_parses = ingest_delta
11409                        .source_anchor_parses
11410                        .saturating_add(package.anchor_parses);
11411                    ingest_delta.source_anchor_asts = ingest_delta
11412                        .source_anchor_asts
11413                        .saturating_add(package.anchor_asts);
11414                    ingest_delta.source_anchor_analyses = ingest_delta
11415                        .source_anchor_analyses
11416                        .saturating_add(package.anchor_analyses);
11417                    ingest_delta.source_compressed_families_prepared = ingest_delta
11418                        .source_compressed_families_prepared
11419                        .saturating_add(package.direct_complete_families);
11420                    ingest_delta.source_compressed_cells_prepared = ingest_delta
11421                        .source_compressed_cells_prepared
11422                        .saturating_add(package.direct_complete_cells);
11423                    ingest_delta.source_partitioned_families_prepared = ingest_delta
11424                        .source_partitioned_families_prepared
11425                        .saturating_add(package.direct_partition_families);
11426                    ingest_delta.source_partition_fragments_prepared = ingest_delta
11427                        .source_partition_fragments_prepared
11428                        .saturating_add(package.direct_fragments);
11429                    ingest_delta.source_partition_span_cells_prepared = ingest_delta
11430                        .source_partition_span_cells_prepared
11431                        .saturating_add(package.direct_partition_cells);
11432                    ingest_delta.source_partition_analyses_reused = ingest_delta
11433                        .source_partition_analyses_reused
11434                        .saturating_add(
11435                            package
11436                                .direct_fragments
11437                                .saturating_sub(package.direct_partition_families),
11438                        );
11439                    ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
11440                        .graph_formula_vertices_avoided_shadow
11441                        .saturating_add(package.direct_cells);
11442                    ingest_delta.ast_roots_avoided_shadow =
11443                        ingest_delta.ast_roots_avoided_shadow.saturating_add(
11444                            package
11445                                .direct_complete_cells
11446                                .saturating_sub(package.direct_complete_families)
11447                                .saturating_add(
11448                                    package
11449                                        .direct_partition_cells
11450                                        .saturating_sub(package.direct_fragments),
11451                                ),
11452                        );
11453                    ingest_delta.edge_rows_avoided_shadow = ingest_delta
11454                        .edge_rows_avoided_shadow
11455                        .saturating_add(package.direct_cells);
11456                    if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11457                    {
11458                        ingest_delta.source_family_promoted = ingest_delta
11459                            .source_family_promoted
11460                            .saturating_add(package.direct_families as u64);
11461                        ingest_delta.source_family_promoted_cells = ingest_delta
11462                            .source_family_promoted_cells
11463                            .saturating_add(package.direct_cells);
11464                        let descendants = package
11465                            .direct_complete_cells
11466                            .saturating_sub(package.direct_complete_families)
11467                            .saturating_add(
11468                                package
11469                                    .direct_partition_cells
11470                                    .saturating_sub(package.direct_partition_families),
11471                            );
11472                        ingest_delta.source_descendant_strings_avoided = ingest_delta
11473                            .source_descendant_strings_avoided
11474                            .saturating_add(descendants);
11475                        ingest_delta.source_descendant_events_avoided = ingest_delta
11476                            .source_descendant_events_avoided
11477                            .saturating_add(descendants);
11478                        ingest_delta.source_descendant_analyses_avoided = ingest_delta
11479                            .source_descendant_analyses_avoided
11480                            .saturating_add(descendants);
11481                    }
11482                    ingest_delta.source_family_fallback =
11483                        ingest_delta.source_family_fallback.saturating_add(
11484                            source
11485                                .families_seen
11486                                .saturating_sub(package.direct_families as u64),
11487                        );
11488                    ingest_delta.source_family_fallback_cells =
11489                        ingest_delta.source_family_fallback_cells.saturating_add(
11490                            source
11491                                .family_cells_seen
11492                                .saturating_sub(package.direct_cells),
11493                        );
11494                }
11495            }
11496            if let Some(report) = plane_report.as_ref() {
11497                ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
11498                ingest_delta.shadow_spans_created = report.spans.len() as u64;
11499            }
11500            self.record_formula_ingest_report(ingest_delta);
11501        }
11502        let commit_window = commit_started.elapsed();
11503        let retained_cells = self
11504            .staged_formula_index
11505            .all_leases()
11506            .into_iter()
11507            .filter_map(|(sheet, lease)| {
11508                formualizer_common::RangeAddress::new(
11509                    sheet, lease.row, lease.col, lease.row, lease.col,
11510                )
11511                .ok()
11512            })
11513            .collect::<Vec<_>>();
11514        let observed_scratch_bytes = self
11515            .active_resource_ledger
11516            .as_ref()
11517            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
11518            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11519        let committed_spans = plane_report
11520            .as_ref()
11521            .map_or(0, |report| report.spans.len() as u64);
11522        let selected_source_families = prepared_packages
11523            .iter()
11524            .map(|package| package.lease.family_count)
11525            .sum::<usize>();
11526        let selected_staged_cells = prepared
11527            .len()
11528            .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
11529        let actual_commit_work = (new_vertices as u64)
11530            .saturating_add(new_edges as u64)
11531            .saturating_add(committed as u64)
11532            .saturating_add(committed_spans)
11533            .saturating_add(prepared.len() as u64)
11534            .saturating_add(prepared_packages.len() as u64);
11535        let report = PreparedTargetGraphReport {
11536            request_id: request_id.unwrap_or_default(),
11537            requested_targets: targets.len(),
11538            normalized_regions: visited_regions.len(),
11539            normalized_target_list: normalized,
11540            selected_staged_cells,
11541            selected_source_families,
11542            retained_staged_cells: self.staged_formula_count(),
11543            selected_cells,
11544            retained_cells,
11545            widened_scope: scope,
11546            widening_reasons: reasons,
11547            revisions: assumptions,
11548            commit_window,
11549            estimated_scratch_bytes: scratch_bytes,
11550            observed_scratch_bytes,
11551            estimated_commit_work: (new_vertices as u64)
11552                .saturating_add(new_edges as u64)
11553                .saturating_add(prepared.len() as u64)
11554                .saturating_add(selected_package_records),
11555            actual_commit_work,
11556            outcome: PreparationOutcome::Prepared,
11557        };
11558        self.observe_target_preparation_report(&report);
11559        Ok(report)
11560    }
11561
11562    /// Build graph for all staged formulas.
11563    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11564        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
11565            engine.build_graph_all_unobserved()
11566        })
11567    }
11568
11569    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11570        let selected = self.staged_formula_count();
11571        let started = crate::instant::FzInstant::now();
11572        self.resource_checkpoint(selected as u64)?;
11573        let scratch_bytes = (selected as u64).saturating_mul(256);
11574        let result = self.with_request_scratch(scratch_bytes, |engine| {
11575            let index_snapshot = engine.staged_formula_index.clone();
11576            let collected = std::mem::take(&mut engine.staged_formulas)
11577                .into_iter()
11578                .collect();
11579            engine.staged_formula_index.clear_all();
11580            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
11581        });
11582        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11583        result
11584    }
11585
11586    /// Build graph for specific sheets (consuming only those staged entries).
11587    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
11588        &mut self,
11589        sheets: I,
11590    ) -> Result<(), formualizer_parse::ExcelError> {
11591        let mut sheets = sheets.into_iter();
11592        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
11593            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
11594            engine.with_request_scratch(name_scratch, |engine| {
11595                // Allocation failure follows the baseline process-fatal policy; it is not a
11596                // recoverable resource error or a new staged-preparation route.
11597                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
11598                engine.charge_bounded_work(names.len() as u64)?;
11599                engine.build_graph_for_sheet_names_unobserved(names)
11600            })
11601        })
11602    }
11603
11604    fn build_graph_for_sheet_names_unobserved(
11605        &mut self,
11606        sheets: Vec<String>,
11607    ) -> Result<(), formualizer_parse::ExcelError> {
11608        let started = crate::instant::FzInstant::now();
11609        let selected = sheets
11610            .iter()
11611            .filter_map(|sheet| self.staged_formulas.get(sheet))
11612            .map(StagedSheet::len)
11613            .sum::<usize>();
11614        self.resource_checkpoint(selected as u64)?;
11615        let scratch_bytes = (selected as u64).saturating_mul(256);
11616        self.reserve_request_scratch(scratch_bytes)?;
11617        let index_snapshot = self.staged_formula_index.clone();
11618        let mut collected = Vec::new();
11619        for sheet in sheets {
11620            if let Some(staged) = self.staged_formulas.remove(&sheet) {
11621                self.index_removed_staged_sheet(&sheet, &staged);
11622                collected.push((sheet, staged));
11623            }
11624        }
11625        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
11626        self.release_request_scratch(scratch_bytes);
11627        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11628        result
11629    }
11630
11631    fn build_graph_from_staged_batches(
11632        &mut self,
11633        collected: StagedFormulaBatches,
11634        share_parse_cache_across_sheets: bool,
11635        staged_index_snapshot: StagedFormulaIndex,
11636    ) -> Result<(), formualizer_parse::ExcelError> {
11637        if collected.is_empty() {
11638            return Ok(());
11639        }
11640        for (sheet, _) in &collected {
11641            let _ = self.add_sheet(sheet);
11642        }
11643
11644        let diagnostics_len = self.formula_parse_diagnostics.len();
11645        let mut collected = collected;
11646        let prepared = match self
11647            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
11648        {
11649            Ok(prepared) => prepared,
11650            Err(error) => {
11651                self.formula_parse_diagnostics.truncate(diagnostics_len);
11652                for (sheet, staged) in collected {
11653                    self.restore_staged_sheet(sheet, staged);
11654                }
11655                self.staged_formula_index = staged_index_snapshot;
11656                return Err(error);
11657            }
11658        };
11659        let (ordinary, compressed, direct) = prepared;
11660
11661        // Deferred packages are consumed exactly once once authority commit starts.
11662        // Pre-commit lock, replay, and parse failures restore them above for retry.
11663        if !ordinary.is_empty()
11664            && let Err(error) = self.ingest_formula_batches(ordinary)
11665        {
11666            self.formula_parse_diagnostics.truncate(diagnostics_len);
11667            for (sheet, staged) in collected {
11668                self.restore_staged_sheet(sheet, staged);
11669            }
11670            self.staged_formula_index = staged_index_snapshot;
11671            return Err(error);
11672        }
11673        if !compressed.is_empty() {
11674            let _ = self.ingest_compressed_formula_source_batches(compressed)?;
11675        }
11676        if !direct.is_empty() {
11677            let _ = self.finish_compressed_formula_sources(direct)?;
11678        }
11679        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
11680        Ok(())
11681    }
11682
11683    fn prepare_staged_formula_batches(
11684        &mut self,
11685        collected: &mut StagedFormulaBatches,
11686        share_parse_cache_across_sheets: bool,
11687    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
11688        let mut ordinary = Vec::new();
11689        let mut compressed = Vec::new();
11690        let mut direct = Vec::new();
11691        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
11692            rustc_hash::FxHashMap::default();
11693        cache.reserve(4096);
11694
11695        for (sheet, staged) in collected {
11696            if !share_parse_cache_across_sheets {
11697                cache.clear();
11698            }
11699            let mut entries: Vec<_> = staged
11700                .entries
11701                .iter()
11702                .cloned()
11703                .map(|(row, col, text)| (row, col, text, None))
11704                .collect();
11705            let mut deferred_source = None;
11706            let mut deferred_fallback = None;
11707            if let Some(package) = staged.deferred_package.as_mut() {
11708                if package.sheet_name != *sheet {
11709                    return Err(ExcelError::new(ExcelErrorKind::Value)
11710                        .with_message("deferred formula package sheet mismatch"));
11711                }
11712                let eligible: Vec<_> = package
11713                    .families
11714                    .iter()
11715                    .filter(|family| !package.invalidated.contains(&family.source_id))
11716                    .cloned()
11717                    .collect();
11718                let eligible_partitions: Vec<_> = package
11719                    .partitioned_families
11720                    .iter()
11721                    .filter(|family| !package.invalidated.contains(&family.source_id))
11722                    .cloned()
11723                    .collect();
11724                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11725                    let mut preparation = self.prepare_source_formula_proposals(
11726                        sheet,
11727                        &eligible,
11728                        &eligible_partitions,
11729                        &package.partitioned_families,
11730                        package.report.source_formula_records_spooled,
11731                        Arc::clone(&package.replay),
11732                        &package.suppressed,
11733                    )?;
11734                    let replay_records = std::mem::take(&mut preparation.eager_replay);
11735                    let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
11736                        let accepted: BTreeMap<_, _> = preparation
11737                            .fragmented
11738                            .iter()
11739                            .map(|fragment| (fragment.source.source_id, &fragment.source))
11740                            .collect();
11741                        replay_records.into_iter().partition(|record| {
11742                            record.partition_owner.is_some_and(|owner| {
11743                                accepted.get(&owner).is_some_and(|source| {
11744                                    Self::exact_replay_record_is_prepared_partition_legacy(
11745                                        source, record,
11746                                    )
11747                                })
11748                            })
11749                        })
11750                    };
11751                    preparation.eager_replay = fragment_legacy;
11752                    entries.extend(ordered_fallback.into_iter().map(|record| {
11753                        (
11754                            record.row,
11755                            record.col,
11756                            record.text,
11757                            Some((record.source_order, record.family, record.partition_owner)),
11758                        )
11759                    }));
11760                    deferred_source = Some((package.report.clone(), preparation));
11761                } else {
11762                    let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
11763                    for partition in &eligible_partitions {
11764                        replay_disposition
11765                            .register_partition(partition, false)
11766                            .map_err(|reason| {
11767                                ExcelError::new(ExcelErrorKind::Value).with_message(reason)
11768                            })?;
11769                    }
11770                    replay_disposition
11771                        .extend_suppressed_excel_coords(package.suppressed.iter().copied());
11772                    let replayed = package
11773                        .replay
11774                        .lock()
11775                        .map_err(|_| {
11776                            ExcelError::new(ExcelErrorKind::Value)
11777                                .with_message("deferred formula spool lock poisoned")
11778                        })?
11779                        .replay(&replay_disposition)
11780                        .map_err(|message| {
11781                            ExcelError::new(ExcelErrorKind::Value).with_message(message)
11782                        })?;
11783                    entries.extend(replayed.into_iter().map(|record| {
11784                        (
11785                            record.row,
11786                            record.col,
11787                            record.text,
11788                            Some((record.source_order, record.family, record.partition_owner)),
11789                        )
11790                    }));
11791                    let mut report = package.report.clone();
11792                    report.source_spool_replays = report.source_spool_replays.saturating_add(1);
11793                    deferred_fallback =
11794                        Some((report, package.families.clone(), eligible_partitions));
11795                }
11796            }
11797
11798            let mut formulas = Vec::new();
11799            let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
11800            for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
11801                let key = if txt.starts_with('=') {
11802                    txt
11803                } else {
11804                    format!("={txt}")
11805                };
11806                let ast_id = if let Some(cached) = cache.get(&key) {
11807                    *cached
11808                } else {
11809                    let parsed = match formualizer_parse::parser::parse(&key) {
11810                        Ok(parsed) => Some(parsed),
11811                        Err(error) => self.handle_formula_parse_error(
11812                            sheet,
11813                            row,
11814                            col,
11815                            &key,
11816                            error.to_string(),
11817                        )?,
11818                    };
11819                    let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
11820                    cache.insert(key.clone(), ast_id);
11821                    ast_id
11822                };
11823
11824                if let Some(ast_id) = ast_id {
11825                    let mut formula = FormulaIngestRecord::new(
11826                        row,
11827                        col,
11828                        ast_id,
11829                        Some(Arc::<str>::from(key.clone())),
11830                    );
11831                    if let Some((order, family, owner)) = source_proof {
11832                        formula = formula.with_source_proof(order, family, owner);
11833                    } else if deferred_source.is_some() {
11834                        formula = formula.with_source_proof(
11835                            crate::engine::SourceFormulaOrder::new(
11836                                staged_order_base.saturating_add(entry_index as u64),
11837                            ),
11838                            None,
11839                            None,
11840                        );
11841                    }
11842                    formulas.push(formula);
11843                }
11844            }
11845
11846            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
11847            if let Some((report, preparation)) = deferred_source {
11848                direct.push((batch, report, preparation));
11849            } else if let Some((report, families, partitions)) = deferred_fallback {
11850                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
11851                    batch.sheet_name.clone(),
11852                    report,
11853                    families,
11854                    partitions,
11855                );
11856                compressed.push((batch, source_batch));
11857            } else if !batch.is_empty() {
11858                ordinary.push(batch);
11859            }
11860        }
11861        Ok((ordinary, compressed, direct))
11862    }
11863
11864    /// Begin bulk Arrow ingest for base values (Phase A)
11865    pub fn begin_bulk_ingest_arrow(
11866        &mut self,
11867    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
11868        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
11869    }
11870
11871    /// Begin bulk updates to Arrow store (Phase C)
11872    pub fn begin_bulk_update_arrow(
11873        &mut self,
11874    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
11875        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
11876    }
11877
11878    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
11879        self.graph.sheet_id(sheet).ok_or(
11880            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11881                name: sheet.to_string(),
11882                reason: "Unknown sheet".to_string(),
11883            },
11884        )
11885    }
11886
11887    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
11888        if row_1based == 0 {
11889            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11890        }
11891        Ok(row_1based - 1)
11892    }
11893
11894    fn normalize_row_range_1based(
11895        start_row_1based: u32,
11896        end_row_1based: u32,
11897    ) -> Result<(u32, u32), crate::engine::EditorError> {
11898        if start_row_1based == 0 || end_row_1based == 0 {
11899            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11900        }
11901        if start_row_1based > end_row_1based {
11902            return Err(crate::engine::EditorError::TransactionFailed {
11903                reason: "Row range start is greater than end".to_string(),
11904            });
11905        }
11906        Ok((start_row_1based - 1, end_row_1based - 1))
11907    }
11908
11909    fn invalidate_row_visibility_mask_cache(&self) {
11910        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11911            cache.clear();
11912        }
11913    }
11914
11915    fn set_row_hidden_by_sheet_id(
11916        &mut self,
11917        sheet_id: SheetId,
11918        row0: u32,
11919        hidden: bool,
11920        source: RowVisibilitySource,
11921    ) -> bool {
11922        let changed = {
11923            let state = self.row_visibility.entry(sheet_id).or_default();
11924            state.set_row_hidden(row0, hidden, source)
11925        };
11926
11927        let remove_entry = self
11928            .row_visibility
11929            .get(&sheet_id)
11930            .map(|state| state.is_empty())
11931            .unwrap_or(false);
11932        if remove_entry {
11933            self.row_visibility.remove(&sheet_id);
11934        }
11935
11936        if changed {
11937            self.invalidate_row_visibility_mask_cache();
11938        }
11939
11940        changed
11941    }
11942
11943    fn set_rows_hidden_by_sheet_id(
11944        &mut self,
11945        sheet_id: SheetId,
11946        start_row0: u32,
11947        end_row0: u32,
11948        hidden: bool,
11949        source: RowVisibilitySource,
11950    ) -> bool {
11951        let changed = {
11952            let state = self.row_visibility.entry(sheet_id).or_default();
11953            state.set_rows_hidden(start_row0, end_row0, hidden, source)
11954        };
11955
11956        let remove_entry = self
11957            .row_visibility
11958            .get(&sheet_id)
11959            .map(|state| state.is_empty())
11960            .unwrap_or(false);
11961        if remove_entry {
11962            self.row_visibility.remove(&sheet_id);
11963        }
11964
11965        if changed {
11966            self.invalidate_row_visibility_mask_cache();
11967        }
11968
11969        changed
11970    }
11971
11972    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
11973        if count == 0 {
11974            return;
11975        }
11976        let mut changed = false;
11977        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
11978            changed = state.insert_rows(before0, count);
11979            state.is_empty()
11980        } else {
11981            false
11982        };
11983        if remove_entry {
11984            self.row_visibility.remove(&sheet_id);
11985        }
11986        if changed {
11987            self.invalidate_row_visibility_mask_cache();
11988        }
11989    }
11990
11991    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
11992        if count == 0 {
11993            return;
11994        }
11995        let mut changed = false;
11996        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
11997            changed = state.delete_rows(start0, count);
11998            state.is_empty()
11999        } else {
12000            false
12001        };
12002        if remove_entry {
12003            self.row_visibility.remove(&sheet_id);
12004        }
12005        if changed {
12006            self.invalidate_row_visibility_mask_cache();
12007        }
12008    }
12009
12010    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12011        if let crate::engine::ChangeEvent::SetRowVisibility {
12012            sheet_id,
12013            row0,
12014            source,
12015            old_hidden,
12016            ..
12017        } = event
12018        {
12019            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12020        }
12021    }
12022
12023    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12024        if let crate::engine::ChangeEvent::SetRowVisibility {
12025            sheet_id,
12026            row0,
12027            source,
12028            new_hidden,
12029            ..
12030        } = event
12031        {
12032            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12033        }
12034    }
12035
12036    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12037        for event in events.iter().rev() {
12038            self.apply_inverse_row_visibility_event(event);
12039        }
12040    }
12041
12042    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12043        for event in events {
12044            self.apply_forward_row_visibility_event(event);
12045        }
12046    }
12047
12048    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12049        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12050            sheet,
12051            row,
12052            col,
12053            old,
12054            ..
12055        } = event
12056        {
12057            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12058        }
12059    }
12060
12061    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12062        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12063            sheet,
12064            row,
12065            col,
12066            new,
12067            ..
12068        } = event
12069        {
12070            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12071        }
12072    }
12073
12074    /// Set a single cell's staged formula text to `target` (clearing it when
12075    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
12076    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12077        match target {
12078            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12079            None => {
12080                self.clear_staged_formula_text(sheet, row, col);
12081            }
12082        }
12083    }
12084
12085    pub fn set_row_hidden(
12086        &mut self,
12087        sheet: &str,
12088        row_1based: u32,
12089        hidden: bool,
12090        source: RowVisibilitySource,
12091    ) -> Result<(), crate::engine::EditorError> {
12092        self.observe_function_semantic_epoch()
12093            .map_err(crate::engine::EditorError::Excel)?;
12094        self.observe_function_semantic_epoch()
12095            .map_err(crate::engine::EditorError::Excel)?;
12096        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12097        let row0 = Self::normalize_row_1based(row_1based)?;
12098        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12099            self.record_formula_plane_structural_change(StructuralScope::Region(
12100                Region::whole_row(sheet_id, row0),
12101            ));
12102            self.mark_data_edited();
12103        }
12104        Ok(())
12105    }
12106
12107    pub fn set_rows_hidden(
12108        &mut self,
12109        sheet: &str,
12110        start_row_1based: u32,
12111        end_row_1based: u32,
12112        hidden: bool,
12113        source: RowVisibilitySource,
12114    ) -> Result<(), crate::engine::EditorError> {
12115        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12116        let (start_row0, end_row0) =
12117            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12118        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12119            if start_row0 == end_row0 {
12120                self.record_formula_plane_structural_change(StructuralScope::Region(
12121                    Region::whole_row(sheet_id, start_row0),
12122                ));
12123            } else {
12124                self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12125            }
12126            self.mark_data_edited();
12127        }
12128        Ok(())
12129    }
12130
12131    pub fn is_row_hidden(
12132        &self,
12133        sheet: &str,
12134        row_1based: u32,
12135        source: Option<RowVisibilitySource>,
12136    ) -> Option<bool> {
12137        let sheet_id = self.graph.sheet_id(sheet)?;
12138        let row0 = row_1based.checked_sub(1)?;
12139        Some(
12140            self.row_visibility
12141                .get(&sheet_id)
12142                .map(|state| state.is_row_hidden(row0, source))
12143                .unwrap_or(false),
12144        )
12145    }
12146
12147    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12148        let sheet_id = self.graph.sheet_id(sheet)?;
12149        Some(
12150            self.row_visibility
12151                .get(&sheet_id)
12152                .map(|state| state.version())
12153                .unwrap_or(0),
12154        )
12155    }
12156
12157    fn build_row_visibility_mask_for_view(
12158        &self,
12159        view: &RangeView<'_>,
12160        mode: VisibilityMaskMode,
12161    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12162        let sheet_rows = view.sheet().nrows as usize;
12163        if sheet_rows == 0 || view.start_row() >= sheet_rows {
12164            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12165        }
12166
12167        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12168        let start_row0 = view.start_row() as u32;
12169        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12170        let version = self
12171            .row_visibility
12172            .get(&sheet_id)
12173            .map(|state| state.version())
12174            .unwrap_or(0);
12175        let key = VisibilityMaskCacheKey {
12176            sheet_id,
12177            start_row0,
12178            end_row0,
12179            mode,
12180            version,
12181        };
12182
12183        if let Ok(cache) = self.row_visibility_mask_cache.read()
12184            && let Some(mask) = cache.get(&key)
12185        {
12186            #[cfg(test)]
12187            visibility_mask_test_hooks::inc_hit();
12188            return Some(mask.clone());
12189        }
12190
12191        #[cfg(test)]
12192        visibility_mask_test_hooks::inc_miss();
12193
12194        let state = self.row_visibility.get(&sheet_id);
12195        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12196        for row0 in start_row0..=end_row0 {
12197            let manual_hidden = state
12198                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12199                .unwrap_or(false);
12200            let filter_hidden = state
12201                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12202                .unwrap_or(false);
12203
12204            let include = match mode {
12205                VisibilityMaskMode::IncludeAll => true,
12206                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12207                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12208                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12209                    !(manual_hidden || filter_hidden)
12210                }
12211            };
12212            out.push(include);
12213        }
12214
12215        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12216        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12217            const MAX_CACHE_ENTRIES: usize = 4096;
12218            if cache.len() >= MAX_CACHE_ENTRIES {
12219                cache.clear();
12220                #[cfg(test)]
12221                visibility_mask_test_hooks::inc_eviction();
12222            }
12223            cache.insert(key, mask.clone());
12224        }
12225
12226        Some(mask)
12227    }
12228
12229    fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
12230        match error {
12231            crate::engine::EditorError::Excel(error) => error,
12232            other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
12233        }
12234    }
12235
12236    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
12237        let changes =
12238            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
12239        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
12240        let provider_revision = self.resolver.planning_semantic_revision();
12241        let provider_changed = provider_revision != self.function_provider_revision_seen;
12242        if !global_changed && !provider_changed {
12243            return Ok(false);
12244        }
12245
12246        let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
12247        let sheets: BTreeSet<_> = self
12248            .graph
12249            .formula_authority()
12250            .plane
12251            .spans
12252            .active_spans()
12253            .filter_map(|span| {
12254                let Some(template) = self
12255                    .graph
12256                    .formula_authority()
12257                    .plane
12258                    .templates
12259                    .get(span.template_id)
12260                else {
12261                    return (provider_changed || global_changed && !changes.complete)
12262                        .then_some(span.domain.sheet_id());
12263                };
12264                let Some(ast) = self
12265                    .graph
12266                    .data_store()
12267                    .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
12268                else {
12269                    return (provider_changed || global_changed && !changes.complete)
12270                        .then_some(span.domain.sheet_id());
12271                };
12272                let global_affected = global_changed
12273                    && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
12274                let provider_affected = provider_changed && Self::ast_contains_function(&ast);
12275                (global_affected || provider_affected).then_some(span.domain.sheet_id())
12276            })
12277            .collect();
12278        let invalidated = !sheets.is_empty();
12279        for sheet_id in sheets {
12280            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
12281                .map_err(Self::editor_error_to_excel)?;
12282        }
12283        if global_changed && !changed.is_empty() || provider_changed {
12284            self.cached_static_schedule = None;
12285        }
12286        self.function_semantic_epoch_seen = changes.epoch;
12287        self.function_provider_revision_seen = provider_revision;
12288        Ok(invalidated)
12289    }
12290
12291    pub(crate) fn ast_uses_changed_function(
12292        ast: &ASTNode,
12293        changed: &BTreeSet<(String, String)>,
12294    ) -> bool {
12295        match &ast.node_type {
12296            ASTNodeType::Function { name, args } => {
12297                let normalized = name.to_uppercase();
12298                let mut spellings = vec![(String::new(), normalized.clone())];
12299                let mut stripped = normalized.as_str();
12300                loop {
12301                    let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
12302                        .iter()
12303                        .find_map(|prefix| stripped.strip_prefix(prefix))
12304                    else {
12305                        break;
12306                    };
12307                    stripped = rest;
12308                    spellings.push((String::new(), stripped.to_string()));
12309                }
12310                let resolved = crate::function_registry::resolve("", name);
12311                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
12312                    || resolved.as_ref().is_some_and(|resolved| {
12313                        changed.contains(&(
12314                            resolved.namespace.clone(),
12315                            resolved.canonical_name.clone(),
12316                        ))
12317                    });
12318                directly_changed
12319                    || resolved.is_none()
12320                    || args
12321                        .iter()
12322                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
12323            }
12324            ASTNodeType::Call { callee, args } => {
12325                Self::ast_uses_changed_function(callee, changed)
12326                    || args
12327                        .iter()
12328                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
12329            }
12330            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
12331            ASTNodeType::BinaryOp { left, right, .. } => {
12332                Self::ast_uses_changed_function(left, changed)
12333                    || Self::ast_uses_changed_function(right, changed)
12334            }
12335            ASTNodeType::Array(rows) => rows
12336                .iter()
12337                .flatten()
12338                .any(|node| Self::ast_uses_changed_function(node, changed)),
12339            _ => false,
12340        }
12341    }
12342
12343    fn ast_contains_function(ast: &ASTNode) -> bool {
12344        match &ast.node_type {
12345            ASTNodeType::Function { .. } => true,
12346            ASTNodeType::Call { callee, args } => {
12347                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
12348            }
12349            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
12350            ASTNodeType::BinaryOp { left, right, .. } => {
12351                Self::ast_contains_function(left) || Self::ast_contains_function(right)
12352            }
12353            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
12354            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
12355        }
12356    }
12357
12358    fn demote_span_containing_cell_for_write(
12359        &mut self,
12360        sheet_id: SheetId,
12361        row0: u32,
12362        col0: u32,
12363    ) -> Result<(), crate::engine::EditorError> {
12364        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12365            return Ok(());
12366        }
12367        let placement = PlacementCoord::new(sheet_id, row0, col0);
12368        let inside_active_span = self
12369            .graph
12370            .formula_authority()
12371            .plane
12372            .spans
12373            .find_at(placement)
12374            .is_some();
12375        if inside_active_span {
12376            self.demote_spans_preserving_computed_overlays(
12377                sheet_id,
12378                Region::point(sheet_id, row0, col0),
12379            )?;
12380        }
12381        Ok(())
12382    }
12383
12384    fn demote_spans_preserving_computed_overlays(
12385        &mut self,
12386        _sheet_id: SheetId,
12387        affected_region: Region,
12388    ) -> Result<(), crate::engine::EditorError> {
12389        // Per-cell write inside a span (or whole-sheet demote via remove_sheet):
12390        // not a structural axis shift. Demote every span whose result or read
12391        // region intersects `affected_region`; leave disjoint spans untouched.
12392        self.demote_spans_for_structural_op_impl(None, affected_region, false)
12393    }
12394
12395    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
12396        Region::rows_from(sheet_id, start_row0)
12397    }
12398
12399    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
12400        Region::cols_from(sheet_id, start_col0)
12401    }
12402
12403    fn structural_dirty_region_for_domain(
12404        domain: &PlacementDomain,
12405        op: StructuralOp,
12406    ) -> Option<Region> {
12407        let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
12408            PlacementDomain::RowRun {
12409                sheet_id,
12410                row_start,
12411                row_end,
12412                col,
12413            } => (*sheet_id, *row_start, *row_end, *col, *col),
12414            PlacementDomain::ColRun {
12415                sheet_id,
12416                row,
12417                col_start,
12418                col_end,
12419            } => (*sheet_id, *row, *row, *col_start, *col_end),
12420            PlacementDomain::Rect {
12421                sheet_id,
12422                row_start,
12423                row_end,
12424                col_start,
12425                col_end,
12426            } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
12427        };
12428        match op {
12429            StructuralOp::InsertRows {
12430                sheet_id: edited,
12431                before,
12432                ..
12433            } if sheet_id == edited && row_end >= before => Some(Region::rect(
12434                sheet_id,
12435                row_start.max(before),
12436                row_end,
12437                col_start,
12438                col_end,
12439            )),
12440            StructuralOp::DeleteRows {
12441                sheet_id: edited,
12442                start,
12443                ..
12444            } if sheet_id == edited && row_end >= start => Some(Region::rect(
12445                sheet_id,
12446                row_start.max(start),
12447                row_end,
12448                col_start,
12449                col_end,
12450            )),
12451            StructuralOp::InsertColumns {
12452                sheet_id: edited,
12453                before,
12454                ..
12455            } if sheet_id == edited && col_end >= before => Some(Region::rect(
12456                sheet_id,
12457                row_start,
12458                row_end,
12459                col_start.max(before),
12460                col_end,
12461            )),
12462            StructuralOp::DeleteColumns {
12463                sheet_id: edited,
12464                start,
12465                ..
12466            } if sheet_id == edited && col_end >= start => Some(Region::rect(
12467                sheet_id,
12468                row_start,
12469                row_end,
12470                col_start.max(start),
12471                col_end,
12472            )),
12473            _ => None,
12474        }
12475    }
12476
12477    fn span_result_region_intersects_affected(
12478        span: &crate::formula_plane::runtime::FormulaSpan,
12479        affected_region: &Region,
12480    ) -> bool {
12481        Region::from_domain(span.result_region.domain()).intersects(affected_region)
12482    }
12483
12484    fn span_any_read_region_intersects_affected(
12485        plane: &FormulaPlane,
12486        span: &crate::formula_plane::runtime::FormulaSpan,
12487        affected_region: &Region,
12488    ) -> bool {
12489        span.read_summary_id
12490            .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
12491            .is_some_and(|summary| {
12492                summary
12493                    .dependencies
12494                    .iter()
12495                    .any(|dependency| dependency.read_region.intersects(affected_region))
12496            })
12497    }
12498
12499    fn insert_formula_plane_dirty_coords_for_span(
12500        &self,
12501        span_ref: FormulaSpanRef,
12502        dirty: ProducerDirtyDomain,
12503        out: &mut FxHashSet<(SheetId, u32, u32)>,
12504    ) -> Result<(), crate::engine::EditorError> {
12505        let authority = self.graph.formula_authority();
12506        let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
12507            ExcelError::new(ExcelErrorKind::NImpl)
12508                .with_message("FormulaPlane dirty transfer referenced a stale span")
12509        })?;
12510        match dirty {
12511            ProducerDirtyDomain::Whole => {
12512                out.extend(
12513                    span.domain
12514                        .iter()
12515                        .map(|coord| (coord.sheet_id, coord.row, coord.col)),
12516                );
12517            }
12518            ProducerDirtyDomain::Cells(cells) => {
12519                out.extend(cells.into_iter().filter_map(|key| {
12520                    let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
12521                    span.domain
12522                        .contains(coord)
12523                        .then_some((coord.sheet_id, coord.row, coord.col))
12524                }));
12525            }
12526            ProducerDirtyDomain::Regions(regions) => {
12527                out.extend(span.domain.iter().filter_map(|coord| {
12528                    let key = crate::formula_plane::region_index::RegionKey::from(coord);
12529                    regions
12530                        .iter()
12531                        .any(|region| region.contains_key(key))
12532                        .then_some((coord.sheet_id, coord.row, coord.col))
12533                }));
12534            }
12535        }
12536        Ok(())
12537    }
12538
12539    fn compute_current_formula_plane_dirty_result_coords(
12540        &self,
12541    ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
12542        use crate::formula_plane::producer::compute_dirty_closure;
12543
12544        let authority = self.graph.formula_authority();
12545        let span_refs = authority.active_span_refs();
12546        let span_refs_by_id = span_refs
12547            .iter()
12548            .copied()
12549            .map(|span_ref| (span_ref.id, span_ref))
12550            .collect::<BTreeMap<_, _>>();
12551        let mut dirty_coords = FxHashSet::default();
12552
12553        for span_ref in self.graph.pending_formula_dirty_whole_spans() {
12554            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12555                self.insert_formula_plane_dirty_coords_for_span(
12556                    span_ref,
12557                    ProducerDirtyDomain::Whole,
12558                    &mut dirty_coords,
12559                )?;
12560            }
12561        }
12562        for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
12563            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12564                self.insert_formula_plane_dirty_coords_for_span(
12565                    span_ref,
12566                    ProducerDirtyDomain::Regions(vec![region]),
12567                    &mut dirty_coords,
12568                )?;
12569            }
12570        }
12571
12572        let pending_changed_regions = self
12573            .graph
12574            .pending_formula_dirty_regions()
12575            .collect::<Vec<_>>();
12576        if pending_changed_regions.is_empty() {
12577            return Ok(dirty_coords);
12578        }
12579
12580        let closure = compute_dirty_closure(
12581            &authority.consumer_reads,
12582            pending_changed_regions,
12583            |producer| authority.producer_results.producer_result_region(producer),
12584        );
12585        if closure.incomplete {
12586            for span_ref in span_refs {
12587                self.insert_formula_plane_dirty_coords_for_span(
12588                    span_ref,
12589                    ProducerDirtyDomain::Whole,
12590                    &mut dirty_coords,
12591                )?;
12592            }
12593            return Ok(dirty_coords);
12594        }
12595        for work in closure.work {
12596            let FormulaProducerId::Span(span_id) = work.producer else {
12597                continue;
12598            };
12599            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12600                continue;
12601            };
12602            self.insert_formula_plane_dirty_coords_for_span(
12603                span_ref,
12604                work.dirty,
12605                &mut dirty_coords,
12606            )?;
12607        }
12608        for fallback in closure.fallbacks {
12609            let FormulaProducerId::Span(span_id) = fallback.consumer else {
12610                continue;
12611            };
12612            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12613                continue;
12614            };
12615            self.insert_formula_plane_dirty_coords_for_span(
12616                span_ref,
12617                ProducerDirtyDomain::Whole,
12618                &mut dirty_coords,
12619            )?;
12620        }
12621
12622        Ok(dirty_coords)
12623    }
12624
12625    /// Demote active FormulaPlane spans affected by a structural edit on `sheet_id`.
12626    ///
12627    /// This is the conservative Option-A correctness path for structural edits: rather than
12628    /// attempting to transform FormulaPlane span domains/templates/indexes, materialize each span
12629    /// placement as an ordinary legacy graph formula at its current coordinate, remove the span,
12630    /// and let the existing VertexEditor structural machinery shift/delete those vertices and
12631    /// adjust their ASTs.  Spans whose formula domain is on `sheet_id` are affected directly; spans
12632    /// on other sheets are also affected when one of their retained read regions targets
12633    /// `sheet_id`, because those read-region coordinates become stale after row/column shifts.
12634    fn demote_spans_for_structural_op(
12635        &mut self,
12636        op: StructuralOp,
12637        affected_region: Region,
12638    ) -> Result<(), crate::engine::EditorError> {
12639        if op.count() == 0 {
12640            return Ok(());
12641        }
12642        // #171: after an axis edit, retained read-summary relocation cannot be
12643        // used as a proof of disconnection. Fail closed for this engine.
12644        self.legacy_island_structural_summaries_trusted = false;
12645        self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
12646    }
12647
12648    fn indexed_structural_candidate_span_refs(
12649        &self,
12650        affected_region: Region,
12651    ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
12652        use crate::formula_plane::region_index::BoundedRegionQueryResult;
12653
12654        let authority = self.graph.formula_authority();
12655        if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
12656            return Err(ExcelError::new(ExcelErrorKind::NImpl)
12657                .with_message("FormulaPlane structural candidate indexes are stale")
12658                .into());
12659        }
12660        let indexed_region = match affected_region.axis_ranges() {
12661            (
12662                crate::formula_plane::region_index::AxisRange::From(start),
12663                crate::formula_plane::region_index::AxisRange::All,
12664            ) => Region {
12665                sheet_id: affected_region.sheet_id(),
12666                rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12667                cols: crate::formula_plane::region_index::AxisRange::All,
12668            },
12669            (
12670                crate::formula_plane::region_index::AxisRange::All,
12671                crate::formula_plane::region_index::AxisRange::From(start),
12672            ) => Region {
12673                sheet_id: affected_region.sheet_id(),
12674                rows: crate::formula_plane::region_index::AxisRange::All,
12675                cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12676            },
12677            _ => affected_region,
12678        };
12679        let max_candidates = self.config.max_formula_plane_cache_candidates;
12680        let result_query = match authority
12681            .producer_results
12682            .query_bounded(indexed_region, max_candidates)
12683        {
12684            BoundedRegionQueryResult::Complete(result) => result,
12685            BoundedRegionQueryResult::Incomplete {
12686                observed_candidates,
12687            } => {
12688                return Err(ExcelError::new(ExcelErrorKind::NImpl)
12689                    .with_message(format!(
12690                        "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
12691                    ))
12692                    .into());
12693            }
12694        };
12695        let remaining = max_candidates.saturating_sub(result_query.matches.len());
12696        let read_query = match authority
12697            .consumer_reads
12698            .query_changed_region_bounded(indexed_region, remaining)
12699        {
12700            BoundedRegionQueryResult::Complete(result) => result,
12701            BoundedRegionQueryResult::Incomplete { .. } => {
12702                return Err(ExcelError::new(ExcelErrorKind::NImpl)
12703                    .with_message("FormulaPlane structural read candidate limit exceeded")
12704                    .into());
12705            }
12706        };
12707
12708        let mut ids = BTreeSet::new();
12709        for matched in result_query.matches {
12710            if let FormulaProducerId::Span(id) = matched.value.producer {
12711                ids.insert(id);
12712            }
12713        }
12714        for matched in read_query.matches {
12715            if let FormulaProducerId::Span(id) = matched.value.consumer {
12716                ids.insert(id);
12717            }
12718        }
12719
12720        ids.into_iter()
12721            .map(|id| {
12722                authority.plane.spans.current_ref(id).ok_or_else(|| {
12723                    crate::engine::EditorError::Excel(
12724                        ExcelError::new(ExcelErrorKind::NImpl)
12725                            .with_message("FormulaPlane structural index referenced a stale span"),
12726                    )
12727                })
12728            })
12729            .collect()
12730    }
12731
12732    fn demote_spans_for_structural_op_impl(
12733        &mut self,
12734        op: Option<StructuralOp>,
12735        affected_region: Region,
12736        clear_computed_overlays: bool,
12737    ) -> Result<(), crate::engine::EditorError> {
12738        struct SpanPlan {
12739            span_ref: FormulaSpanRef,
12740            sheet_id: SheetId,
12741            ast: ASTNode,
12742            origin_row: u32,
12743            origin_col: u32,
12744            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12745            placements: Vec<(u32, u32)>,
12746        }
12747
12748        fn substitute_literal_slots_for_template_placement(
12749            ast: &ASTNode,
12750            binding: &[LiteralValue],
12751        ) -> ASTNode {
12752            fn clone_with_slots(
12753                ast: &ASTNode,
12754                binding: &[LiteralValue],
12755                next: &mut usize,
12756                in_array: bool,
12757            ) -> ASTNode {
12758                let node_type = match &ast.node_type {
12759                    ASTNodeType::Literal(_) if !in_array => {
12760                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
12761                        *next = next.saturating_add(1);
12762                        ASTNodeType::Literal(value)
12763                    }
12764                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
12765                    ASTNodeType::Omitted => ASTNodeType::Omitted,
12766                    ASTNodeType::Reference {
12767                        original,
12768                        reference,
12769                    } => ASTNodeType::Reference {
12770                        original: original.clone(),
12771                        reference: reference.clone(),
12772                    },
12773                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
12774                        op: op.clone(),
12775                        expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
12776                    },
12777                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
12778                        op: op.clone(),
12779                        left: Box::new(clone_with_slots(left, binding, next, in_array)),
12780                        right: Box::new(clone_with_slots(right, binding, next, in_array)),
12781                    },
12782                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
12783                        name: name.clone(),
12784                        args: args
12785                            .iter()
12786                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
12787                            .collect(),
12788                    },
12789                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
12790                        callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
12791                        args: args
12792                            .iter()
12793                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
12794                            .collect(),
12795                    },
12796                    ASTNodeType::Array(rows) => ASTNodeType::Array(
12797                        rows.iter()
12798                            .map(|row| {
12799                                row.iter()
12800                                    .map(|cell| clone_with_slots(cell, binding, next, true))
12801                                    .collect()
12802                            })
12803                            .collect(),
12804                    ),
12805                };
12806                ASTNode::new(node_type, ast.source_token.clone())
12807            }
12808            let mut next = 0usize;
12809            clone_with_slots(ast, binding, &mut next, false)
12810        }
12811
12812        let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
12813        self.formula_plane_structural_span_candidates = self
12814            .formula_plane_structural_span_candidates
12815            .saturating_add(span_refs.len() as u64);
12816        if span_refs.is_empty() {
12817            return Ok(());
12818        }
12819        let dirty_span_coords = if clear_computed_overlays {
12820            FxHashSet::default()
12821        } else {
12822            self.compute_current_formula_plane_dirty_result_coords()?
12823        };
12824
12825        /// A template AST rewrite accompanying a span shift: the op displaced
12826        /// the target of an absolute read, which origin relocation cannot
12827        /// repoint (issue #168). The adjusted AST is re-interned as a fresh
12828        /// arena AST + template record at the apply site; keys are re-derived
12829        /// from the rewritten AST so the new record can never collide with
12830        /// the (immutable, possibly shared) pre-rewrite template.
12831        struct SpanTemplateRewrite {
12832            adjusted_ast: ASTNode,
12833            exact_canonical_key: Arc<str>,
12834            parameterized_canonical_key: Arc<str>,
12835            formula_text: Option<Arc<str>>,
12836            /// Canonical form of the rewritten AST at the new origin, used to
12837            /// rebuild the binding set's template-derived slot data (the
12838            /// literal slot map is keyed by arena node ids of the template
12839            /// AST; value-ref slot patterns carry canonical reference
12840            /// coordinates — both go stale across a rewrite).
12841            canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
12842            value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
12843        }
12844
12845        struct ShiftPlan {
12846            span_ref: FormulaSpanRef,
12847            template_id: crate::formula_plane::ids::FormulaTemplateId,
12848            new_origin_row: u32,
12849            new_origin_col: u32,
12850            new_domain: crate::formula_plane::runtime::PlacementDomain,
12851            new_read_summary: Option<SpanReadSummary>,
12852            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12853            force_binding_residual_axes: bool,
12854            rewrite: Option<SpanTemplateRewrite>,
12855            dirty_region: Option<Region>,
12856        }
12857
12858        /// StructuralOp and ShiftOperation are field-for-field identical
12859        /// (both 0-based); the adjuster type is the legacy shared one.
12860        fn shift_operation_for(
12861            op: StructuralOp,
12862        ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
12863            use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
12864            match op {
12865                StructuralOp::InsertRows {
12866                    sheet_id,
12867                    before,
12868                    count,
12869                } => ShiftOperation::InsertRows {
12870                    sheet_id,
12871                    before,
12872                    count,
12873                },
12874                StructuralOp::DeleteRows {
12875                    sheet_id,
12876                    start,
12877                    count,
12878                } => ShiftOperation::DeleteRows {
12879                    sheet_id,
12880                    start,
12881                    count,
12882                },
12883                StructuralOp::InsertColumns {
12884                    sheet_id,
12885                    before,
12886                    count,
12887                } => ShiftOperation::InsertColumns {
12888                    sheet_id,
12889                    before,
12890                    count,
12891                },
12892                StructuralOp::DeleteColumns {
12893                    sheet_id,
12894                    start,
12895                    count,
12896                } => ShiftOperation::DeleteColumns {
12897                    sheet_id,
12898                    start,
12899                    count,
12900                },
12901            }
12902        }
12903
12904        /// Split a straddled span into an unshifted upper half (which keeps the
12905        /// span id via `replace_span_geometry`) and a shifted lower half (a
12906        /// fresh span). Domains and origins here are POST-insert for the lower
12907        /// half, mirroring `ShiftPlan`.
12908        struct SplitPlan {
12909            span_ref: FormulaSpanRef,
12910            sheet_id: SheetId,
12911            template_id: crate::formula_plane::ids::FormulaTemplateId,
12912            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12913            is_constant_result: bool,
12914            upper_domain: crate::formula_plane::runtime::PlacementDomain,
12915            upper_read_summary: Option<SpanReadSummary>,
12916            lower_new_origin_row: u32,
12917            lower_new_origin_col: u32,
12918            lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
12919            lower_new_read_summary: Option<SpanReadSummary>,
12920            lower_force_binding_residual_axes: bool,
12921            lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
12922            upper_dirty_region: Option<Region>,
12923            lower_dirty_region: Option<Region>,
12924        }
12925
12926        fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
12927            u32::try_from(i64::from(value).checked_add(delta)?).ok()
12928        }
12929
12930        /// Structural transform of a read summary for a NO-REWRITE shift.
12931        /// Regions move with the op; projection rules keep their absolute
12932        /// bounds (the AST is untouched) but must shift their RELATIVE
12933        /// bounds by `-origin_delta`: rules store placement-relative
12934        /// offsets, which equal `authored_coordinate - template_origin`,
12935        /// and a moved origin over an unchanged AST changes that difference
12936        /// (otherwise incremental dirty projection silently goes stale —
12937        /// issue #168 family). Rewrite plans do NOT use this transform;
12938        /// their summaries are re-derived from the rewritten canonical
12939        /// template instead.
12940        fn shifted_read_summary(
12941            read_summary: &SpanReadSummary,
12942            new_result_region: Region,
12943            op: StructuralOp,
12944            row_delta: i64,
12945            col_delta: i64,
12946            origin_row_delta: i64,
12947            origin_col_delta: i64,
12948        ) -> Option<SpanReadSummary> {
12949            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
12950            for dependency in &read_summary.dependencies {
12951                let read_region = match op.classify_region(dependency.read_region) {
12952                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
12953                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
12954                        dependency.read_region
12955                    }
12956                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
12957                        ..
12958                    } => dependency
12959                        .read_region
12960                        .project_through_axis_shift(row_delta, col_delta)?,
12961                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles
12962                    | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
12963                        return None;
12964                    }
12965                };
12966                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
12967                    read_region,
12968                    projection: dependency
12969                        .projection
12970                        .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
12971                });
12972            }
12973            Some(SpanReadSummary {
12974                result_region: new_result_region,
12975                dependencies,
12976            })
12977        }
12978
12979        fn compact_axis_through_delete(
12980            min: u32,
12981            max: u32,
12982            start: u32,
12983            count: u32,
12984        ) -> Option<(u32, u32)> {
12985            let end = start.saturating_add(count);
12986            if max < start || min >= end {
12987                return Some((min.saturating_sub(count), max.saturating_sub(count)));
12988            }
12989            let keeps_left = min < start;
12990            let keeps_right = max >= end;
12991            match (keeps_left, keeps_right) {
12992                (false, false) => None,
12993                (true, false) => Some((min, start.checked_sub(1)?)),
12994                (false, true) => Some((start, max.checked_sub(count)?)),
12995                (true, true) => Some((min, max.checked_sub(count)?)),
12996            }
12997        }
12998
12999        fn compact_domain_through_delete(
13000            domain: &PlacementDomain,
13001            op: StructuralOp,
13002        ) -> Option<PlacementDomain> {
13003            match (domain, op) {
13004                (
13005                    PlacementDomain::RowRun {
13006                        sheet_id,
13007                        row_start,
13008                        row_end,
13009                        col,
13010                    },
13011                    StructuralOp::DeleteRows { start, count, .. },
13012                ) => {
13013                    let (row_start, row_end) =
13014                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13015                    Some(PlacementDomain::row_run(
13016                        *sheet_id, row_start, row_end, *col,
13017                    ))
13018                }
13019                (
13020                    PlacementDomain::Rect {
13021                        sheet_id,
13022                        row_start,
13023                        row_end,
13024                        col_start,
13025                        col_end,
13026                    },
13027                    StructuralOp::DeleteRows { start, count, .. },
13028                ) => {
13029                    let (row_start, row_end) =
13030                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13031                    Some(PlacementDomain::rect(
13032                        *sheet_id, row_start, row_end, *col_start, *col_end,
13033                    ))
13034                }
13035                (
13036                    PlacementDomain::ColRun {
13037                        sheet_id,
13038                        row,
13039                        col_start,
13040                        col_end,
13041                    },
13042                    StructuralOp::DeleteColumns { start, count, .. },
13043                ) => {
13044                    let (col_start, col_end) =
13045                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13046                    Some(PlacementDomain::col_run(
13047                        *sheet_id, *row, col_start, col_end,
13048                    ))
13049                }
13050                (
13051                    PlacementDomain::Rect {
13052                        sheet_id,
13053                        row_start,
13054                        row_end,
13055                        col_start,
13056                        col_end,
13057                    },
13058                    StructuralOp::DeleteColumns { start, count, .. },
13059                ) => {
13060                    let (col_start, col_end) =
13061                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13062                    Some(PlacementDomain::rect(
13063                        *sheet_id, *row_start, *row_end, col_start, col_end,
13064                    ))
13065                }
13066                _ => None,
13067            }
13068        }
13069
13070        fn compact_axis_range_through_delete(
13071            axis: crate::formula_plane::region_index::AxisRange,
13072            start: u32,
13073            count: u32,
13074        ) -> Option<crate::formula_plane::region_index::AxisRange> {
13075            use crate::formula_plane::region_index::AxisRange;
13076            match axis {
13077                AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13078                    .map(|(point, _)| AxisRange::Point(point)),
13079                AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13080                    .map(|(min, max)| AxisRange::Span(min, max)),
13081                AxisRange::All => Some(AxisRange::All),
13082                AxisRange::From(_) | AxisRange::To(_) => None,
13083            }
13084        }
13085
13086        fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13087            let (rows, cols) = region.axis_ranges();
13088            match op {
13089                StructuralOp::DeleteRows {
13090                    sheet_id,
13091                    start,
13092                    count,
13093                } if region.sheet_id() == sheet_id => Some(Region {
13094                    sheet_id,
13095                    rows: compact_axis_range_through_delete(rows, start, count)?,
13096                    cols,
13097                }),
13098                StructuralOp::DeleteColumns {
13099                    sheet_id,
13100                    start,
13101                    count,
13102                } if region.sheet_id() == sheet_id => Some(Region {
13103                    sheet_id,
13104                    rows,
13105                    cols: compact_axis_range_through_delete(cols, start, count)?,
13106                }),
13107                _ => Some(region),
13108            }
13109        }
13110
13111        fn compact_read_summary_through_delete(
13112            read_summary: &SpanReadSummary,
13113            new_result_region: Region,
13114            op: StructuralOp,
13115        ) -> Option<SpanReadSummary> {
13116            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13117            for dependency in &read_summary.dependencies {
13118                let read_region = match op.classify_region(dependency.read_region) {
13119                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13120                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13121                        dependency.read_region
13122                    }
13123                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13124                        ..
13125                    } => {
13126                        let (row_delta, col_delta) = op.axis_shift_delta();
13127                        dependency
13128                            .read_region
13129                            .project_through_axis_shift(row_delta, col_delta)?
13130                    }
13131                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13132                        compact_region_through_delete(dependency.read_region, op)?
13133                    }
13134                    crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13135                        return None;
13136                    }
13137                };
13138                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13139                    read_region,
13140                    projection: dependency.projection,
13141                });
13142            }
13143            Some(SpanReadSummary {
13144                result_region: new_result_region,
13145                dependencies,
13146            })
13147        }
13148
13149        /// Re-derive a span read summary for a sub-domain of the original
13150        /// result region by replaying each retained projection rule against
13151        /// the half's result region — the same derivation ingest performs in
13152        /// `SpanReadSummary::from_formula_summary`, minus the dependency
13153        /// re-analysis. Returns None (caller demotes) when any rule cannot
13154        /// produce explicit read regions.
13155        fn rederive_read_summary_for_result(
13156            read_summary: &SpanReadSummary,
13157            result_region: Region,
13158        ) -> Option<SpanReadSummary> {
13159            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13160            for dependency in &read_summary.dependencies {
13161                let sheet_id = dependency.read_region.sheet_id();
13162                let read_regions = dependency
13163                    .projection
13164                    .read_regions_for_result(sheet_id, result_region)
13165                    .ok()?;
13166                for read_region in read_regions {
13167                    let dependency = crate::formula_plane::producer::SpanReadDependency {
13168                        read_region,
13169                        projection: dependency.projection,
13170                    };
13171                    if !dependencies.contains(&dependency) {
13172                        dependencies.push(dependency);
13173                    }
13174                }
13175            }
13176            Some(SpanReadSummary {
13177                result_region,
13178                dependencies,
13179            })
13180        }
13181
13182        /// Plan a conservative span split for a mid-domain insert. Returns
13183        /// None whenever the split is not provably clean — missing template,
13184        /// masked span, non-trivial literal bindings, an unsplittable domain,
13185        /// a half that does not re-classify to NoOp (upper) / Shift (lower),
13186        /// or an overlay punch-out that straddles the boundary — in which case
13187        /// the caller falls back to demoting the whole span.
13188        fn plan_span_split(
13189            authority: &crate::formula_plane::authority::FormulaAuthority,
13190            span: &crate::formula_plane::runtime::FormulaSpan,
13191            span_ref: FormulaSpanRef,
13192            read_summary: Option<&SpanReadSummary>,
13193            op: StructuralOp,
13194        ) -> Option<SplitPlan> {
13195            use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13196
13197            if span.intrinsic_mask_id.is_some() {
13198                // v1 does not slice intrinsic masks.
13199                return None;
13200            }
13201            let binding_set = span
13202                .binding_set_id
13203                .and_then(|id| authority.plane.binding_sets.get(id));
13204            if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13205                // Dictionary/affine per-placement literal bindings are keyed by
13206                // domain ordinal; splitting re-bases the lower half's ordinals.
13207                // Only trivially-uniform bindings are safe to carry across.
13208                return None;
13209            }
13210
13211            let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13212            let upper_result_region = Region::from_domain(&upper_domain);
13213            let lower_result_region = Region::from_domain(&lower_domain);
13214            let (upper_read_summary, lower_read_summary) = match read_summary {
13215                Some(summary) => (
13216                    Some(rederive_read_summary_for_result(
13217                        summary,
13218                        upper_result_region,
13219                    )?),
13220                    Some(rederive_read_summary_for_result(
13221                        summary,
13222                        lower_result_region,
13223                    )?),
13224                ),
13225                None => (None, None),
13226            };
13227
13228            // Re-classify each half in the pre-insert frame. The upper half
13229            // must be a strict no-op (unshifted result, unshifted reads); the
13230            // lower half must be a clean whole-span shift.
13231            let upper_span = crate::formula_plane::runtime::FormulaSpan {
13232                domain: upper_domain.clone(),
13233                result_region: ResultRegion::scalar_cells(upper_domain.clone()),
13234                ..span.clone()
13235            };
13236            if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
13237                != SpanShiftPlan::NoOp
13238            {
13239                return None;
13240            }
13241            let lower_span = crate::formula_plane::runtime::FormulaSpan {
13242                domain: lower_domain.clone(),
13243                result_region: ResultRegion::scalar_cells(lower_domain.clone()),
13244                ..span.clone()
13245            };
13246            let SpanShiftPlan::Shift {
13247                row_delta,
13248                col_delta,
13249                origin_row_delta,
13250                origin_col_delta,
13251                rewrite_absolute_reads,
13252            } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
13253            else {
13254                return None;
13255            };
13256            if rewrite_absolute_reads {
13257                // A displaced absolute read in the lower half implies the
13258                // upper half reads it too and cannot be a NoOp, so this is
13259                // unreachable when the upper check above passed; keep the
13260                // guard so a future classifier change cannot silently skip
13261                // the template rewrite.
13262                return None;
13263            }
13264
13265            // Span relocation is the ownership boundary for demotion and
13266            // formula lookup. Canonically shared templates may have a different
13267            // authoring origin, so never derive split provenance from them.
13268            let lower_new_origin_row =
13269                checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
13270            let lower_new_origin_col =
13271                checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
13272            let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
13273            let lower_new_result_region = Region::from_domain(&lower_new_domain);
13274            let lower_new_read_summary = match lower_read_summary.as_ref() {
13275                Some(summary) => Some(shifted_read_summary(
13276                    summary,
13277                    lower_new_result_region,
13278                    op,
13279                    row_delta,
13280                    col_delta,
13281                    origin_row_delta,
13282                    origin_col_delta,
13283                )?),
13284                None => None,
13285            };
13286
13287            // Overlay punch-outs sourced from this span: entries entirely
13288            // before the boundary stay with the (id-retaining) upper half;
13289            // entries at/after the boundary transfer to the new lower span.
13290            // A straddling entry means the split is not clean.
13291            let mut lower_overlay_refs = Vec::new();
13292            for overlay_ref in authority
13293                .plane
13294                .formula_overlay
13295                .refs_for_source_span(span_ref)
13296            {
13297                let entry = authority.plane.formula_overlay.get(overlay_ref)?;
13298                match op.classify_region(Region::from_domain(&entry.domain)) {
13299                    AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
13300                    AxisShiftCase::EntirelyAboveShift { .. } => {
13301                        lower_overlay_refs.push(overlay_ref);
13302                    }
13303                    AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
13304                        return None;
13305                    }
13306                }
13307            }
13308
13309            let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
13310                !binding_set.value_ref_slots.is_empty()
13311                    && (origin_row_delta != 0 || origin_col_delta != 0)
13312            });
13313
13314            Some(SplitPlan {
13315                span_ref,
13316                sheet_id: span.sheet_id,
13317                template_id: span.template_id,
13318                binding_set_id: span.binding_set_id,
13319                is_constant_result: span.is_constant_result,
13320                upper_domain,
13321                upper_read_summary,
13322                lower_new_origin_row,
13323                lower_new_origin_col,
13324                lower_new_domain,
13325                lower_new_read_summary,
13326                lower_force_binding_residual_axes,
13327                lower_overlay_refs,
13328                upper_dirty_region: None,
13329                lower_dirty_region: None,
13330            })
13331        }
13332
13333        /// Carry a 1-based template origin coordinate through a delete of
13334        /// `count` rows/columns starting at 0-based `start`. Returns None when
13335        /// the delete removes the origin coordinate itself: the template's
13336        /// relative reference deltas are anchored there, so the caller must
13337        /// demote rather than guess a new anchor.
13338        fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
13339            let origin0 = origin.checked_sub(1)?;
13340            let end = start.saturating_add(count);
13341            if origin0 < start {
13342                Some(origin)
13343            } else if origin0 >= end {
13344                Some(origin - count)
13345            } else {
13346                None
13347            }
13348        }
13349
13350        /// Carry the template origin through a delete instead of rebasing it
13351        /// to the compacted domain's start. Rebasing is only correct when the
13352        /// origin coincides with the domain start; split/shifted spans keep
13353        /// their original origin (Shift{origin_delta: 0} convention), and a
13354        /// head-overlapping delete moves the domain start away from the
13355        /// origin, so rebasing silently re-anchors every relative read.
13356        fn origin_through_delete(
13357            origin_row: u32,
13358            origin_col: u32,
13359            op: StructuralOp,
13360        ) -> Option<(u32, u32)> {
13361            match op {
13362                StructuralOp::DeleteRows { start, count, .. } => Some((
13363                    origin_coord_through_delete(origin_row, start, count)?,
13364                    origin_col,
13365                )),
13366                StructuralOp::DeleteColumns { start, count, .. } => Some((
13367                    origin_row,
13368                    origin_coord_through_delete(origin_col, start, count)?,
13369                )),
13370                StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
13371            }
13372        }
13373
13374        let mut shift_plans = Vec::new();
13375        let mut split_plans = Vec::new();
13376        let mut remove_refs = Vec::new();
13377        let mut demote_refs = Vec::new();
13378        for span_ref in span_refs {
13379            let authority = self.graph.formula_authority();
13380            let Some(span) = authority.plane.spans.get(span_ref) else {
13381                continue;
13382            };
13383            let read_summary = span
13384                .read_summary_id
13385                .and_then(|id| authority.plane.span_read_summaries.get(id));
13386            let Some(op) = op else {
13387                // Non-structural demote path (per-cell write into span, or
13388                // remove_sheet's whole-sheet sweep). Only demote spans whose
13389                // result or read region intersects affected_region; leave
13390                // disjoint spans untouched.
13391                let result_region_affected =
13392                    Self::span_result_region_intersects_affected(span, &affected_region);
13393                let read_region_affected = Self::span_any_read_region_intersects_affected(
13394                    &authority.plane,
13395                    span,
13396                    &affected_region,
13397                );
13398                if result_region_affected || read_region_affected {
13399                    demote_refs.push(span_ref);
13400                }
13401                continue;
13402            };
13403            match classify_span_for_op(span, read_summary, op) {
13404                SpanShiftPlan::NoOp => {}
13405                SpanShiftPlan::Remove => {
13406                    remove_refs.push(span_ref);
13407                }
13408                SpanShiftPlan::Demote {
13409                    reason:
13410                        crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
13411                } => {
13412                    let binding_compaction_safe = span
13413                        .binding_set_id
13414                        .and_then(|id| authority.plane.binding_sets.get(id))
13415                        .is_none_or(|binding_set| binding_set.is_single_literal_binding());
13416                    // Compaction projects read regions through the delete but
13417                    // keeps the template AST: an absolute read whose target
13418                    // the delete displaces would keep its stale coordinate
13419                    // (issue #168), so demote to the per-cell path instead.
13420                    let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
13421                        !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
13422                            summary, op,
13423                        )
13424                    });
13425                    if binding_compaction_safe
13426                        && absolute_reads_compaction_safe
13427                        && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
13428                    {
13429                        let new_result_region = Region::from_domain(&new_domain);
13430                        let new_read_summary = if let Some(summary) = read_summary {
13431                            compact_read_summary_through_delete(summary, new_result_region, op)
13432                        } else {
13433                            None
13434                        };
13435                        let Some(template) = authority.plane.templates.get(span.template_id) else {
13436                            return Err(ExcelError::new(ExcelErrorKind::Ref)
13437                                .with_message(
13438                                    "FormulaPlane delete compaction found a span with a missing template",
13439                                )
13440                                .into());
13441                        };
13442                        let carried_origin =
13443                            origin_through_delete(template.origin_row, template.origin_col, op);
13444                        if (read_summary.is_none() || new_read_summary.is_some())
13445                            && let Some((new_origin_row, new_origin_col)) = carried_origin
13446                        {
13447                            let force_binding_residual_axes = span
13448                                .binding_set_id
13449                                .and_then(|id| authority.plane.binding_sets.get(id))
13450                                .is_some_and(|binding_set| {
13451                                    !binding_set.value_ref_slots.is_empty()
13452                                        && (new_origin_row != template.origin_row
13453                                            || new_origin_col != template.origin_col)
13454                                });
13455                            let dirty_region =
13456                                Self::structural_dirty_region_for_domain(&new_domain, op);
13457                            shift_plans.push(ShiftPlan {
13458                                span_ref,
13459                                template_id: span.template_id,
13460                                new_origin_row,
13461                                new_origin_col,
13462                                new_domain,
13463                                new_read_summary,
13464                                binding_set_id: span.binding_set_id,
13465                                force_binding_residual_axes,
13466                                rewrite: None,
13467                                dirty_region,
13468                            });
13469                        } else {
13470                            demote_refs.push(span_ref);
13471                        }
13472                    } else {
13473                        demote_refs.push(span_ref);
13474                    }
13475                }
13476                SpanShiftPlan::Demote { .. } => {
13477                    demote_refs.push(span_ref);
13478                }
13479                SpanShiftPlan::Split => {
13480                    match plan_span_split(authority, span, span_ref, read_summary, op) {
13481                        Some(mut plan) => {
13482                            plan.upper_dirty_region =
13483                                Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
13484                            plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
13485                                &plan.lower_new_domain,
13486                                op,
13487                            );
13488                            split_plans.push(plan);
13489                        }
13490                        // Not provably clean: demote the whole span (the
13491                        // pre-split conservative path).
13492                        None => demote_refs.push(span_ref),
13493                    }
13494                }
13495                SpanShiftPlan::Shift {
13496                    row_delta,
13497                    col_delta,
13498                    origin_row_delta,
13499                    origin_col_delta,
13500                    rewrite_absolute_reads,
13501                } => {
13502                    let Some(template) = authority.plane.templates.get(span.template_id) else {
13503                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13504                            .with_message("FormulaPlane shift found a span with a missing template")
13505                            .into());
13506                    };
13507                    let Some(new_origin_row) =
13508                        checked_shift_u32(template.origin_row, origin_row_delta)
13509                    else {
13510                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13511                            .with_message("FormulaPlane shift overflowed template origin row")
13512                            .into());
13513                    };
13514                    let Some(new_origin_col) =
13515                        checked_shift_u32(template.origin_col, origin_col_delta)
13516                    else {
13517                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13518                            .with_message("FormulaPlane shift overflowed template origin column")
13519                            .into());
13520                    };
13521                    let Some(new_domain) =
13522                        span.domain.project_through_axis_shift(row_delta, col_delta)
13523                    else {
13524                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13525                            .with_message("FormulaPlane shift overflowed span domain")
13526                            .into());
13527                    };
13528                    let new_result_region = Region::from_domain(&new_domain);
13529                    let force_binding_residual_axes = span
13530                        .binding_set_id
13531                        .and_then(|id| authority.plane.binding_sets.get(id))
13532                        .is_some_and(|binding_set| {
13533                            !binding_set.value_ref_slots.is_empty()
13534                                && (origin_row_delta != 0 || origin_col_delta != 0)
13535                        });
13536                    let rewrite_and_summary = if rewrite_absolute_reads {
13537                        // Reify the (immutable, possibly shared) template
13538                        // AST and repoint its displaced references through
13539                        // the shared adjuster — the same relocation the
13540                        // per-cell path applies, so span-ON stays equal to
13541                        // span-OFF (issue #168). Canonical keys are
13542                        // re-derived from the rewritten AST at the new
13543                        // origin, so the fresh record cannot collide with
13544                        // the pre-rewrite template.
13545                        //
13546                        // FRAME GUARD: the adjuster relocates by AUTHORED
13547                        // coordinate, but a template's authoring frame may
13548                        // diverge from its domain (split lower halves keep
13549                        // the original anchor; pinned-origin shifts move the
13550                        // domain without the origin). The rewrite is only
13551                        // equivalent to per-cell adjustment when every
13552                        // relative read's authored coordinate classifies
13553                        // against the op boundary the same way its read
13554                        // region does — otherwise the rewrite shears
13555                        // relative reads by the op count. Demote instead;
13556                        // see rewrite_frame_is_sound for the full invariant.
13557                        let frame_sound = read_summary.is_some_and(|summary| {
13558                            crate::formula_plane::structural_shift::rewrite_frame_is_sound(
13559                                summary,
13560                                template.origin_row,
13561                                template.origin_col,
13562                                op,
13563                            )
13564                        });
13565                        if !frame_sound {
13566                            demote_refs.push(span_ref);
13567                            continue;
13568                        }
13569                        let Some(ast) = self
13570                            .graph
13571                            .data_store()
13572                            .retrieve_ast(template.ast_id, self.graph.sheet_reg())
13573                        else {
13574                            demote_refs.push(span_ref);
13575                            continue;
13576                        };
13577                        let shift_op = shift_operation_for(op);
13578                        let adjuster =
13579                            crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
13580                        let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
13581                            span.sheet_id,
13582                            self.graph.sheet_reg(),
13583                        );
13584                        let Some(adjusted_ast) =
13585                            adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
13586                        else {
13587                            // The classifier saw a displaced absolute read,
13588                            // so an unchanged AST is a contract violation;
13589                            // demote conservatively rather than shift with a
13590                            // stale template.
13591                            demote_refs.push(span_ref);
13592                            continue;
13593                        };
13594                        let canonical = crate::formula_plane::template_canonical::canonicalize_template(
13595                            &adjusted_ast,
13596                            new_origin_row,
13597                            new_origin_col,
13598                        );
13599                        if !canonical.labels.is_authority_supported() {
13600                            demote_refs.push(span_ref);
13601                            continue;
13602                        }
13603                        // The rewrite touches only references; the literal
13604                        // slot layout must survive unchanged or the binding
13605                        // set's per-placement literal bindings no longer
13606                        // line up. Demote defensively if it differs.
13607                        let literal_slots_compatible = span
13608                            .binding_set_id
13609                            .and_then(|id| authority.plane.binding_sets.get(id))
13610                            .is_none_or(|binding_set| {
13611                                binding_set.literal_slots.as_ref()
13612                                    == canonical.literal_slot_descriptors.as_ref()
13613                            });
13614                        if !literal_slots_compatible {
13615                            demote_refs.push(span_ref);
13616                            continue;
13617                        }
13618                        // Value-ref slot patterns are re-derived from the
13619                        // rewritten canonical template so memoized reads
13620                        // resolve the repointed coordinates (issue #168).
13621                        // Placement-relative memo KEYS are additionally
13622                        // protected by force_binding_residual_axes below
13623                        // (the rewrite always moves the origin, so the
13624                        // existing origin-delta condition fires).
13625                        let value_ref_slots = Arc::from(
13626                            crate::formula_plane::placement::value_ref_slot_descriptors(
13627                                &canonical.expr,
13628                            )
13629                            .into_boxed_slice(),
13630                        );
13631                        // The read summary is RE-DERIVED from the rewritten
13632                        // canonical template rather than structurally
13633                        // transformed: a structural transform would keep the
13634                        // old projection rules, whose absolute indices (and
13635                        // relative offsets, via the moved origin) no longer
13636                        // match the rewritten AST — incremental writes to
13637                        // the repointed targets would then silently fail to
13638                        // dirty the span. Re-derivation rebuilds rules in
13639                        // the (rewritten AST, new origin) frame, keeping
13640                        // `result_region == domain region` by construction.
13641                        let dependency_summary =
13642                            crate::formula_plane::dependency_summary::summarize_canonical_template(
13643                                &canonical,
13644                            );
13645                        let rewritten_result_region =
13646                            ResultRegion::scalar_cells(new_domain.clone());
13647                        let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
13648                            span.sheet_id,
13649                            &rewritten_result_region,
13650                            &dependency_summary,
13651                            self.graph.sheet_reg(),
13652                        ) else {
13653                            demote_refs.push(span_ref);
13654                            continue;
13655                        };
13656                        let formula_text = Some(Arc::<str>::from(
13657                            formualizer_parse::pretty::canonical_formula(&adjusted_ast),
13658                        ));
13659                        Some((
13660                            SpanTemplateRewrite {
13661                                exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
13662                                parameterized_canonical_key: Arc::<str>::from(
13663                                    canonical.parameterized_key.payload(),
13664                                ),
13665                                formula_text,
13666                                canonical_expr: canonical.expr,
13667                                value_ref_slots,
13668                                adjusted_ast,
13669                            },
13670                            rederived_summary,
13671                        ))
13672                    } else {
13673                        None
13674                    };
13675                    let (rewrite, new_read_summary) = match rewrite_and_summary {
13676                        Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
13677                        None => {
13678                            let new_read_summary = if let Some(summary) = read_summary {
13679                                Some(
13680                                    shifted_read_summary(
13681                                        summary,
13682                                        new_result_region,
13683                                        op,
13684                                        row_delta,
13685                                        col_delta,
13686                                        origin_row_delta,
13687                                        origin_col_delta,
13688                                    )
13689                                    .ok_or_else(|| {
13690                                        ExcelError::new(ExcelErrorKind::Ref).with_message(
13691                                            "FormulaPlane shift could not project read summary",
13692                                        )
13693                                    })?,
13694                                )
13695                            } else {
13696                                None
13697                            };
13698                            (None, new_read_summary)
13699                        }
13700                    };
13701                    let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
13702                    shift_plans.push(ShiftPlan {
13703                        span_ref,
13704                        template_id: span.template_id,
13705                        new_origin_row,
13706                        new_origin_col,
13707                        new_domain,
13708                        new_read_summary,
13709                        binding_set_id: span.binding_set_id,
13710                        force_binding_residual_axes,
13711                        rewrite,
13712                        dirty_region,
13713                    });
13714                }
13715            }
13716        }
13717        let span_geometry_changed =
13718            !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
13719        let mut span_dirty_deltas = Vec::new();
13720        if span_geometry_changed {
13721            // Rewritten template ASTs must be interned into the graph's AST
13722            // arena before the authority is borrowed mutably; the literal
13723            // slot map is keyed by the freshly interned arena node ids.
13724            let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
13725            for plan in shift_plans {
13726                let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
13727                    let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
13728                    let template_slot_map =
13729                        crate::formula_plane::placement::build_template_slot_map(
13730                            ast_id,
13731                            self.graph.data_store(),
13732                            &rewrite.canonical_expr,
13733                        );
13734                    (ast_id, template_slot_map)
13735                });
13736                prepared_shift_plans.push((plan, rewrite_prepared));
13737            }
13738            let authority = self.graph.formula_authority_mut();
13739            for span_ref in remove_refs {
13740                authority.plane.remove_overlays_for_source_span(span_ref);
13741                authority.plane.remove_span(span_ref);
13742            }
13743            for (plan, rewrite_prepared) in prepared_shift_plans {
13744                let dirty_region = plan.dirty_region;
13745                let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
13746                    let Some((ast_id, template_slot_map)) = rewrite_prepared else {
13747                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13748                            .with_message("FormulaPlane shift lost a rewritten template AST")
13749                            .into());
13750                    };
13751                    let Some(template_id) = authority.plane.intern_rewritten_template(
13752                        plan.template_id,
13753                        ast_id,
13754                        rewrite.exact_canonical_key,
13755                        rewrite.parameterized_canonical_key,
13756                        rewrite.formula_text,
13757                        plan.new_origin_row,
13758                        plan.new_origin_col,
13759                    ) else {
13760                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13761                            .with_message("FormulaPlane shift could not intern rewritten template")
13762                            .into());
13763                    };
13764                    (
13765                        template_id,
13766                        Some((template_slot_map, rewrite.value_ref_slots)),
13767                    )
13768                } else {
13769                    let Some(template_id) = authority.plane.intern_shifted_template_origin(
13770                        plan.template_id,
13771                        plan.new_origin_row,
13772                        plan.new_origin_col,
13773                    ) else {
13774                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13775                            .with_message("FormulaPlane shift could not clone template origin")
13776                            .into());
13777                    };
13778                    (template_id, None)
13779                };
13780                if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
13781                    && let Some(binding_set_id) = plan.binding_set_id
13782                {
13783                    authority.plane.set_binding_template_slots(
13784                        binding_set_id,
13785                        template_slot_map,
13786                        value_ref_slots,
13787                    );
13788                }
13789                if let Some(binding_set_id) = plan.binding_set_id {
13790                    let Some(template) = authority.plane.templates.get(template_id) else {
13791                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13792                            .with_message("FormulaPlane shift could not find shifted template")
13793                            .into());
13794                    };
13795                    let (ast_id, origin_row, origin_col) =
13796                        (template.ast_id, template.origin_row, template.origin_col);
13797                    authority.plane.set_binding_template_anchor(
13798                        binding_set_id,
13799                        ast_id,
13800                        origin_row,
13801                        origin_col,
13802                    );
13803                }
13804                let read_summary_id = plan
13805                    .new_read_summary
13806                    .map(|summary| authority.plane.insert_span_read_summary(summary));
13807                let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
13808                if !authority.plane.replace_span_geometry(
13809                    plan.span_ref,
13810                    template_id,
13811                    plan.new_domain,
13812                    result_region,
13813                    read_summary_id,
13814                ) {
13815                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13816                        .with_message("FormulaPlane shift could not update span geometry")
13817                        .into());
13818                }
13819                if let Some(region) = dirty_region {
13820                    span_dirty_deltas.push((plan.span_ref, region));
13821                }
13822                if plan.force_binding_residual_axes
13823                    && let Some(binding_set_id) = plan.binding_set_id
13824                {
13825                    // Value-ref memoization keys are placement-relative. When a
13826                    // structural op moves the formula origin while keeping some
13827                    // precedents fixed (e.g. insert a column before a formula
13828                    // family that reads column A), those keys no longer name
13829                    // the same producer cells. Keep correctness by forcing
13830                    // placement offsets into the key so memoization falls back
13831                    // to per-placement work rather than broadcasting stale
13832                    // representative values.
13833                    authority.plane.force_binding_residual_axes(binding_set_id);
13834                }
13835            }
13836            for plan in split_plans {
13837                let upper_dirty_region = plan.upper_dirty_region;
13838                let lower_dirty_region = plan.lower_dirty_region;
13839                let Some(split_relocation) = authority
13840                    .plane
13841                    .spans
13842                    .get(plan.span_ref)
13843                    .map(|span| span.ast_relocation)
13844                else {
13845                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13846                        .with_message("FormulaPlane split lost its anchor AST state")
13847                        .into());
13848                };
13849                // Lower half: a fresh span at the POST-insert coordinates,
13850                // mirroring the Shift arm's template/binding handling.
13851                let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
13852                    plan.template_id,
13853                    plan.lower_new_origin_row,
13854                    plan.lower_new_origin_col,
13855                ) else {
13856                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13857                        .with_message("FormulaPlane split could not clone template origin")
13858                        .into());
13859                };
13860                let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
13861                    let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
13862                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13863                            .with_message(
13864                                "FormulaPlane split found a span with a missing binding set",
13865                            )
13866                            .into());
13867                    };
13868                    // Safe to clone wholesale: `plan_span_split` only accepts
13869                    // single-literal-binding sets, whose per-placement lookups
13870                    // are ordinal-independent.
13871                    let clone = source.clone();
13872                    let new_binding_set_id = authority.plane.insert_binding_set(clone);
13873                    let Some(template) = authority.plane.templates.get(lower_template_id) else {
13874                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13875                            .with_message("FormulaPlane split could not find shifted template")
13876                            .into());
13877                    };
13878                    let (ast_id, origin_row, origin_col) =
13879                        (template.ast_id, template.origin_row, template.origin_col);
13880                    authority.plane.set_binding_template_anchor(
13881                        new_binding_set_id,
13882                        ast_id,
13883                        origin_row,
13884                        origin_col,
13885                    );
13886                    Some(new_binding_set_id)
13887                } else {
13888                    None
13889                };
13890                let lower_read_summary_id = plan
13891                    .lower_new_read_summary
13892                    .map(|summary| authority.plane.insert_span_read_summary(summary));
13893                let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
13894                let lower_ref = authority.plane.insert_span_with_ast_relocation(
13895                    crate::formula_plane::runtime::NewFormulaSpan {
13896                        sheet_id: plan.sheet_id,
13897                        template_id: lower_template_id,
13898                        domain: plan.lower_new_domain,
13899                        result_region: lower_result_region,
13900                        intrinsic_mask_id: None,
13901                        read_summary_id: lower_read_summary_id,
13902                        binding_set_id: lower_binding_set_id,
13903                        is_constant_result: plan.is_constant_result,
13904                    },
13905                    crate::formula_plane::runtime::SpanAstRelocation {
13906                        ast_id: split_relocation.ast_id,
13907                        anchor_row: plan.lower_new_origin_row,
13908                        anchor_col: plan.lower_new_origin_col,
13909                    },
13910                );
13911                if let Some(region) = lower_dirty_region {
13912                    span_dirty_deltas.push((lower_ref, region));
13913                }
13914                if let Some(binding_set_id) = lower_binding_set_id {
13915                    authority
13916                        .plane
13917                        .set_binding_span_ref(binding_set_id, lower_ref);
13918                    if plan.lower_force_binding_residual_axes {
13919                        // Same contract as the Shift arm: a moved origin with
13920                        // stationary value-ref precedents invalidates
13921                        // placement-relative memoization keys.
13922                        authority.plane.force_binding_residual_axes(binding_set_id);
13923                    }
13924                }
13925                // The lower half inherits the parent's defined-name
13926                // invalidation registrations; the upper half keeps them by
13927                // retaining the span id.
13928                let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
13929                if !name_keys.is_empty() {
13930                    authority
13931                        .plane
13932                        .register_span_name_dependents(lower_ref, &name_keys);
13933                }
13934                for overlay_ref in plan.lower_overlay_refs {
13935                    if !authority
13936                        .plane
13937                        .set_overlay_source_span(overlay_ref, Some(lower_ref))
13938                    {
13939                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13940                            .with_message(
13941                                "FormulaPlane split could not re-point overlay provenance",
13942                            )
13943                            .into());
13944                    }
13945                }
13946                // Upper half: keep the span id, truncate the domain, and swap
13947                // in a summary re-derived for the truncated result region.
13948                let upper_read_summary_id = plan
13949                    .upper_read_summary
13950                    .map(|summary| authority.plane.insert_span_read_summary(summary));
13951                let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
13952                if !authority.plane.replace_span_geometry_with_ast_relocation(
13953                    plan.span_ref,
13954                    plan.template_id,
13955                    split_relocation,
13956                    plan.upper_domain,
13957                    upper_result_region,
13958                    upper_read_summary_id,
13959                ) {
13960                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13961                        .with_message("FormulaPlane split could not update span geometry")
13962                        .into());
13963                }
13964                if let Some(region) = upper_dirty_region {
13965                    span_dirty_deltas.push((plan.span_ref, region));
13966                }
13967            }
13968            authority.rebuild_indexes();
13969        }
13970        for (span_ref, region) in span_dirty_deltas {
13971            self.graph.mark_formula_span_region_dirty(span_ref, region);
13972        }
13973
13974        let mut span_plans = Vec::new();
13975        for span_ref in demote_refs {
13976            let authority = self.graph.formula_authority();
13977            let Some(span) = authority.plane.spans.get(span_ref) else {
13978                continue;
13979            };
13980            let relocation = span.ast_relocation;
13981            let ast = self
13982                .graph
13983                .data_store()
13984                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
13985                .ok_or_else(|| {
13986                    ExcelError::new(ExcelErrorKind::Ref).with_message(
13987                        "FormulaPlane demotion could not retrieve the span anchor AST",
13988                    )
13989                })?;
13990            let placements = span
13991                .domain
13992                .iter()
13993                .map(|placement| (placement.row + 1, placement.col + 1))
13994                .collect();
13995            span_plans.push(SpanPlan {
13996                span_ref,
13997                sheet_id: span.sheet_id,
13998                ast,
13999                origin_row: relocation.anchor_row,
14000                origin_col: relocation.anchor_col,
14001                binding_set_id: span.binding_set_id,
14002                placements,
14003            });
14004        }
14005        if span_plans.is_empty() {
14006            return Ok(());
14007        }
14008
14009        let mut relocated = Vec::new();
14010        let mut placement_cells = Vec::new();
14011        for plan in &span_plans {
14012            for &(row, col) in &plan.placements {
14013                let row_delta = i64::from(row) - i64::from(plan.origin_row);
14014                let col_delta = i64::from(col) - i64::from(plan.origin_col);
14015                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14016                    let authority = self.graph.formula_authority();
14017                    if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14018                        if binding_set.is_single_literal_binding() {
14019                            plan.ast.clone()
14020                        } else {
14021                            let placement = crate::formula_plane::runtime::PlacementCoord::new(
14022                                plan.sheet_id,
14023                                row.saturating_sub(1),
14024                                col.saturating_sub(1),
14025                            );
14026                            let binding =
14027                                authority.plane.spans.get(plan.span_ref).and_then(|span| {
14028                                    binding_set
14029                                        .literal_bindings_for_placement(&span.domain, placement)
14030                                });
14031                            if let Some(binding) = binding {
14032                                substitute_literal_slots_for_template_placement(
14033                                    &plan.ast,
14034                                    binding.as_ref(),
14035                                )
14036                            } else {
14037                                plan.ast.clone()
14038                            }
14039                        }
14040                    } else {
14041                        plan.ast.clone()
14042                    }
14043                } else {
14044                    plan.ast.clone()
14045                };
14046                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14047                relocated.push((plan.sheet_id, row, col, ast));
14048                placement_cells.push((plan.sheet_id, row, col));
14049            }
14050        }
14051        let planned_by_sheet = {
14052            let mut pipeline = self.ingest_pipeline();
14053            let mut planned_by_sheet: BTreeMap<
14054                SheetId,
14055                Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14056            > = BTreeMap::new();
14057            for (formula_sheet_id, row, col, ast) in relocated {
14058                let placement =
14059                    CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14060                let ingested =
14061                    pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14062                planned_by_sheet.entry(formula_sheet_id).or_default().push((
14063                    row,
14064                    col,
14065                    ingested.ast_id,
14066                    ingested.dep_plan,
14067                ));
14068            }
14069            planned_by_sheet
14070        };
14071        {
14072            let authority = self.graph.formula_authority_mut();
14073            for plan in &span_plans {
14074                authority
14075                    .plane
14076                    .remove_overlays_for_source_span(plan.span_ref);
14077                authority.plane.remove_span(plan.span_ref);
14078            }
14079            authority.rebuild_indexes();
14080        }
14081        if clear_computed_overlays {
14082            // Only clear placement cells whose coordinate intersects the affected
14083            // structural region. The structural-op contract preserves cells
14084            // BEFORE the structural boundary; the legacy `clear_computed_overlay_after_*`
14085            // call honors that. Demoting a span whose footprint straddles the
14086            // boundary still must not clear cells before the boundary, even
14087            // though the span as a whole is demoted.
14088            self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14089        }
14090        for (formula_sheet_id, planned) in planned_by_sheet {
14091            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14092            self.graph
14093                .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14094        }
14095        if !clear_computed_overlays {
14096            for (formula_sheet_id, row, col) in &placement_cells {
14097                let row0 = row.saturating_sub(1);
14098                let col0 = col.saturating_sub(1);
14099                if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14100                    continue;
14101                }
14102                let cell =
14103                    CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14104                if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14105                    self.graph.set_dirty(vertex_id, false);
14106                }
14107            }
14108        }
14109        Ok(())
14110    }
14111
14112    /// Prepare an exact-ref, additions-only refinement of FormulaPlane spans
14113    /// into legacy graph formulas. All fallible relocation, dependency analysis,
14114    /// identifier reservation, and conflict checks happen before commit.
14115    pub(crate) fn prepare_formula_span_demotion(
14116        &mut self,
14117        span_refs: &[FormulaSpanRef],
14118    ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14119        #[cfg(test)]
14120        let fault = self
14121            .formula_span_demotion_fault_for_test
14122            .take()
14123            .unwrap_or_default();
14124        #[cfg(not(test))]
14125        let fault = FormulaSpanDemotionFault::None;
14126
14127        if fault == FormulaSpanDemotionFault::AstPreparation {
14128            return Err(FormulaSpanDemotionError::Injected(fault));
14129        }
14130
14131        fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14132            fn visit(
14133                ast: &ASTNode,
14134                binding: &[LiteralValue],
14135                next: &mut usize,
14136                in_array: bool,
14137            ) -> ASTNode {
14138                let node_type = match &ast.node_type {
14139                    ASTNodeType::Literal(_) if !in_array => {
14140                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14141                        *next = next.saturating_add(1);
14142                        ASTNodeType::Literal(value)
14143                    }
14144                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14145                    ASTNodeType::Omitted => ASTNodeType::Omitted,
14146                    ASTNodeType::Reference {
14147                        original,
14148                        reference,
14149                    } => ASTNodeType::Reference {
14150                        original: original.clone(),
14151                        reference: reference.clone(),
14152                    },
14153                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14154                        op: op.clone(),
14155                        expr: Box::new(visit(expr, binding, next, in_array)),
14156                    },
14157                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14158                        op: op.clone(),
14159                        left: Box::new(visit(left, binding, next, in_array)),
14160                        right: Box::new(visit(right, binding, next, in_array)),
14161                    },
14162                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
14163                        name: name.clone(),
14164                        args: args
14165                            .iter()
14166                            .map(|arg| visit(arg, binding, next, in_array))
14167                            .collect(),
14168                    },
14169                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14170                        callee: Box::new(visit(callee, binding, next, in_array)),
14171                        args: args
14172                            .iter()
14173                            .map(|arg| visit(arg, binding, next, in_array))
14174                            .collect(),
14175                    },
14176                    ASTNodeType::Array(rows) => ASTNodeType::Array(
14177                        rows.iter()
14178                            .map(|row| {
14179                                row.iter()
14180                                    .map(|cell| visit(cell, binding, next, true))
14181                                    .collect()
14182                            })
14183                            .collect(),
14184                    ),
14185                };
14186                ASTNode::new(node_type, ast.source_token.clone())
14187            }
14188            let mut next = 0;
14189            visit(ast, binding, &mut next, false)
14190        }
14191
14192        struct SpanMaterialization {
14193            span_ref: FormulaSpanRef,
14194            sheet_id: SheetId,
14195            ast: ASTNode,
14196            origin_row: u32,
14197            origin_col: u32,
14198            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
14199            domain: PlacementDomain,
14200        }
14201
14202        let request_id = self
14203            .active_evaluation_resource_request
14204            .as_ref()
14205            .map(|request| request.request_id);
14206        let resource_error =
14207            |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
14208                FormulaSpanDemotionError::Resource(
14209                    crate::engine::ResourceLedgerError::Exhausted(
14210                        formualizer_common::ResourceExhaustionDetail {
14211                            reason,
14212                            limit,
14213                            observed,
14214                            request_id,
14215                        },
14216                    )
14217                    .into_excel_error(),
14218                )
14219            };
14220        let authority = self.graph.formula_authority();
14221        let expected_plane_epoch = authority.plane.epoch().0;
14222        let expected_indexes_epoch = authority.indexes_epoch();
14223        let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
14224        let mut unique = Vec::new();
14225        unique.try_reserve_exact(span_refs.len()).map_err(|_| {
14226            resource_error(
14227                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14228                u64::MAX,
14229                span_refs.len() as u64,
14230            )
14231        })?;
14232        let mut plans = Vec::new();
14233        plans.try_reserve_exact(span_refs.len()).map_err(|_| {
14234            resource_error(
14235                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14236                u64::MAX,
14237                span_refs.len() as u64,
14238            )
14239        })?;
14240        let mut placement_count = 0usize;
14241        let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
14242        for &span_ref in span_refs {
14243            if unique.contains(&span_ref) {
14244                continue;
14245            }
14246            unique.push(span_ref);
14247            let span = authority
14248                .plane
14249                .spans
14250                .get(span_ref)
14251                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14252            let relocation = span.ast_relocation;
14253            let ast = self
14254                .graph
14255                .data_store()
14256                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14257                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14258            let count = usize::try_from(span.domain.cell_count())
14259                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
14260            placement_count = placement_count
14261                .checked_add(count)
14262                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14263            if u64::try_from(placement_count)
14264                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
14265                > self.workbook_load_limits.max_formula_plane_fallback_cells
14266            {
14267                return Err(resource_error(
14268                    formualizer_common::ResourceExhaustionReason::MaterializationCells,
14269                    self.workbook_load_limits.max_formula_plane_fallback_cells,
14270                    placement_count as u64,
14271                ));
14272            }
14273            let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
14274            *sheet_count = sheet_count
14275                .checked_add(span.domain.cell_count())
14276                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14277            if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
14278                return Err(resource_error(
14279                    formualizer_common::ResourceExhaustionReason::Admission,
14280                    self.workbook_load_limits.max_sheet_logical_cells,
14281                    *sheet_count,
14282                ));
14283            }
14284            let domain_in_bounds = match &span.domain {
14285                PlacementDomain::RowRun { row_end, col, .. } => {
14286                    *row_end < self.workbook_load_limits.max_sheet_rows
14287                        && *col < self.workbook_load_limits.max_sheet_cols
14288                }
14289                PlacementDomain::ColRun { row, col_end, .. } => {
14290                    *row < self.workbook_load_limits.max_sheet_rows
14291                        && *col_end < self.workbook_load_limits.max_sheet_cols
14292                }
14293                PlacementDomain::Rect {
14294                    row_end, col_end, ..
14295                } => {
14296                    *row_end < self.workbook_load_limits.max_sheet_rows
14297                        && *col_end < self.workbook_load_limits.max_sheet_cols
14298                }
14299            };
14300            if !domain_in_bounds {
14301                return Err(resource_error(
14302                    formualizer_common::ResourceExhaustionReason::Admission,
14303                    u64::from(self.workbook_load_limits.max_sheet_rows)
14304                        .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
14305                    span.domain.cell_count(),
14306                ));
14307            }
14308            plans.push(SpanMaterialization {
14309                span_ref,
14310                sheet_id: span.sheet_id,
14311                ast,
14312                origin_row: relocation.anchor_row,
14313                origin_col: relocation.anchor_col,
14314                binding_set_id: span.binding_set_id,
14315                domain: span.domain.clone(),
14316            });
14317        }
14318        let mut relocated = Vec::new();
14319        relocated.try_reserve_exact(placement_count).map_err(|_| {
14320            resource_error(
14321                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14322                u64::MAX,
14323                placement_count as u64,
14324            )
14325        })?;
14326        for plan in &plans {
14327            for coord in plan.domain.iter() {
14328                let row = coord.row.saturating_add(1);
14329                let col = coord.col.saturating_add(1);
14330                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14331                    let authority = self.graph.formula_authority();
14332                    let binding_set = authority
14333                        .plane
14334                        .binding_sets
14335                        .get(binding_set_id)
14336                        .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14337                    if binding_set.is_single_literal_binding() {
14338                        plan.ast.clone()
14339                    } else {
14340                        let placement = PlacementCoord::new(
14341                            plan.sheet_id,
14342                            row.saturating_sub(1),
14343                            col.saturating_sub(1),
14344                        );
14345                        let span = authority
14346                            .plane
14347                            .spans
14348                            .get(plan.span_ref)
14349                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14350                        let binding = binding_set
14351                            .literal_bindings_for_placement(&span.domain, placement)
14352                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14353                        substitute_literal_slots(&plan.ast, binding.as_ref())
14354                    }
14355                } else {
14356                    plan.ast.clone()
14357                };
14358                let row_delta = i64::from(row) - i64::from(plan.origin_row);
14359                let col_delta = i64::from(col) - i64::from(plan.origin_col);
14360                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
14361                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
14362                relocated.push((plan.sheet_id, row, col, ast));
14363            }
14364        }
14365
14366        let mut analyzed = Vec::new();
14367        analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
14368            resource_error(
14369                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14370                u64::MAX,
14371                relocated.len() as u64,
14372            )
14373        })?;
14374        {
14375            let mut pipeline = self.ingest_pipeline();
14376            for (sheet_id, row, col, ast) in relocated {
14377                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
14378                let ingested = pipeline
14379                    .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
14380                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
14381                analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
14382            }
14383        }
14384
14385        if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
14386            return Err(FormulaSpanDemotionError::Injected(fault));
14387        }
14388        let legacy_graph = self
14389            .graph
14390            .prepare_legacy_graph_plan_multi_sheet(analyzed)
14391            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14392        self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
14393            .map_err(FormulaSpanDemotionError::Resource)?;
14394
14395        Ok(PreparedFormulaSpanDemotion {
14396            span_refs: unique,
14397            expected_plane_epoch,
14398            expected_indexes_epoch,
14399            expected_indexed_plane_epoch,
14400            placement_count,
14401            legacy_graph,
14402            fault,
14403        })
14404    }
14405
14406    pub(crate) fn validate_prepared_formula_span_demotion(
14407        &self,
14408        prepared: &PreparedFormulaSpanDemotion,
14409    ) -> Result<(), FormulaSpanDemotionError> {
14410        let authority = self.graph.formula_authority();
14411        if authority.plane.epoch().0 != prepared.expected_plane_epoch
14412            || authority.indexes_epoch() != prepared.expected_indexes_epoch
14413            || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14414            || prepared
14415                .span_refs
14416                .iter()
14417                .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14418        {
14419            return Err(FormulaSpanDemotionError::StaleAuthority);
14420        }
14421        self.graph
14422            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14423            .map_err(FormulaSpanDemotionError::LegacyGraph)
14424    }
14425
14426    /// Validate every recoverable assumption and fault before the first
14427    /// mutation, then compose the existing prevalidated graph append with exact
14428    /// authority removal. Allocator OOM/panic remains process-fatal, matching
14429    /// the E2/E3 prepared-transaction contract; there is no recoverable branch
14430    /// after graph application begins.
14431    pub(crate) fn commit_prepared_formula_span_demotion(
14432        &mut self,
14433        prepared: PreparedFormulaSpanDemotion,
14434    ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
14435        self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
14436            .map_err(FormulaSpanDemotionError::Resource)?;
14437        if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
14438            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14439        }
14440        self.graph
14441            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14442            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14443        if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
14444            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14445        }
14446        {
14447            let authority = self.graph.formula_authority();
14448            if authority.plane.epoch().0 != prepared.expected_plane_epoch
14449                || authority.indexes_epoch() != prepared.expected_indexes_epoch
14450                || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14451                || prepared
14452                    .span_refs
14453                    .iter()
14454                    .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14455            {
14456                return Err(FormulaSpanDemotionError::StaleAuthority);
14457            }
14458        }
14459        if matches!(
14460            prepared.fault,
14461            FormulaSpanDemotionFault::AllocationReservation
14462                | FormulaSpanDemotionFault::BeforeFirstMutation
14463        ) {
14464            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14465        }
14466
14467        let PreparedFormulaSpanDemotion {
14468            span_refs,
14469            placement_count,
14470            legacy_graph,
14471            ..
14472        } = prepared;
14473        let _ = self
14474            .graph
14475            .apply_prevalidated_legacy_graph_plan(legacy_graph);
14476        let authority = self.graph.formula_authority_mut();
14477        for span_ref in &span_refs {
14478            authority.plane.remove_overlays_for_source_span(*span_ref);
14479            let _ = authority.plane.remove_span(*span_ref);
14480        }
14481        let _ = authority.rebuild_indexes();
14482        self.mark_topology_edited();
14483        Ok(FormulaSpanDemotionReport {
14484            spans_demoted: span_refs.len(),
14485            placements_materialized: placement_count,
14486        })
14487    }
14488
14489    #[cfg(test)]
14490    pub(crate) fn set_formula_span_demotion_fault_for_test(
14491        &mut self,
14492        fault: FormulaSpanDemotionFault,
14493    ) {
14494        self.formula_span_demotion_fault_for_test = Some(fault);
14495    }
14496
14497    #[cfg(test)]
14498    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
14499        self.force_non_cycle_schedule_fallback_for_test = true;
14500    }
14501
14502    #[cfg(test)]
14503    pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
14504        self.mixed_topology_index_builds_for_test
14505    }
14506
14507    /// Collect the [`FormulaSpanRef`]s for span producers the mixed scheduler
14508    /// reported as cycle members (gotcha G8, refs #112). These spans must be
14509    /// demoted to legacy so the cycle members are resolved on the legacy SCC
14510    /// path; see [`Self::demote_cyclic_spans`].
14511    fn collect_cyclic_span_refs(
14512        &self,
14513        schedule: &MixedSchedule,
14514        span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
14515    ) -> Vec<FormulaSpanRef> {
14516        let mut refs = Vec::new();
14517        for fallback in &schedule.fallbacks {
14518            if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
14519                continue;
14520            }
14521            if let FormulaProducerId::Span(span_id) = fallback.producer
14522                && let Some(span_ref) = span_refs_by_id.get(&span_id)
14523                && !refs.contains(span_ref)
14524            {
14525                refs.push(*span_ref);
14526            }
14527        }
14528        refs
14529    }
14530
14531    /// Demote the given cyclic spans to legacy graph vertices so their member
14532    /// cells participate in the legacy Tarjan SCC pass (gotcha G8, refs #112).
14533    ///
14534    /// Uses one exact-ref prepared transaction for every cyclic span reported
14535    /// by this schedule. All recoverable validation and injected faults precede
14536    /// the first graph/authority mutation, so a multi-span or multi-sheet cycle
14537    /// cannot publish only an earlier subset.
14538    fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
14539        let materialization_started = crate::instant::FzInstant::now();
14540        let prepared = self
14541            .prepare_formula_span_demotion(span_refs)
14542            .map_err(|error| {
14543                if let FormulaSpanDemotionError::Resource(error) = error {
14544                    error
14545                } else {
14546                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14547                        "FormulaPlane cycle-member span demotion preparation failed: {error}"
14548                    ))
14549                }
14550            })?;
14551        let report = self
14552            .commit_prepared_formula_span_demotion(prepared)
14553            .map_err(|error| {
14554                ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14555                    "FormulaPlane cycle-member span demotion commit failed: {error}"
14556                ))
14557            })?;
14558        self.observe_materialization(
14559            report.placements_materialized,
14560            true,
14561            materialization_started.elapsed(),
14562        );
14563        self.formula_plane_cycle_member_span_demotions = self
14564            .formula_plane_cycle_member_span_demotions
14565            .saturating_add(report.spans_demoted as u64);
14566        Self::record_shadow_fallback_reason(
14567            &mut self.formula_ingest_report_total,
14568            PlacementFallbackReason::CycleMember,
14569            report.spans_demoted as u64,
14570        );
14571        Ok(())
14572    }
14573
14574    /// Evaluate residual *legacy-only* cyclic SCCs before the FormulaPlane
14575    /// mixed schedule runs (gotcha G8, refs #112).
14576    ///
14577    /// After cyclic spans are demoted to legacy ([`Self::demote_cyclic_spans`]),
14578    /// every cycle member is a graph vertex, so the cycle is now visible to the
14579    /// legacy Tarjan pass and lives entirely among legacy producers. The mixed
14580    /// schedule treats any cycle as not authoritative-safe; rather than abandon
14581    /// the surviving spans by falling through to a pure-legacy `evaluate_all`,
14582    /// stamp/evaluate just the cyclic SCC units here (`handle_cycle_unit` honors
14583    /// `CycleDetection::Static` vs `Runtime`), clear their dirty flags, and let
14584    /// the mixed schedule proceed cycle-free over the surviving spans plus the
14585    /// acyclic legacy work.
14586    ///
14587    /// Returns the number of cyclic SCC units that stamped at least one cell.
14588    fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
14589        let dirty = self.graph.get_evaluation_vertices();
14590        if dirty.is_empty() {
14591            return Ok(0);
14592        }
14593        let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
14594        let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
14595        let mut cycle_errors = 0usize;
14596        let mut stamped_vertices: Vec<VertexId> = Vec::new();
14597        for &unit in &schedule.units {
14598            let ScheduleUnit::Cycle(i) = unit else {
14599                continue;
14600            };
14601            let members = schedule.unit_cycle(i);
14602            let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
14603            if stamped > 0 {
14604                cycle_errors += 1;
14605            }
14606            stamped_vertices.extend(members.iter().copied());
14607        }
14608        // Clear dirty only on the cyclic members so the subsequent mixed
14609        // schedule no longer sees them as dirty legacy producers (which is what
14610        // surfaced the cycle). Acyclic legacy work stays dirty and is scheduled
14611        // normally alongside the surviving spans.
14612        if !stamped_vertices.is_empty() {
14613            self.resource_checkpoint(0)?;
14614            self.graph.clear_dirty_flags(&stamped_vertices);
14615        }
14616        Ok(cycle_errors)
14617    }
14618
14619    fn materialize_deferred_sheet_before_structural_edit(
14620        &mut self,
14621        sheet: &str,
14622    ) -> Result<(), crate::engine::EditorError> {
14623        if self.staged_formulas.contains_key(sheet) {
14624            self.build_graph_for_sheets([sheet])?;
14625        }
14626        Ok(())
14627    }
14628
14629    /// Insert rows (1-based) and mirror into Arrow store when enabled
14630    pub fn insert_rows(
14631        &mut self,
14632        sheet: &str,
14633        before: u32,
14634        count: u32,
14635    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14636    {
14637        if count == 0 {
14638            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14639        }
14640        self.observe_function_semantic_epoch()
14641            .map_err(crate::engine::EditorError::Excel)?;
14642        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14643        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14644        let sheet_id = self.ensure_known_sheet_id(sheet)?;
14645        let before0 = before.saturating_sub(1);
14646        let affected_region = Self::structural_row_region(sheet_id, before0);
14647        let op = StructuralOp::InsertRows {
14648            sheet_id,
14649            before: before0,
14650            count,
14651        };
14652        self.demote_spans_for_structural_op(op, affected_region)?;
14653        let summary = {
14654            let mut editor = VertexEditor::new(&mut self.graph);
14655            editor.insert_rows(sheet_id, before0, count)?
14656        };
14657        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14658            let before0 = before0 as usize;
14659            asheet.insert_rows(before0, count as usize);
14660        }
14661        self.mark_moved_formula_vertices_dirty(&summary);
14662        self.clear_computed_overlay_after_row(sheet, before0 as usize);
14663        self.shift_row_visibility_insert(sheet_id, before0, count);
14664        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14665        self.mark_topology_edited();
14666        Ok(summary)
14667    }
14668
14669    /// Delete rows (1-based) and mirror into Arrow store when enabled
14670    pub fn delete_rows(
14671        &mut self,
14672        sheet: &str,
14673        start: u32,
14674        count: u32,
14675    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14676    {
14677        if count == 0 {
14678            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14679        }
14680        self.observe_function_semantic_epoch()
14681            .map_err(crate::engine::EditorError::Excel)?;
14682        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14683        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14684        let sheet_id = self.ensure_known_sheet_id(sheet)?;
14685        let start0 = start.saturating_sub(1);
14686        let affected_region = Self::structural_row_region(sheet_id, start0);
14687        let op = StructuralOp::DeleteRows {
14688            sheet_id,
14689            start: start0,
14690            count,
14691        };
14692        self.demote_spans_for_structural_op(op, affected_region)?;
14693        let summary = {
14694            let mut editor = VertexEditor::new(&mut self.graph);
14695            editor.delete_rows(sheet_id, start0, count)?
14696        };
14697        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14698            let start0 = start0 as usize;
14699            asheet.delete_rows(start0, count as usize);
14700        }
14701        self.mark_moved_formula_vertices_dirty(&summary);
14702        self.clear_computed_overlay_after_row(sheet, start0 as usize);
14703        self.shift_row_visibility_delete(sheet_id, start0, count);
14704        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14705        self.mark_topology_edited();
14706        Ok(summary)
14707    }
14708
14709    /// Insert columns (1-based) and mirror into Arrow store when enabled
14710    pub fn insert_columns(
14711        &mut self,
14712        sheet: &str,
14713        before: u32,
14714        count: u32,
14715    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14716    {
14717        if count == 0 {
14718            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14719        }
14720        self.observe_function_semantic_epoch()
14721            .map_err(crate::engine::EditorError::Excel)?;
14722        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14723        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14724        let sheet_id = self.graph.sheet_id(sheet).ok_or(
14725            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14726                name: sheet.to_string(),
14727                reason: "Unknown sheet".to_string(),
14728            },
14729        )?;
14730        let before0 = before.saturating_sub(1);
14731        let affected_region = Self::structural_col_region(sheet_id, before0);
14732        let op = StructuralOp::InsertColumns {
14733            sheet_id,
14734            before: before0,
14735            count,
14736        };
14737        self.demote_spans_for_structural_op(op, affected_region)?;
14738        let summary = {
14739            let mut editor = VertexEditor::new(&mut self.graph);
14740            editor.insert_columns(sheet_id, before0, count)?
14741        };
14742        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14743            let before0 = before0 as usize;
14744            asheet.insert_columns(before0, count as usize);
14745        }
14746        self.mark_moved_formula_vertices_dirty(&summary);
14747        self.clear_computed_overlay_after_col(sheet, before0 as usize);
14748        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14749        self.mark_topology_edited();
14750        Ok(summary)
14751    }
14752
14753    /// Delete columns (1-based) and mirror into Arrow store when enabled
14754    pub fn delete_columns(
14755        &mut self,
14756        sheet: &str,
14757        start: u32,
14758        count: u32,
14759    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14760    {
14761        if count == 0 {
14762            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14763        }
14764        self.observe_function_semantic_epoch()
14765            .map_err(crate::engine::EditorError::Excel)?;
14766        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14767        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14768        let sheet_id = self.graph.sheet_id(sheet).ok_or(
14769            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14770                name: sheet.to_string(),
14771                reason: "Unknown sheet".to_string(),
14772            },
14773        )?;
14774        let start0 = start.saturating_sub(1);
14775        let affected_region = Self::structural_col_region(sheet_id, start0);
14776        let op = StructuralOp::DeleteColumns {
14777            sheet_id,
14778            start: start0,
14779            count,
14780        };
14781        self.demote_spans_for_structural_op(op, affected_region)?;
14782        let summary = {
14783            let mut editor = VertexEditor::new(&mut self.graph);
14784            editor.delete_columns(sheet_id, start0, count)?
14785        };
14786        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14787            let start0 = start0 as usize;
14788            asheet.delete_columns(start0, count as usize);
14789        }
14790        self.mark_moved_formula_vertices_dirty(&summary);
14791        self.clear_computed_overlay_after_col(sheet, start0 as usize);
14792        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14793        self.mark_topology_edited();
14794        Ok(summary)
14795    }
14796    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
14797    fn arrow_used_row_bounds(
14798        &self,
14799        sheet: &str,
14800        start_col: u32,
14801        end_col: u32,
14802    ) -> Option<(u32, u32)> {
14803        let a = self.sheet_store().sheet(sheet)?;
14804        if a.columns.is_empty() {
14805            return None;
14806        }
14807        let sc0 = start_col.saturating_sub(1) as usize;
14808        let ec0 = end_col.saturating_sub(1) as usize;
14809        let col_hi = a.columns.len().saturating_sub(1);
14810        if sc0 > col_hi {
14811            return None;
14812        }
14813        let ec0 = ec0.min(col_hi);
14814        // Pass-scoped cache with snapshot guard
14815        let snap = self.data_snapshot_id();
14816        let mut min_r0: Option<usize> = None;
14817        for ci in sc0..=ec0 {
14818            let sheet_id = self.graph.sheet_id(sheet)?;
14819            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
14820                g.as_ref()
14821                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14822            }) {
14823                let mv = mv as usize;
14824                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
14825                continue;
14826            }
14827            // Compute and store
14828            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14829            if let Ok(mut g) = self.row_bounds_cache.write() {
14830                g.get_or_insert_with(|| RowBoundsCache::new(snap))
14831                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
14832            }
14833            if let Some(m) = min_c {
14834                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
14835            }
14836        }
14837        min_r0?;
14838        let mut max_r0: Option<usize> = None;
14839        for ci in sc0..=ec0 {
14840            let sheet_id = self.graph.sheet_id(sheet)?;
14841            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
14842                g.as_ref()
14843                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14844            }) {
14845                let mv = mv as usize;
14846                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
14847                continue;
14848            }
14849            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14850            if let Ok(mut g) = self.row_bounds_cache.write() {
14851                g.get_or_insert_with(|| RowBoundsCache::new(snap))
14852                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
14853            }
14854            if let Some(m) = max_c {
14855                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
14856            }
14857        }
14858        match (min_r0, max_r0) {
14859            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
14860            _ => None,
14861        }
14862    }
14863
14864    fn scan_column_used_bounds(
14865        a: &crate::arrow_store::ArrowSheet,
14866        ci: usize,
14867    ) -> (Option<u32>, Option<u32>) {
14868        let col = &a.columns[ci];
14869
14870        // Min: scan dense chunks first, then sparse chunks in ascending index order.
14871        let mut min_r0: Option<u32> = None;
14872        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
14873            let tags = chunk.type_tag.values();
14874            for (off, &t) in tags.iter().enumerate() {
14875                let overlay_non_empty = chunk
14876                    .overlay
14877                    .get(off)
14878                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14879                    .unwrap_or(false)
14880                    || chunk
14881                        .computed_overlay
14882                        .get(off)
14883                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14884                        .unwrap_or(false);
14885                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14886                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14887                        break;
14888                    };
14889                    let row0 = chunk_start + off;
14890                    min_r0 = Some(row0 as u32);
14891                    break;
14892                }
14893            }
14894            if min_r0.is_some() {
14895                break;
14896            }
14897        }
14898        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
14899            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14900            sparse_idxs.sort_unstable();
14901            for chunk_idx in sparse_idxs {
14902                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14903                    continue;
14904                };
14905                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14906                    continue;
14907                };
14908                let tags = chunk.type_tag.values();
14909                for (off, &t) in tags.iter().enumerate() {
14910                    let overlay_non_empty = chunk
14911                        .overlay
14912                        .get(off)
14913                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14914                        .unwrap_or(false)
14915                        || chunk
14916                            .computed_overlay
14917                            .get(off)
14918                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14919                            .unwrap_or(false);
14920                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14921                        let row0 = chunk_start + off;
14922                        min_r0 = Some(row0 as u32);
14923                        break;
14924                    }
14925                }
14926                if min_r0.is_some() {
14927                    break;
14928                }
14929            }
14930        }
14931
14932        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
14933        let mut max_r0: Option<u32> = None;
14934        if !col.sparse_chunks.is_empty() {
14935            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14936            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
14937            for chunk_idx in sparse_idxs {
14938                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14939                    continue;
14940                };
14941                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14942                    continue;
14943                };
14944                let tags = chunk.type_tag.values();
14945                for (rev_idx, &t) in tags.iter().enumerate().rev() {
14946                    let overlay_non_empty = chunk
14947                        .overlay
14948                        .get(rev_idx)
14949                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14950                        .unwrap_or(false)
14951                        || chunk
14952                            .computed_overlay
14953                            .get(rev_idx)
14954                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14955                            .unwrap_or(false);
14956                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14957                        let row0 = chunk_start + rev_idx;
14958                        max_r0 = Some(row0 as u32);
14959                        break;
14960                    }
14961                }
14962                if max_r0.is_some() {
14963                    break;
14964                }
14965            }
14966        }
14967        if max_r0.is_none() {
14968            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
14969                let tags = chunk.type_tag.values();
14970                for (rev_idx, &t) in tags.iter().enumerate().rev() {
14971                    let overlay_non_empty = chunk
14972                        .overlay
14973                        .get(rev_idx)
14974                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14975                        .unwrap_or(false)
14976                        || chunk
14977                            .computed_overlay
14978                            .get(rev_idx)
14979                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14980                            .unwrap_or(false);
14981                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14982                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14983                            break;
14984                        };
14985                        let row0 = chunk_start + rev_idx;
14986                        max_r0 = Some(row0 as u32);
14987                        break;
14988                    }
14989                }
14990                if max_r0.is_some() {
14991                    break;
14992                }
14993            }
14994        }
14995
14996        (min_r0, max_r0)
14997    }
14998
14999    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
15000    fn arrow_used_col_bounds(
15001        &self,
15002        sheet: &str,
15003        start_row: u32,
15004        end_row: u32,
15005    ) -> Option<(u32, u32)> {
15006        let a = self.sheet_store().sheet(sheet)?;
15007        if a.columns.is_empty() {
15008            return None;
15009        }
15010        let sr0 = start_row.saturating_sub(1) as usize;
15011        let er0 = end_row.saturating_sub(1) as usize;
15012        if sr0 > er0 {
15013            return None;
15014        }
15015        // Map start/end rows into chunk ranges
15016        // We will scan each column for any non-empty within [sr0..=er0]
15017        let mut min_c0: Option<usize> = None;
15018        let mut max_c0: Option<usize> = None;
15019        // Precompute chunk bounds for row range
15020        for (ci, col) in a.columns.iter().enumerate() {
15021            let mut any_in_range = false;
15022
15023            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15024                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15025                    return false;
15026                };
15027                let chunk_len = chunk.type_tag.len();
15028                if chunk_len == 0 {
15029                    return false;
15030                }
15031                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15032                // check intersection
15033                if sr0 > chunk_end || er0 < chunk_start {
15034                    return false;
15035                }
15036                let start_off = sr0.max(chunk_start) - chunk_start;
15037                let end_off = er0.min(chunk_end) - chunk_start;
15038                let tags = chunk.type_tag.values();
15039                for off in start_off..=end_off {
15040                    let overlay_non_empty = chunk
15041                        .overlay
15042                        .get(off)
15043                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15044                        .unwrap_or(false)
15045                        || chunk
15046                            .computed_overlay
15047                            .get(off)
15048                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15049                            .unwrap_or(false);
15050                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15051                        return true;
15052                    }
15053                }
15054                false
15055            };
15056
15057            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15058                if scan_chunk(chunk_idx, chunk) {
15059                    any_in_range = true;
15060                    break;
15061                }
15062            }
15063
15064            if !any_in_range && !col.sparse_chunks.is_empty() {
15065                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15066                    if scan_chunk(chunk_idx, chunk) {
15067                        any_in_range = true;
15068                        break;
15069                    }
15070                }
15071            }
15072
15073            if any_in_range {
15074                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
15075                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
15076            }
15077        }
15078        match (min_c0, max_c0) {
15079            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15080            _ => None,
15081        }
15082    }
15083
15084    fn formula_row_bounds_for_columns(
15085        &self,
15086        sheet: &str,
15087        start_col: u32,
15088        end_col: u32,
15089    ) -> Option<(u32, u32)> {
15090        let sheet_id = self.graph.sheet_id(sheet)?;
15091        let sc0 = start_col.saturating_sub(1);
15092        let ec0 = end_col.saturating_sub(1);
15093        let mut min_r0: Option<u32> = None;
15094        let mut max_r0: Option<u32> = None;
15095
15096        if let Some(index) = self.graph.sheet_index(sheet_id) {
15097            for vid in index.vertices_in_col_range(sc0, ec0) {
15098                if !matches!(
15099                    self.graph.get_vertex_kind(vid),
15100                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15101                ) {
15102                    continue;
15103                }
15104                let row0 = self.graph.vertex_coord(vid).row();
15105                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15106                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15107            }
15108        } else {
15109            for vid in self.graph.vertices_in_sheet(sheet_id) {
15110                if !matches!(
15111                    self.graph.get_vertex_kind(vid),
15112                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15113                ) {
15114                    continue;
15115                }
15116                let coord = self.graph.vertex_coord(vid);
15117                let col0 = coord.col();
15118                if col0 < sc0 || col0 > ec0 {
15119                    continue;
15120                }
15121                let row0 = coord.row();
15122                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15123                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15124            }
15125        }
15126
15127        match (min_r0, max_r0) {
15128            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15129            _ => None,
15130        }
15131    }
15132
15133    fn formula_col_bounds_for_rows(
15134        &self,
15135        sheet: &str,
15136        start_row: u32,
15137        end_row: u32,
15138    ) -> Option<(u32, u32)> {
15139        let sheet_id = self.graph.sheet_id(sheet)?;
15140        let sr0 = start_row.saturating_sub(1);
15141        let er0 = end_row.saturating_sub(1);
15142        let mut min_c0: Option<u32> = None;
15143        let mut max_c0: Option<u32> = None;
15144
15145        if let Some(index) = self.graph.sheet_index(sheet_id) {
15146            for vid in index.vertices_in_row_range(sr0, er0) {
15147                if !matches!(
15148                    self.graph.get_vertex_kind(vid),
15149                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15150                ) {
15151                    continue;
15152                }
15153                let col0 = self.graph.vertex_coord(vid).col();
15154                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15155                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15156            }
15157        } else {
15158            for vid in self.graph.vertices_in_sheet(sheet_id) {
15159                if !matches!(
15160                    self.graph.get_vertex_kind(vid),
15161                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15162                ) {
15163                    continue;
15164                }
15165                let coord = self.graph.vertex_coord(vid);
15166                let row0 = coord.row();
15167                if row0 < sr0 || row0 > er0 {
15168                    continue;
15169                }
15170                let col0 = coord.col();
15171                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15172                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15173            }
15174        }
15175
15176        match (min_c0, max_c0) {
15177            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15178            _ => None,
15179        }
15180    }
15181
15182    fn union_used_bounds(
15183        first: Option<(u32, u32)>,
15184        second: Option<(u32, u32)>,
15185    ) -> Option<(u32, u32)> {
15186        match (first, second) {
15187            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
15188            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
15189            (None, None) => None,
15190        }
15191    }
15192
15193    /// Mirror a single cell value into the Arrow overlay if enabled.
15194    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
15195    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
15196        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15197            return;
15198        }
15199        if self.arrow_sheets.sheet(sheet).is_none() {
15200            self.arrow_sheets
15201                .sheets
15202                .push(crate::arrow_store::ArrowSheet {
15203                    name: std::sync::Arc::<str>::from(sheet),
15204                    date_system: self.config.date_system,
15205                    columns: Vec::new(),
15206                    nrows: 0,
15207                    chunk_starts: Vec::new(),
15208                    chunk_rows: 32 * 1024,
15209                });
15210        }
15211
15212        let row0 = row.saturating_sub(1) as usize;
15213        let col0 = col.saturating_sub(1) as usize;
15214
15215        let asheet = self
15216            .arrow_sheets
15217            .sheet_mut(sheet)
15218            .expect("ArrowSheet must exist");
15219
15220        let cur_cols = asheet.columns.len();
15221        if col0 >= cur_cols {
15222            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15223        }
15224
15225        if row0 >= asheet.nrows as usize {
15226            if asheet.columns.is_empty() {
15227                asheet.insert_columns(0, 1);
15228            }
15229            asheet.ensure_row_capacity(row0 + 1);
15230        }
15231        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
15232            let ov =
15233                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
15234            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
15235                let _ = ch.overlay.set(in_off, ov);
15236                // A user edit must invalidate any computed (formula/spill) overlay entry at
15237                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
15238                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
15239                // the edited base value under the read cascade.
15240                ch.computed_overlay.remove(in_off)
15241            } else {
15242                return;
15243            };
15244            // Heuristic compaction: > len/50 or > 1024
15245            let abs_threshold = 1024usize;
15246            let frac_den = 50usize;
15247            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
15248            if freed > 0 {
15249                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
15250            }
15251            self.adjust_computed_overlay_bytes(computed_delta);
15252        }
15253    }
15254
15255    /// Remove a delta-overlay entry for a single cell (if present).
15256    ///
15257    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
15258    /// does not continue to mask computed overlay outputs.
15259    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
15260        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15261            return;
15262        }
15263        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15264            return;
15265        };
15266        let row0 = row.saturating_sub(1) as usize;
15267        let col0 = col.saturating_sub(1) as usize;
15268        if row0 >= asheet.nrows as usize {
15269            return;
15270        }
15271        if col0 >= asheet.columns.len() {
15272            return;
15273        }
15274        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15275            return;
15276        };
15277        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
15278            let _ = ch.overlay.remove(in_off);
15279        }
15280    }
15281
15282    fn clear_computed_overlay_col_row_range(
15283        &mut self,
15284        sheet: &str,
15285        col0: usize,
15286        start_row0: usize,
15287        end_row0_exclusive: usize,
15288    ) {
15289        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15290            return;
15291        }
15292        if start_row0 >= end_row0_exclusive {
15293            return;
15294        }
15295
15296        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15297            return;
15298        };
15299        if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
15300            return;
15301        }
15302        let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
15303        if start_row0 >= end_row0_exclusive {
15304            return;
15305        }
15306
15307        let starts = asheet.chunk_starts.clone();
15308        let nrows = asheet.nrows as usize;
15309        let mut delta = 0isize;
15310        let Some(col) = asheet.columns.get_mut(col0) else {
15311            return;
15312        };
15313        for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15314            let Some(&chunk_start) = starts.get(chunk_idx) else {
15315                continue;
15316            };
15317            let chunk_end = starts
15318                .get(chunk_idx + 1)
15319                .copied()
15320                .unwrap_or(nrows)
15321                .min(chunk_start.saturating_add(ch.len()));
15322            let clear_start = start_row0.max(chunk_start);
15323            let clear_end = end_row0_exclusive.min(chunk_end);
15324            if clear_start >= clear_end {
15325                continue;
15326            }
15327            if clear_start == chunk_start && clear_end == chunk_end {
15328                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15329            } else {
15330                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15331                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15332                delta = delta.saturating_add(
15333                    ch.computed_overlay
15334                        .remove_range(start_in_chunk..end_in_chunk),
15335                );
15336            }
15337        }
15338        for (chunk_idx, ch) in &mut col.sparse_chunks {
15339            let Some(&chunk_start) = starts.get(*chunk_idx) else {
15340                continue;
15341            };
15342            let chunk_end = starts
15343                .get(*chunk_idx + 1)
15344                .copied()
15345                .unwrap_or(nrows)
15346                .min(chunk_start.saturating_add(ch.len()));
15347            let clear_start = start_row0.max(chunk_start);
15348            let clear_end = end_row0_exclusive.min(chunk_end);
15349            if clear_start >= clear_end {
15350                continue;
15351            }
15352            if clear_start == chunk_start && clear_end == chunk_end {
15353                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15354            } else {
15355                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15356                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15357                delta = delta.saturating_add(
15358                    ch.computed_overlay
15359                        .remove_range(start_in_chunk..end_in_chunk),
15360                );
15361            }
15362        }
15363        self.adjust_computed_overlay_bytes(delta);
15364    }
15365
15366    fn clear_computed_overlay_cells_in_region(
15367        &mut self,
15368        cells: &[(SheetId, u32, u32)],
15369        affected_region: &Region,
15370    ) {
15371        let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
15372        for (formula_sheet_id, row, col) in cells {
15373            let row0 = row.saturating_sub(1);
15374            let col0 = col.saturating_sub(1);
15375            let placement_region = Region::point(*formula_sheet_id, row0, col0);
15376            if placement_region.intersects(affected_region) {
15377                by_col
15378                    .entry((*formula_sheet_id, col0))
15379                    .or_default()
15380                    .push(row0);
15381            }
15382        }
15383
15384        for ((formula_sheet_id, col0), mut rows) in by_col {
15385            rows.sort_unstable();
15386            rows.dedup();
15387            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
15388            let mut start = rows[0];
15389            let mut prev = rows[0];
15390            for row in rows.into_iter().skip(1) {
15391                if row == prev.saturating_add(1) {
15392                    prev = row;
15393                    continue;
15394                }
15395                self.clear_computed_overlay_col_row_range(
15396                    &sheet_name,
15397                    col0 as usize,
15398                    start as usize,
15399                    prev.saturating_add(1) as usize,
15400                );
15401                start = row;
15402                prev = row;
15403            }
15404            self.clear_computed_overlay_col_row_range(
15405                &sheet_name,
15406                col0 as usize,
15407                start as usize,
15408                prev.saturating_add(1) as usize,
15409            );
15410        }
15411    }
15412
15413    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
15414        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15415            return;
15416        }
15417
15418        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15419            return;
15420        };
15421        if start_row0 >= asheet.nrows as usize {
15422            return;
15423        }
15424
15425        let starts = asheet.chunk_starts.clone();
15426        let nrows = asheet.nrows as usize;
15427        let mut delta = 0isize;
15428        for col in &mut asheet.columns {
15429            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15430                let Some(&chunk_start) = starts.get(chunk_idx) else {
15431                    continue;
15432                };
15433                let chunk_end = starts
15434                    .get(chunk_idx + 1)
15435                    .copied()
15436                    .unwrap_or(nrows)
15437                    .min(chunk_start.saturating_add(ch.len()));
15438                if chunk_end <= start_row0 {
15439                    continue;
15440                }
15441                if chunk_start >= start_row0 {
15442                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15443                } else {
15444                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15445                    delta = delta
15446                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15447                }
15448            }
15449
15450            for (chunk_idx, ch) in &mut col.sparse_chunks {
15451                let Some(&chunk_start) = starts.get(*chunk_idx) else {
15452                    continue;
15453                };
15454                let chunk_end = starts
15455                    .get(*chunk_idx + 1)
15456                    .copied()
15457                    .unwrap_or(nrows)
15458                    .min(chunk_start.saturating_add(ch.len()));
15459                if chunk_end <= start_row0 {
15460                    continue;
15461                }
15462                if chunk_start >= start_row0 {
15463                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15464                } else {
15465                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15466                    delta = delta
15467                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15468                }
15469            }
15470        }
15471        self.adjust_computed_overlay_bytes(delta);
15472    }
15473
15474    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
15475        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15476            return;
15477        }
15478
15479        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15480            return;
15481        };
15482        if start_col0 >= asheet.columns.len() {
15483            return;
15484        }
15485
15486        let mut delta = 0isize;
15487        for col in asheet.columns.iter_mut().skip(start_col0) {
15488            for ch in &mut col.chunks {
15489                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15490            }
15491            for ch in col.sparse_chunks.values_mut() {
15492                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15493            }
15494        }
15495        self.adjust_computed_overlay_bytes(delta);
15496    }
15497
15498    #[inline]
15499    fn literal_to_overlay_value(
15500        value: &LiteralValue,
15501        date_system: crate::engine::DateSystem,
15502    ) -> crate::arrow_store::OverlayValue {
15503        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
15504    }
15505
15506    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
15507        self.arrow_sheets
15508            .sheet(sheet)
15509            .map(|sheet| sheet.date_system)
15510            .unwrap_or(self.config.date_system)
15511    }
15512
15513    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
15514    /// absent and explicit `Empty`.
15515    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15516        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15517            return None;
15518        }
15519        let asheet = self.arrow_sheets.sheet(sheet)?;
15520        let row0 = row.saturating_sub(1) as usize;
15521        let col0 = col.saturating_sub(1) as usize;
15522        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15523            return None;
15524        }
15525        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15526        let ch = asheet.columns[col0].chunk(ch_idx)?;
15527        ch.overlay
15528            .get_scalar(in_off)
15529            .map(|ov| ov.to_literal_for(asheet.date_system))
15530    }
15531
15532    /// Read a single cell's computed overlay entry (if present), preserving the distinction
15533    /// between absent and explicit `Empty`.
15534    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15535        if !(self.config.arrow_storage_enabled
15536            && self.config.delta_overlay_enabled
15537            && self.config.write_formula_overlay_enabled)
15538        {
15539            return None;
15540        }
15541        let asheet = self.arrow_sheets.sheet(sheet)?;
15542        let row0 = row.saturating_sub(1) as usize;
15543        let col0 = col.saturating_sub(1) as usize;
15544        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15545            return None;
15546        }
15547        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15548        let ch = asheet.columns[col0].chunk(ch_idx)?;
15549        ch.computed_overlay
15550            .get_scalar(in_off)
15551            .map(|ov| ov.to_literal_for(asheet.date_system))
15552    }
15553
15554    fn set_delta_overlay_cell_raw(
15555        &mut self,
15556        sheet: &str,
15557        row: u32,
15558        col: u32,
15559        value: Option<LiteralValue>,
15560    ) {
15561        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15562            return;
15563        }
15564
15565        self.ensure_arrow_sheet(sheet);
15566        let date_system = self.arrow_sheet_date_system(sheet);
15567        let ov_opt = value
15568            .as_ref()
15569            .map(|value| Self::literal_to_overlay_value(value, date_system));
15570        let row0 = row.saturating_sub(1) as usize;
15571        let col0 = col.saturating_sub(1) as usize;
15572        let asheet = self
15573            .arrow_sheets
15574            .sheet_mut(sheet)
15575            .expect("ArrowSheet must exist");
15576
15577        let cur_cols = asheet.columns.len();
15578        if col0 >= cur_cols {
15579            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15580        }
15581        if row0 >= asheet.nrows as usize {
15582            if asheet.columns.is_empty() {
15583                asheet.insert_columns(0, 1);
15584            }
15585            asheet.ensure_row_capacity(row0 + 1);
15586        }
15587
15588        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15589            return;
15590        };
15591        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15592            return;
15593        };
15594
15595        if let Some(ov) = ov_opt {
15596            let _ = ch.overlay.set(in_off, ov);
15597        } else {
15598            let _ = ch.overlay.remove(in_off);
15599        }
15600    }
15601
15602    fn set_computed_overlay_cell_raw(
15603        &mut self,
15604        sheet: &str,
15605        row: u32,
15606        col: u32,
15607        value: Option<LiteralValue>,
15608    ) {
15609        if !(self.config.arrow_storage_enabled
15610            && self.config.delta_overlay_enabled
15611            && self.config.write_formula_overlay_enabled)
15612        {
15613            return;
15614        }
15615
15616        self.ensure_arrow_sheet(sheet);
15617        let date_system = self.arrow_sheet_date_system(sheet);
15618        let ov_opt = value
15619            .as_ref()
15620            .map(|value| Self::literal_to_overlay_value(value, date_system));
15621        let row0 = row.saturating_sub(1) as usize;
15622        let col0 = col.saturating_sub(1) as usize;
15623        let asheet = self
15624            .arrow_sheets
15625            .sheet_mut(sheet)
15626            .expect("ArrowSheet must exist");
15627
15628        let cur_cols = asheet.columns.len();
15629        if col0 >= cur_cols {
15630            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15631        }
15632        if row0 >= asheet.nrows as usize {
15633            if asheet.columns.is_empty() {
15634                asheet.insert_columns(0, 1);
15635            }
15636            asheet.ensure_row_capacity(row0 + 1);
15637        }
15638
15639        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15640            return;
15641        };
15642        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15643            return;
15644        };
15645
15646        let delta = if let Some(ov) = ov_opt {
15647            ch.computed_overlay.set(in_off, ov)
15648        } else {
15649            ch.computed_overlay.remove(in_off)
15650        };
15651        self.adjust_computed_overlay_bytes(delta);
15652    }
15653
15654    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
15655        use crate::engine::ArrowOp;
15656
15657        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
15658            Box::new(batch.ops.iter().rev())
15659        } else {
15660            Box::new(batch.ops.iter())
15661        };
15662
15663        for op in iter {
15664            match op {
15665                ArrowOp::SetDeltaCell {
15666                    sheet_id,
15667                    row0,
15668                    col0,
15669                    old,
15670                    new,
15671                } => {
15672                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15673                    let v = if undo { old.clone() } else { new.clone() };
15674                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15675                }
15676                ArrowOp::SetComputedCell {
15677                    sheet_id,
15678                    row0,
15679                    col0,
15680                    old,
15681                    new,
15682                } => {
15683                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15684                    let v = if undo { old.clone() } else { new.clone() };
15685                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15686                }
15687                ArrowOp::RestoreComputedRect {
15688                    sheet_id,
15689                    sr0,
15690                    sc0,
15691                    er0,
15692                    ec0,
15693                    old,
15694                    new,
15695                } => {
15696                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15697                    let vals = if undo { old } else { new };
15698                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
15699                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
15700                    for r in 0..height {
15701                        for c in 0..width {
15702                            let v = vals
15703                                .get(r)
15704                                .and_then(|row| row.get(c))
15705                                .cloned()
15706                                .unwrap_or(LiteralValue::Empty);
15707                            self.set_computed_overlay_cell_raw(
15708                                &sheet,
15709                                *sr0 + 1 + r as u32,
15710                                *sc0 + 1 + c as u32,
15711                                Some(v),
15712                            );
15713                        }
15714                    }
15715                }
15716                ArrowOp::InsertRows {
15717                    sheet_id,
15718                    before0,
15719                    count,
15720                } => {
15721                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15722                    self.ensure_arrow_sheet(&sheet);
15723                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15724                        if undo {
15725                            asheet.delete_rows(*before0 as usize, *count as usize);
15726                        } else {
15727                            asheet.insert_rows(*before0 as usize, *count as usize);
15728                        }
15729                    }
15730                }
15731                ArrowOp::InsertCols {
15732                    sheet_id,
15733                    before0,
15734                    count,
15735                } => {
15736                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15737                    self.ensure_arrow_sheet(&sheet);
15738                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15739                        if undo {
15740                            asheet.delete_columns(*before0 as usize, *count as usize);
15741                        } else {
15742                            asheet.insert_columns(*before0 as usize, *count as usize);
15743                        }
15744                    }
15745                }
15746            }
15747        }
15748    }
15749
15750    fn record_spill_ops_into_arrow_undo(
15751        &mut self,
15752        undo: &mut crate::engine::ArrowUndoBatch,
15753        events: &[crate::engine::ChangeEvent],
15754    ) {
15755        use crate::engine::ChangeEvent;
15756        use formualizer_common::LiteralValue;
15757
15758        #[allow(clippy::type_complexity)]
15759        let rect_from_snapshot =
15760            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
15761             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
15762                if snap.target_cells.is_empty() {
15763                    return None;
15764                }
15765                let sheet_id = snap.target_cells[0].sheet_id;
15766                let sr0 = snap.target_cells[0].coord.row();
15767                let sc0 = snap.target_cells[0].coord.col();
15768                if snap.values.is_empty() || snap.values[0].is_empty() {
15769                    return None;
15770                }
15771                let h = snap.values.len() as u32;
15772                let w = snap.values[0].len() as u32;
15773                let er0 = sr0.saturating_add(h.saturating_sub(1));
15774                let ec0 = sc0.saturating_add(w.saturating_sub(1));
15775                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
15776            };
15777
15778        for ev in events {
15779            match ev {
15780                ChangeEvent::SpillCommitted { old, new, .. } => {
15781                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
15782                        let old_vals = if let Some(old_snap) = old {
15783                            rect_from_snapshot(old_snap)
15784                                .map(|(_, _, _, _, _, v)| v)
15785                                .unwrap_or_else(|| {
15786                                    vec![
15787                                        vec![LiteralValue::Empty; new_vals[0].len()];
15788                                        new_vals.len()
15789                                    ]
15790                                })
15791                        } else {
15792                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
15793                        };
15794                        undo.record_restore_computed_rect(
15795                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15796                        );
15797                    }
15798                }
15799                ChangeEvent::SpillCleared { old, .. } => {
15800                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
15801                        let new_vals =
15802                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
15803                        undo.record_restore_computed_rect(
15804                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15805                        );
15806                    }
15807                }
15808                _ => {}
15809            }
15810        }
15811    }
15812
15813    /// Mirror a value into the computed overlay (formula/spill outputs).
15814    ///
15815    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
15816    /// If the cap is exceeded, computed overlays are compacted into base lanes.
15817    fn mirror_value_to_computed_overlay(
15818        &mut self,
15819        sheet: &str,
15820        row: u32,
15821        col: u32,
15822        value: &LiteralValue,
15823    ) {
15824        if !(self.config.arrow_storage_enabled
15825            && self.config.delta_overlay_enabled
15826            && self.config.write_formula_overlay_enabled)
15827        {
15828            return;
15829        }
15830        if self.computed_overlay_mirroring_disabled {
15831            return;
15832        }
15833
15834        let date_system = self.arrow_sheet_date_system(sheet);
15835        let ov = Self::literal_to_overlay_value(value, date_system);
15836        self.write_computed_overlay_value_0based(
15837            sheet,
15838            row.saturating_sub(1),
15839            col.saturating_sub(1),
15840            ov,
15841        );
15842    }
15843
15844    fn write_computed_overlay_value_0based(
15845        &mut self,
15846        sheet: &str,
15847        row0: u32,
15848        col0: u32,
15849        value: OverlayValue,
15850    ) {
15851        if !(self.config.arrow_storage_enabled
15852            && self.config.delta_overlay_enabled
15853            && self.config.write_formula_overlay_enabled)
15854        {
15855            return;
15856        }
15857        if self.computed_overlay_mirroring_disabled {
15858            return;
15859        }
15860
15861        self.ensure_arrow_sheet(sheet);
15862
15863        let row0 = row0 as usize;
15864        let col0 = col0 as usize;
15865        let asheet = self
15866            .arrow_sheets
15867            .sheet_mut(sheet)
15868            .expect("ArrowSheet must exist");
15869
15870        let cur_cols = asheet.columns.len();
15871        if col0 >= cur_cols {
15872            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15873        }
15874
15875        if row0 >= asheet.nrows as usize {
15876            if asheet.columns.is_empty() {
15877                asheet.insert_columns(0, 1);
15878            }
15879            asheet.ensure_row_capacity(row0 + 1);
15880        }
15881
15882        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15883            return;
15884        };
15885        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15886            return;
15887        };
15888
15889        let delta = ch.computed_overlay.set_scalar(in_off, value);
15890        self.adjust_computed_overlay_bytes(delta);
15891
15892        if let Some(cap) = self.config.max_overlay_memory_bytes
15893            && self.computed_overlay_bytes_estimate > cap
15894        {
15895            self.disable_computed_overlay_mirroring_due_to_budget(cap);
15896        }
15897    }
15898
15899    pub(crate) fn plan_computed_write_coalescing(
15900        &self,
15901        buffer: &ComputedWriteBuffer,
15902    ) -> ComputedWriteCoalescingPlan {
15903        self.plan_computed_write_coalescing_from_writes(buffer.writes().iter().cloned())
15904    }
15905
15906    fn plan_owned_computed_write_coalescing(
15907        &self,
15908        writes: Vec<ComputedWrite>,
15909    ) -> ComputedWriteCoalescingPlan {
15910        self.plan_computed_write_coalescing_from_writes(writes)
15911    }
15912
15913    fn plan_computed_write_coalescing_from_writes(
15914        &self,
15915        writes: impl IntoIterator<Item = ComputedWrite>,
15916    ) -> ComputedWriteCoalescingPlan {
15917        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
15918            BTreeMap::new();
15919        let mut input_cells = 0usize;
15920
15921        for write in writes {
15922            match write {
15923                ComputedWrite::Cell {
15924                    seq,
15925                    sheet_id,
15926                    row0,
15927                    col0,
15928                    value,
15929                } => {
15930                    input_cells = input_cells.saturating_add(1);
15931                    self.push_computed_write_plan_entry(
15932                        &mut groups,
15933                        seq,
15934                        sheet_id,
15935                        row0,
15936                        col0,
15937                        value,
15938                    );
15939                }
15940                ComputedWrite::Rect {
15941                    seq,
15942                    sheet_id,
15943                    sr0,
15944                    sc0,
15945                    values,
15946                } => {
15947                    for (r_off, row) in values.into_iter().enumerate() {
15948                        for (c_off, value) in row.into_iter().enumerate() {
15949                            input_cells = input_cells.saturating_add(1);
15950                            self.push_computed_write_plan_entry(
15951                                &mut groups,
15952                                seq,
15953                                sheet_id,
15954                                sr0.saturating_add(r_off as u32),
15955                                sc0.saturating_add(c_off as u32),
15956                                value,
15957                            );
15958                        }
15959                    }
15960                }
15961            }
15962        }
15963
15964        let mut plan = ComputedWriteCoalescingPlan {
15965            chunks: Vec::with_capacity(groups.len()),
15966            input_cells,
15967            coalesced_cells: 0,
15968            overwritten_cells: 0,
15969        };
15970        for (key, entries) in groups {
15971            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(key, entries);
15972            plan.coalesced_cells = plan
15973                .coalesced_cells
15974                .saturating_add(chunk_plan.entries.len());
15975            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
15976            plan.chunks.push(chunk_plan);
15977        }
15978        debug_assert_eq!(
15979            plan.input_cells,
15980            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
15981        );
15982        plan
15983    }
15984
15985    fn push_computed_write_plan_entry(
15986        &self,
15987        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
15988        seq: u64,
15989        sheet_id: SheetId,
15990        row0: u32,
15991        col0: u32,
15992        value: OverlayValue,
15993    ) {
15994        let (chunk_idx, chunk_start_row0, row_in_chunk) =
15995            self.locate_computed_write_chunk(sheet_id, row0);
15996        let key = ComputedWriteChunkKey {
15997            sheet_id,
15998            col0,
15999            chunk_idx,
16000            chunk_start_row0,
16001        };
16002        groups
16003            .entry(key)
16004            .or_default()
16005            .push(ComputedWriteChunkEntryPlan {
16006                row_in_chunk,
16007                seq,
16008                value,
16009            });
16010    }
16011
16012    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
16013        let sheet_name = self.graph.sheet_name(sheet_id);
16014        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
16015            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
16016        }
16017        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
16018    }
16019
16020    fn locate_row_in_sheet_for_computed_write_plan(
16021        sheet: &crate::arrow_store::ArrowSheet,
16022        row0: usize,
16023    ) -> (usize, u32, usize) {
16024        if row0 < sheet.nrows as usize
16025            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
16026        {
16027            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
16028            return (chunk_idx, chunk_start as u32, row_in_chunk);
16029        }
16030
16031        let chunk_rows = sheet.chunk_rows.max(1);
16032        if sheet.chunk_starts.is_empty() {
16033            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
16034        }
16035
16036        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
16037        let mut chunk_start = sheet.chunk_starts[chunk_idx];
16038        while chunk_start.saturating_add(chunk_rows) <= row0 {
16039            chunk_idx = chunk_idx.saturating_add(1);
16040            chunk_start = chunk_start.saturating_add(chunk_rows);
16041        }
16042        (
16043            chunk_idx,
16044            chunk_start as u32,
16045            row0.saturating_sub(chunk_start),
16046        )
16047    }
16048
16049    fn locate_row_in_empty_sheet_for_computed_write_plan(
16050        row0: usize,
16051        chunk_rows: usize,
16052    ) -> (usize, u32, usize) {
16053        let chunk_rows = chunk_rows.max(1);
16054        let chunk_idx = row0 / chunk_rows;
16055        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
16056        (
16057            chunk_idx,
16058            chunk_start as u32,
16059            row0.saturating_sub(chunk_start),
16060        )
16061    }
16062
16063    #[cfg(test)]
16064    pub(crate) fn debug_plan_computed_write_coalescing(
16065        &self,
16066        buffer: &ComputedWriteBuffer,
16067    ) -> ComputedWriteCoalescingPlan {
16068        self.plan_computed_write_coalescing(buffer)
16069    }
16070
16071    fn record_computed_write_buffer_delta(
16072        &self,
16073        buffer: &ComputedWriteBuffer,
16074        delta: &mut DeltaCollector,
16075    ) {
16076        if delta.mode == DeltaMode::Off {
16077            return;
16078        }
16079        let mut final_values = BTreeMap::new();
16080        for write in buffer.writes() {
16081            match write {
16082                ComputedWrite::Cell {
16083                    sheet_id,
16084                    row0,
16085                    col0,
16086                    value,
16087                    ..
16088                } => {
16089                    final_values.insert((*sheet_id, *row0, *col0), value.clone());
16090                }
16091                ComputedWrite::Rect {
16092                    sheet_id,
16093                    sr0,
16094                    sc0,
16095                    values,
16096                    ..
16097                } => {
16098                    for (row_offset, row) in values.iter().enumerate() {
16099                        for (col_offset, value) in row.iter().enumerate() {
16100                            final_values.insert(
16101                                (
16102                                    *sheet_id,
16103                                    sr0.saturating_add(row_offset as u32),
16104                                    sc0.saturating_add(col_offset as u32),
16105                                ),
16106                                value.clone(),
16107                            );
16108                        }
16109                    }
16110                }
16111            }
16112        }
16113        for ((sheet_id, row0, col0), value) in final_values {
16114            let old = self
16115                .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
16116                .unwrap_or(LiteralValue::Empty);
16117            if old != value.to_literal_for(self.config.date_system) {
16118                delta.record_cell(sheet_id, row0, col0);
16119            }
16120        }
16121    }
16122
16123    pub(crate) fn flush_computed_write_buffer(
16124        &mut self,
16125        buffer: &mut ComputedWriteBuffer,
16126    ) -> Result<(), ExcelError> {
16127        if buffer.is_empty() {
16128            return Ok(());
16129        }
16130
16131        // Keep ownership of all pending writes until the final request
16132        // checkpoint and bounded commit-window preflight succeed so failures
16133        // remain retry safe. The plan and flush that follow are prevalidated
16134        // and contain no cancellation point.
16135        self.resource_checkpoint(0)?;
16136        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
16137        let plan = self.plan_owned_computed_write_coalescing(buffer.take_writes());
16138        self.flush_computed_write_plan(plan);
16139        self.observe_evaluation_commit_window(commit_started);
16140
16141        Ok(())
16142    }
16143
16144    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
16145        for chunk in plan.chunks {
16146            self.flush_computed_write_chunk_plan(chunk);
16147        }
16148    }
16149
16150    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
16151        match &chunk.shape {
16152            ComputedWriteChunkPlanShape::Point => {
16153                self.flush_computed_write_chunk_plan_as_points(chunk);
16154            }
16155            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
16156                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
16157            }
16158            ComputedWriteChunkPlanShape::DenseRange { .. } => {
16159                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16160            }
16161            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
16162                if Self::should_emit_computed_run_fragment(*len, *runs) {
16163                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
16164                } else {
16165                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16166                }
16167            }
16168        }
16169    }
16170
16171    #[inline]
16172    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
16173        runs <= len / 2
16174    }
16175
16176    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
16177        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
16178        for entry in chunk.entries {
16179            let row0 = chunk
16180                .chunk_start_row0
16181                .saturating_add(entry.row_in_chunk as u32);
16182            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
16183        }
16184    }
16185
16186    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
16187        &mut self,
16188        chunk: ComputedWriteChunkPlan,
16189    ) {
16190        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
16191        let sheet_id = chunk.sheet_id;
16192        let col0 = chunk.col0;
16193        let chunk_idx = chunk.chunk_idx;
16194        let chunk_start_row0 = chunk.chunk_start_row0;
16195        let items: Vec<(usize, OverlayValue)> = chunk
16196            .entries
16197            .into_iter()
16198            .map(|entry| (entry.row_in_chunk, entry.value))
16199            .collect();
16200        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
16201            items,
16202            point_estimate,
16203        ) {
16204            Some(Ok(fragment)) => {
16205                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
16206            }
16207            Some(Err(cells)) => {
16208                self.flush_computed_overlay_cells_as_points(
16209                    sheet_id,
16210                    col0,
16211                    chunk_start_row0,
16212                    cells,
16213                );
16214            }
16215            None => {}
16216        }
16217    }
16218
16219    #[inline]
16220    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
16221        chunk
16222            .entries
16223            .iter()
16224            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
16225            .fold(0usize, usize::saturating_add)
16226    }
16227
16228    fn flush_computed_overlay_cells_as_points(
16229        &mut self,
16230        sheet_id: SheetId,
16231        col0: u32,
16232        chunk_start_row0: u32,
16233        cells: Vec<(usize, OverlayValue)>,
16234    ) {
16235        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16236        for (row_in_chunk, value) in cells {
16237            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
16238            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
16239        }
16240    }
16241
16242    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16243        if chunk.entries.is_empty() {
16244            return;
16245        }
16246        let start = chunk.entries[0].row_in_chunk;
16247        let values: Vec<OverlayValue> =
16248            chunk.entries.into_iter().map(|entry| entry.value).collect();
16249        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
16250            self.apply_computed_overlay_fragment(
16251                chunk.sheet_id,
16252                chunk.col0,
16253                chunk.chunk_idx,
16254                fragment,
16255            );
16256        }
16257    }
16258
16259    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16260        if chunk.entries.is_empty() {
16261            return;
16262        }
16263        let start = chunk.entries[0].row_in_chunk;
16264        let values: Vec<OverlayValue> =
16265            chunk.entries.into_iter().map(|entry| entry.value).collect();
16266        if let Some(fragment) = OverlayFragment::run_range(start, values) {
16267            self.apply_computed_overlay_fragment(
16268                chunk.sheet_id,
16269                chunk.col0,
16270                chunk.chunk_idx,
16271                fragment,
16272            );
16273        }
16274    }
16275
16276    fn apply_computed_overlay_fragment(
16277        &mut self,
16278        sheet_id: SheetId,
16279        col0: u32,
16280        chunk_idx: usize,
16281        fragment: OverlayFragment,
16282    ) {
16283        if !(self.config.arrow_storage_enabled
16284            && self.config.delta_overlay_enabled
16285            && self.config.write_formula_overlay_enabled)
16286        {
16287            return;
16288        }
16289        if self.computed_overlay_mirroring_disabled {
16290            return;
16291        }
16292
16293        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16294        self.ensure_arrow_sheet(&sheet_name);
16295
16296        let col0 = col0 as usize;
16297        let asheet = self
16298            .arrow_sheets
16299            .sheet_mut(&sheet_name)
16300            .expect("ArrowSheet must exist");
16301
16302        let cur_cols = asheet.columns.len();
16303        if col0 >= cur_cols {
16304            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16305        }
16306
16307        let start_row0 = asheet
16308            .chunk_starts
16309            .get(chunk_idx)
16310            .copied()
16311            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
16312        let required_rows =
16313            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
16314        if required_rows > asheet.nrows as usize {
16315            if asheet.columns.is_empty() {
16316                asheet.insert_columns(0, 1);
16317            }
16318            asheet.ensure_row_capacity(required_rows);
16319        }
16320
16321        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
16322            return;
16323        };
16324        let delta = ch.computed_overlay.apply_fragment(fragment);
16325        self.adjust_computed_overlay_bytes(delta);
16326
16327        if let Some(cap) = self.config.max_overlay_memory_bytes
16328            && self.computed_overlay_bytes_estimate > cap
16329        {
16330            self.disable_computed_overlay_mirroring_due_to_budget(cap);
16331        }
16332    }
16333
16334    #[inline]
16335    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
16336        if delta >= 0 {
16337            self.computed_overlay_bytes_estimate = self
16338                .computed_overlay_bytes_estimate
16339                .saturating_add(delta as usize);
16340        } else {
16341            self.computed_overlay_bytes_estimate = self
16342                .computed_overlay_bytes_estimate
16343                .saturating_sub((-delta) as usize);
16344        }
16345    }
16346
16347    fn clear_all_computed_overlays(&mut self) {
16348        let mut freed_total = 0usize;
16349        for sh in self.arrow_sheets.sheets.iter_mut() {
16350            for col in sh.columns.iter_mut() {
16351                for ch in col.chunks.iter_mut() {
16352                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16353                }
16354                for ch in col.sparse_chunks.values_mut() {
16355                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16356                }
16357            }
16358        }
16359        self.computed_overlay_bytes_estimate = self
16360            .computed_overlay_bytes_estimate
16361            .saturating_sub(freed_total);
16362    }
16363
16364    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
16365        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
16366        // Handle budget pressure by compacting computed overlays into base lanes.
16367        self.compact_all_computed_overlays();
16368    }
16369
16370    /// Fold all computed overlay entries across all sheets into their base arrays.
16371    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
16372    fn compact_all_computed_overlays(&mut self) {
16373        let mut freed_total = 0usize;
16374        for sheet in self.arrow_sheets.sheets.iter_mut() {
16375            for col_idx in 0..sheet.columns.len() {
16376                // Dense chunks
16377                let num_dense = sheet.columns[col_idx].chunks.len();
16378                for ch_idx in 0..num_dense {
16379                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
16380                }
16381                // Sparse chunks
16382                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
16383                    .sparse_chunks
16384                    .keys()
16385                    .copied()
16386                    .collect();
16387                for ch_idx in sparse_keys {
16388                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
16389                }
16390            }
16391        }
16392        self.computed_overlay_bytes_estimate = self
16393            .computed_overlay_bytes_estimate
16394            .saturating_sub(freed_total);
16395        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16396    }
16397
16398    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
16399        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
16400    }
16401
16402    fn record_vertex_value_to_overlay(
16403        &mut self,
16404        vertex_id: VertexId,
16405        value: &LiteralValue,
16406        computed_writes: Option<&mut ComputedWriteBuffer>,
16407    ) -> Result<(), ExcelError> {
16408        if !(self.config.arrow_storage_enabled
16409            && self.config.delta_overlay_enabled
16410            && self.config.write_formula_overlay_enabled)
16411        {
16412            return Ok(());
16413        }
16414        if self.computed_overlay_mirroring_disabled {
16415            return Ok(());
16416        }
16417        if !matches!(
16418            self.graph.get_vertex_kind(vertex_id),
16419            VertexKind::FormulaScalar | VertexKind::FormulaArray
16420        ) {
16421            return Ok(());
16422        }
16423        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
16424            return Ok(());
16425        };
16426        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
16427        let date_system = self.arrow_sheet_date_system(&sheet_name);
16428        let ov = Self::literal_to_overlay_value(value, date_system);
16429        if let Some(buffer) = computed_writes {
16430            buffer.push_cell(cell.sheet_id, cell.coord.row(), cell.coord.col(), ov);
16431            if self.should_flush_computed_write_buffer(buffer) {
16432                self.flush_computed_write_buffer(buffer)?;
16433            }
16434        } else {
16435            self.write_computed_overlay_value_0based(
16436                &sheet_name,
16437                cell.coord.row(),
16438                cell.coord.col(),
16439                ov,
16440            );
16441        }
16442        Ok(())
16443    }
16444
16445    #[inline]
16446    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
16447        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
16448            if cap == 0 {
16449                return false;
16450            }
16451            self.computed_overlay_bytes_estimate
16452                .saturating_add(buffer.estimated_bytes())
16453                > cap
16454        })
16455    }
16456
16457    /// Estimated memory usage for computed overlays (formula/spill mirroring).
16458    pub fn overlay_memory_usage(&self) -> usize {
16459        self.computed_overlay_bytes_estimate
16460    }
16461
16462    #[cfg(test)]
16463    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
16464        self.overlay_compactions
16465    }
16466
16467    #[cfg(test)]
16468    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
16469        let mut total = 0usize;
16470        for sheet in &self.arrow_sheets.sheets {
16471            for column in &sheet.columns {
16472                for chunk in &column.chunks {
16473                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16474                }
16475                for chunk in column.sparse_chunks.values() {
16476                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16477                }
16478            }
16479        }
16480        self.computed_overlay_bytes_estimate = total;
16481        total
16482    }
16483
16484    fn resolve_sheet_locator_for_write(
16485        &mut self,
16486        loc: formualizer_common::SheetLocator<'_>,
16487        current_sheet: &str,
16488    ) -> Result<SheetId, ExcelError> {
16489        Ok(match loc {
16490            formualizer_common::SheetLocator::Id(id) => id,
16491            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
16492            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
16493        })
16494    }
16495
16496    fn resolve_sheet_locator_for_read(
16497        &self,
16498        loc: formualizer_common::SheetLocator<'_>,
16499        current_sheet: &str,
16500    ) -> Result<SheetId, ExcelError> {
16501        match loc {
16502            formualizer_common::SheetLocator::Id(id) => Ok(id),
16503            formualizer_common::SheetLocator::Name(name) => self
16504                .graph
16505                .sheet_id(name.as_ref())
16506                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16507            formualizer_common::SheetLocator::Current => self
16508                .graph
16509                .sheet_id(current_sheet)
16510                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16511        }
16512    }
16513
16514    /// Set a cell value
16515    pub fn set_cell_value(
16516        &mut self,
16517        sheet: &str,
16518        row: u32,
16519        col: u32,
16520        value: LiteralValue,
16521    ) -> Result<(), ExcelError> {
16522        self.observe_function_semantic_epoch()?;
16523        let sheet_existed = self.graph.sheet_id(sheet).is_some();
16524        let sheet_id = self.graph.sheet_id_mut(sheet);
16525        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16526        let replaced_formula =
16527            self.graph
16528                .get_vertex_id_for_address(&cell_ref)
16529                .is_some_and(|vertex| {
16530                    matches!(
16531                        self.graph.get_vertex_kind(*vertex),
16532                        VertexKind::FormulaScalar | VertexKind::FormulaArray
16533                    )
16534                });
16535        self.demote_span_containing_cell_for_write(
16536            sheet_id,
16537            row.saturating_sub(1),
16538            col.saturating_sub(1),
16539        )
16540        .map_err(Self::editor_error_to_excel)?;
16541        self.graph.set_cell_value(sheet, row, col, value.clone())?;
16542        self.record_formula_plane_changed_cell(sheet, row, col);
16543        if !sheet_existed || replaced_formula {
16544            self.mark_topology_edited();
16545        }
16546        // Mirror into Arrow overlay when enabled
16547        self.mirror_value_to_overlay(sheet, row, col, &value);
16548        // Advance snapshot to reflect external mutation
16549        self.snapshot_id
16550            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16551        self.has_edited = true;
16552        Ok(())
16553    }
16554
16555    /// Record a single-cell change in FormulaPlane authority so the next
16556    /// `evaluate_all` under `AuthoritativeExperimental` can derive bounded
16557    /// span work from `FormulaConsumerReadIndex` instead of recomputing every
16558    /// active span.
16559    fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
16560        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16561            return;
16562        }
16563        let sheet_id = self.graph.sheet_id_mut(sheet);
16564        self.record_formula_plane_structural_change(StructuralScope::Cell {
16565            sheet: sheet_id,
16566            row: row.saturating_sub(1),
16567            col: col.saturating_sub(1),
16568        });
16569    }
16570
16571    fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
16572        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16573            return;
16574        }
16575
16576        match event {
16577            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16578                self.record_formula_plane_structural_change(StructuralScope::Cell {
16579                    sheet: addr.sheet_id,
16580                    row: addr.coord.row(),
16581                    col: addr.coord.col(),
16582                });
16583            }
16584            ChangeEvent::SpillCommitted { new, .. } => {
16585                if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
16586                    self.record_formula_plane_structural_change(scope);
16587                }
16588            }
16589            ChangeEvent::SpillCleared { old, .. } => {
16590                if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
16591                    self.record_formula_plane_structural_change(scope);
16592                }
16593            }
16594            ChangeEvent::DefineName { .. }
16595            | ChangeEvent::UpdateName { .. }
16596            | ChangeEvent::DeleteName { .. }
16597            | ChangeEvent::NamedRangeAdjusted { .. } => {
16598                // Direct name events are preflighted by the logged-name APIs.
16599                // Structural entry points preflight spans before emitting a
16600                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
16601                self.record_formula_plane_structural_change(StructuralScope::AllSheets);
16602            }
16603            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
16604                // Structural entry points publish their axis delta once after
16605                // graph, Arrow, and span geometry commits.
16606            }
16607            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
16608                self.record_formula_plane_structural_change(StructuralScope::Region(
16609                    Region::whole_row(*sheet_id, *row0),
16610                ));
16611            }
16612            ChangeEvent::AddVertex { .. }
16613            | ChangeEvent::RemoveVertex { .. }
16614            | ChangeEvent::EdgeAdded { .. }
16615            | ChangeEvent::EdgeRemoved { .. }
16616            | ChangeEvent::CompoundStart { .. }
16617            | ChangeEvent::CompoundEnd { .. }
16618            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
16619        }
16620    }
16621
16622    fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
16623        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16624            return;
16625        }
16626
16627        match scope {
16628            StructuralScope::Cell { sheet, row, col } => {
16629                self.graph
16630                    .mark_formula_region_dirty(Region::point(sheet, row, col));
16631            }
16632            StructuralScope::Region(region) => {
16633                self.graph.mark_formula_region_dirty(region);
16634            }
16635            StructuralScope::Sheet(sheet_id) => {
16636                self.graph
16637                    .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
16638            }
16639            StructuralScope::RemovedSheet(sheet_id) => {
16640                let removed_refs = {
16641                    let authority = self.graph.formula_authority();
16642                    authority
16643                        .active_span_refs()
16644                        .into_iter()
16645                        .filter(|span_ref| {
16646                            authority
16647                                .plane
16648                                .spans
16649                                .get(*span_ref)
16650                                .map(|span| span.sheet_id == sheet_id)
16651                                .unwrap_or(false)
16652                        })
16653                        .collect::<Vec<_>>()
16654                };
16655
16656                {
16657                    let authority = self.graph.formula_authority_mut();
16658                    for span_ref in removed_refs {
16659                        authority.plane.remove_span(span_ref);
16660                    }
16661                    let _ = authority.rebuild_indexes();
16662                }
16663                self.graph
16664                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16665            }
16666            StructuralScope::OpaqueGlobal => {
16667                let _ = self.graph.formula_authority_mut().rebuild_indexes();
16668                self.graph
16669                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16670            }
16671            StructuralScope::AllSheets => {
16672                // Name/table mutations dirty their exact graph dependents,
16673                // and name-dependent spans are demoted by the name invalidation
16674                // registry. A topology rebuild is cache invalidation only.
16675                let _ = self.graph.formula_authority_mut().rebuild_indexes();
16676            }
16677        }
16678    }
16679
16680    fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
16681        let first = cells.first()?;
16682        let sheet_id = first.sheet_id;
16683        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
16684            return Some(StructuralScope::OpaqueGlobal);
16685        }
16686        let mut row_start = first.coord.row();
16687        let mut row_end = row_start;
16688        let mut col_start = first.coord.col();
16689        let mut col_end = col_start;
16690        for cell in cells.iter().skip(1) {
16691            row_start = row_start.min(cell.coord.row());
16692            row_end = row_end.max(cell.coord.row());
16693            col_start = col_start.min(cell.coord.col());
16694            col_end = col_end.max(cell.coord.col());
16695        }
16696        Some(StructuralScope::Region(Region::rect(
16697            sheet_id, row_start, row_end, col_start, col_end,
16698        )))
16699    }
16700
16701    pub fn set_cell_value_ref(
16702        &mut self,
16703        cell: formualizer_common::SheetCellRef<'_>,
16704        current_sheet: &str,
16705        value: LiteralValue,
16706    ) -> Result<(), ExcelError> {
16707        let owned = cell.into_owned();
16708        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16709        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16710        self.set_cell_value(
16711            &sheet_name,
16712            owned.coord.row() + 1,
16713            owned.coord.col() + 1,
16714            value,
16715        )
16716    }
16717
16718    pub fn set_cell_formula_ref(
16719        &mut self,
16720        cell: formualizer_common::SheetCellRef<'_>,
16721        current_sheet: &str,
16722        ast: ASTNode,
16723    ) -> Result<(), ExcelError> {
16724        let owned = cell.into_owned();
16725        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16726        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16727        self.set_cell_formula(
16728            &sheet_name,
16729            owned.coord.row() + 1,
16730            owned.coord.col() + 1,
16731            ast,
16732        )
16733    }
16734
16735    pub fn get_cell_value_ref(
16736        &self,
16737        cell: formualizer_common::SheetCellRef<'_>,
16738        current_sheet: &str,
16739    ) -> Result<Option<LiteralValue>, ExcelError> {
16740        let owned = cell.into_owned();
16741        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
16742        let sheet_name = self.graph.sheet_name(sheet_id);
16743        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
16744    }
16745
16746    pub fn resolve_range_view_sheet_ref<'c>(
16747        &'c self,
16748        r: &formualizer_common::SheetRef<'_>,
16749        current_sheet: &str,
16750    ) -> Result<RangeView<'c>, ExcelError> {
16751        use formualizer_common::SheetLocator;
16752
16753        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
16754            match loc {
16755                SheetLocator::Current => Ok(None),
16756                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
16757                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
16758            }
16759        };
16760
16761        let rt = match r {
16762            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
16763                sheet: sheet_to_opt_name(cell.sheet.clone())?,
16764                row: cell.coord.row() + 1,
16765                col: cell.coord.col() + 1,
16766                row_abs: cell.coord.row_abs(),
16767                col_abs: cell.coord.col_abs(),
16768            },
16769            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
16770                sheet: sheet_to_opt_name(range.sheet.clone())?,
16771                start_row: range.start_row.map(|b| b.index + 1),
16772                start_col: range.start_col.map(|b| b.index + 1),
16773                end_row: range.end_row.map(|b| b.index + 1),
16774                end_col: range.end_col.map(|b| b.index + 1),
16775                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
16776                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
16777                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
16778                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
16779            },
16780        };
16781
16782        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
16783    }
16784
16785    /// Set a cell formula
16786    pub fn set_cell_formula(
16787        &mut self,
16788        sheet: &str,
16789        row: u32,
16790        col: u32,
16791        ast: ASTNode,
16792    ) -> Result<(), ExcelError> {
16793        self.observe_function_semantic_epoch()?;
16794        let sheet_id = self.graph.sheet_id_mut(sheet);
16795        self.demote_span_containing_cell_for_write(
16796            sheet_id,
16797            row.saturating_sub(1),
16798            col.saturating_sub(1),
16799        )
16800        .map_err(Self::editor_error_to_excel)?;
16801        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16802        let ingested = {
16803            let mut pipeline = self.ingest_pipeline();
16804            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
16805        };
16806        self.graph.set_cell_formula_with_plan(
16807            sheet,
16808            row,
16809            col,
16810            ingested.ast_id,
16811            &ingested.dep_plan,
16812            ingested.dep_plan.volatile,
16813            ingested.dep_plan.dynamic,
16814        )?;
16815        self.record_formula_plane_changed_cell(sheet, row, col);
16816
16817        // If the cell previously held a user value in the delta overlay, it must not continue
16818        // to mask the formula result under Arrow-canonical reads (overlay precedence is
16819        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
16820        // because an explicit `Empty` overlay would still take precedence over computed values.
16821        self.clear_delta_overlay_cell(sheet, row, col);
16822
16823        // Advance snapshot to reflect external mutation
16824        self.mark_topology_edited();
16825        Ok(())
16826    }
16827
16828    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
16829    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
16830    where
16831        I: IntoIterator<Item = (u32, u32, ASTNode)>,
16832    {
16833        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
16834        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
16835        let sheet_id = self.graph.sheet_id_mut(sheet);
16836        let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
16837            let placement =
16838                PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
16839            self.graph
16840                .formula_authority()
16841                .plane
16842                .spans
16843                .find_at(placement)
16844                .is_some()
16845        });
16846        if writes_inside_active_span {
16847            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
16848                .map_err(Self::editor_error_to_excel)?;
16849        }
16850        let ingested = {
16851            let mut pipeline = self.ingest_pipeline();
16852            let inputs = collected.into_iter().map(|(row, col, ast)| {
16853                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16854                (FormulaAstInput::Tree(ast), placement, None)
16855            });
16856            pipeline.ingest_batch(inputs)?
16857        };
16858        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
16859            .into_iter()
16860            .map(|formula| {
16861                (
16862                    formula.placement.coord.row() + 1,
16863                    formula.placement.coord.col() + 1,
16864                    formula.ast_id,
16865                    formula.dep_plan,
16866                )
16867            })
16868            .collect();
16869        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
16870        for (row, col) in edited_cells {
16871            self.record_formula_plane_changed_cell(sheet, row, col);
16872        }
16873        // Single topology bump after batch
16874        if n > 0 {
16875            self.mark_topology_edited();
16876        }
16877        Ok(n)
16878    }
16879
16880    #[inline]
16881    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
16882        match v {
16883            LiteralValue::Empty => None,
16884            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
16885            other => Some(other),
16886        }
16887    }
16888
16889    /// Get a cell value
16890    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16891        self.read_cell_value(sheet, row, col)
16892            .and_then(Self::normalize_public_cell_read)
16893    }
16894
16895    /// Unified internal read API for a single cell value (Arrow-truth).
16896    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16897        let asheet = self.sheet_store().sheet(sheet)?;
16898        let r0 = row.saturating_sub(1) as usize;
16899        let c0 = col.saturating_sub(1) as usize;
16900        let v = asheet.get_cell_value(r0, c0);
16901        if matches!(v, LiteralValue::Empty) {
16902            None
16903        } else {
16904            Some(v)
16905        }
16906    }
16907
16908    /// Unified internal read API for a range of cell values (Arrow-truth).
16909    pub(crate) fn read_range_values(
16910        &self,
16911        sheet: &str,
16912        sr: u32,
16913        sc: u32,
16914        er: u32,
16915        ec: u32,
16916    ) -> RangeView<'_> {
16917        let Some(asheet) = self.sheet_store().sheet(sheet) else {
16918            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
16919        };
16920        if er < sr || ec < sc {
16921            return asheet.range_view(1, 1, 0, 0);
16922        }
16923        let sr0 = sr.saturating_sub(1) as usize;
16924        let sc0 = sc.saturating_sub(1) as usize;
16925        let er0 = er.saturating_sub(1) as usize;
16926        let ec0 = ec.saturating_sub(1) as usize;
16927        asheet.range_view(sr0, sc0, er0, ec0)
16928    }
16929
16930    pub(crate) fn formula_plane_span_ast_at(
16931        &self,
16932        span_ref: FormulaSpanRef,
16933        placement: PlacementCoord,
16934    ) -> Option<ASTNode> {
16935        let authority = self.graph.formula_authority();
16936        let span = authority.plane.spans.get(span_ref)?;
16937        let relocation = span.ast_relocation;
16938        let mut ast = self
16939            .graph
16940            .data_store()
16941            .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
16942        if let Some(binding_set_id) = span.binding_set_id {
16943            let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
16944            if !binding_set.is_single_literal_binding() {
16945                let binding =
16946                    binding_set.literal_bindings_for_placement(&span.domain, placement)?;
16947                ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
16948            }
16949        }
16950        let row = placement.row.checked_add(1)?;
16951        let col = placement.col.checked_add(1)?;
16952        let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
16953        let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
16954        relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
16955    }
16956
16957    /// Get formula AST (if any) and current stored value for a cell
16958    pub fn get_cell(
16959        &self,
16960        sheet: &str,
16961        row: u32,
16962        col: u32,
16963    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
16964        let v = self.get_cell_value(sheet, row, col);
16965        let sheet_id = self.graph.sheet_id(sheet)?;
16966        let coord = Coord::from_excel(row, col, true, true);
16967        let cell = CellRef::new(sheet_id, coord);
16968        let placement =
16969            crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
16970        let handle = self
16971            .graph
16972            .formula_authority()
16973            .plane
16974            .resolve_formula_at(placement, None);
16975        match handle.resolution {
16976            crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
16977                // Span authority wins before graph lookup. Missing or invalid
16978                // relocation state is fail-closed: never expose a stale legacy
16979                // vertex for a coordinate owned by a span.
16980                let ast = self.formula_plane_span_ast_at(span, placement);
16981                return Some((ast, v));
16982            }
16983            crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
16984                let ast = self
16985                    .graph
16986                    .formula_authority()
16987                    .plane
16988                    .formula_overlay
16989                    .get(overlay_ref)
16990                    .and_then(|overlay| match overlay.kind {
16991                        crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
16992                            template_id,
16993                        ) => self
16994                            .graph
16995                            .formula_authority()
16996                            .plane
16997                            .templates
16998                            .get(template_id)
16999                            .and_then(|template| {
17000                                self.graph
17001                                    .data_store()
17002                                    .retrieve_ast(template.ast_id, self.graph.sheet_reg())
17003                            }),
17004                        _ => None,
17005                    });
17006                return Some((ast, v));
17007            }
17008            _ => {}
17009        }
17010
17011        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
17012            let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
17013                self.graph
17014                    .data_store()
17015                    .retrieve_ast(ast_id, self.graph.sheet_reg())
17016            });
17017            Some((ast, v))
17018        } else if v.is_some() {
17019            Some((None, v))
17020        } else {
17021            None
17022        }
17023    }
17024
17025    /// Begin batch operations - defer CSR rebuilds for better performance
17026    pub fn begin_batch(&mut self) {
17027        self.graph.begin_batch();
17028    }
17029
17030    /// End batch operations and trigger CSR rebuild
17031    pub fn end_batch(&mut self) {
17032        self.graph.end_batch();
17033    }
17034
17035    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
17036    /// every edit's dirty propagation queues its sources instead of running
17037    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
17038    /// the union with ONE multi-source propagation (O(component) instead of
17039    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
17040    ///
17041    /// Callers MUST run `end_deferred_dirty` on every exit path, including
17042    /// error returns; evaluation entry points `debug_assert` no scope leaked.
17043    pub fn begin_deferred_dirty(&mut self) {
17044        self.graph.begin_deferred_dirty();
17045    }
17046
17047    /// End a deferred-dirty scope, flushing the queued propagation when the
17048    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
17049    pub fn end_deferred_dirty(&mut self) {
17050        let _ = self.graph.end_deferred_dirty();
17051    }
17052
17053    /// Total vertices processed by dirty-propagation BFS loops since graph
17054    /// creation. Perf-shape observability only (cross-crate tests assert
17055    /// batched edits propagate O(component), not O(edits × component)).
17056    pub fn dirty_propagation_visits(&self) -> u64 {
17057        self.graph.dirty_propagation_visits()
17058    }
17059
17060    /// Evaluate a single vertex.
17061    /// This is the core of the sequential evaluation logic for Milestone 3.1.
17062    #[inline]
17063    fn record_cell_if_changed(
17064        delta: &mut DeltaCollector,
17065        cell: &CellRef,
17066        old: &LiteralValue,
17067        new: &LiteralValue,
17068    ) {
17069        if old != new {
17070            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17071        }
17072    }
17073
17074    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17075        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
17076            engine.observe_function_semantic_epoch()?;
17077            // A direct request selects exactly one vertex, regardless of its formula kind.
17078            engine.resource_checkpoint(1)?;
17079            let is_formula = engine.graph.vertex_exists(vertex_id)
17080                && matches!(
17081                    engine.graph.get_vertex_kind(vertex_id),
17082                    VertexKind::FormulaScalar | VertexKind::FormulaArray
17083                );
17084            if is_formula {
17085                engine.begin_evaluation_request();
17086                engine.graph.flush_pending_edge_deltas();
17087                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
17088                    vertex_id,
17089                )];
17090                if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17091                    && engine.graph.formula_authority().active_span_count() > 0
17092                {
17093                    engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
17094                } else {
17095                    engine.evaluate_legacy_target_roots(&roots, None)?;
17096                }
17097            } else if engine.config.formula_plane_mode
17098                == FormulaPlaneMode::AuthoritativeExperimental
17099                && engine.graph.formula_authority().active_span_count() > 0
17100            {
17101                engine.begin_evaluation_request();
17102                engine.graph.flush_pending_edge_deltas();
17103                engine.evaluate_authoritative_formula_plane(None, None)?;
17104            }
17105            engine.evaluate_vertex_impl(vertex_id, None)
17106        })
17107    }
17108
17109    fn evaluate_vertex_impl(
17110        &mut self,
17111        vertex_id: VertexId,
17112        delta: Option<&mut DeltaCollector>,
17113    ) -> Result<LiteralValue, ExcelError> {
17114        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
17115        // names, and other non-formula vertices. Only formula publication needs the C1a final
17116        // deadline checkpoint and effects pipeline.
17117        if !self.graph.vertex_exists(vertex_id) {
17118            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17119                .with_message(format!("Vertex not found: {vertex_id:?}")));
17120        }
17121        if self.active_resource_ledger.is_some()
17122            && matches!(
17123                self.graph.get_vertex_kind(vertex_id),
17124                VertexKind::FormulaScalar | VertexKind::FormulaArray
17125            )
17126        {
17127            let value = self
17128                .evaluate_vertex_immutable(vertex_id)
17129                .unwrap_or_else(LiteralValue::Error);
17130            let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
17131            // Do not publish the selected result until the outer request's deadline succeeds.
17132            self.resource_checkpoint(0)?;
17133            let mut delta = delta;
17134            for effect in &effects {
17135                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
17136            }
17137            return Ok(value);
17138        }
17139
17140        let mut delta = delta;
17141
17142        // Get vertex kind and check if it needs evaluation
17143        let kind = self.graph.get_vertex_kind(vertex_id);
17144        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17145
17146        let ast_id = match kind {
17147            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17148                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
17149                    ast_id
17150                } else {
17151                    return Ok(LiteralValue::Number(0.0));
17152                }
17153            }
17154            VertexKind::Empty | VertexKind::Cell => {
17155                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17156                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17157                    let row = cell_ref.coord.row() + 1;
17158                    let col = cell_ref.coord.col() + 1;
17159                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17160                        return Ok(v);
17161                    }
17162                }
17163                return Ok(LiteralValue::Number(0.0));
17164            }
17165            VertexKind::NamedScalar => {
17166                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
17167                return Ok(value);
17168            }
17169            VertexKind::NamedArray => {
17170                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
17171                return Ok(value);
17172            }
17173            VertexKind::InfiniteRange
17174            | VertexKind::Range
17175            | VertexKind::External
17176            | VertexKind::Table => {
17177                // Not directly evaluatable here.
17178                return Ok(LiteralValue::Number(0.0));
17179            }
17180        };
17181
17182        // The interpreter uses a reference to the engine as the context.
17183        let sheet_name = self.graph.sheet_name(sheet_id);
17184        let cell_ref = self
17185            .graph
17186            .get_cell_ref(vertex_id)
17187            .expect("cell ref for vertex");
17188        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17189
17190        let result =
17191            interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
17192
17193        // If array result, perform spill from the anchor cell
17194        match result {
17195            Ok(cv) => {
17196                let result_literal =
17197                    crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
17198                match result_literal {
17199                    LiteralValue::Array(rows) => {
17200                        // Update kind to FormulaArray for tracking
17201                        self.graph
17202                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
17203                        // Build target cells rectangle starting from anchor
17204                        let anchor = self
17205                            .graph
17206                            .get_cell_ref(vertex_id)
17207                            .expect("cell ref for vertex");
17208                        let sheet_id = anchor.sheet_id;
17209                        let h = rows.len() as u32;
17210                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
17211
17212                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
17213                        let spill_cells = (h as u64).saturating_mul(w as u64);
17214                        if spill_cells > self.config.spill.max_spill_cells as u64 {
17215                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17216                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17217                                .with_message("SpillTooLarge")
17218                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17219                                    expected_rows: h,
17220                                    expected_cols: w,
17221                                });
17222                            let spill_val = LiteralValue::Error(spill_err.clone());
17223                            if let Some(d) = delta.as_deref_mut() {
17224                                let old = self
17225                                    .read_cell_value(
17226                                        self.graph.sheet_name(anchor.sheet_id),
17227                                        anchor.coord.row() + 1,
17228                                        anchor.coord.col() + 1,
17229                                    )
17230                                    .unwrap_or(LiteralValue::Empty);
17231                                if old != spill_val {
17232                                    d.record_cell(
17233                                        anchor.sheet_id,
17234                                        anchor.coord.row(),
17235                                        anchor.coord.col(),
17236                                    );
17237                                }
17238                            }
17239                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
17240                            if self.config.arrow_storage_enabled
17241                                && self.config.delta_overlay_enabled
17242                                && self.config.write_formula_overlay_enabled
17243                            {
17244                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17245                                self.mirror_value_to_computed_overlay(
17246                                    &sheet_name,
17247                                    anchor.coord.row() + 1,
17248                                    anchor.coord.col() + 1,
17249                                    &spill_val,
17250                                );
17251                            }
17252                            return Ok(spill_val);
17253                        }
17254                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
17255                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
17256                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
17257                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
17258                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
17259                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
17260                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17261                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17262                                .with_message("Spill exceeds sheet bounds")
17263                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17264                                    expected_rows: h,
17265                                    expected_cols: w,
17266                                });
17267                            let spill_val = LiteralValue::Error(spill_err.clone());
17268                            if let Some(d) = delta.as_deref_mut() {
17269                                let old = self
17270                                    .read_cell_value(
17271                                        self.graph.sheet_name(anchor.sheet_id),
17272                                        anchor.coord.row() + 1,
17273                                        anchor.coord.col() + 1,
17274                                    )
17275                                    .unwrap_or(LiteralValue::Empty);
17276                                if old != spill_val {
17277                                    d.record_cell(
17278                                        anchor.sheet_id,
17279                                        anchor.coord.row(),
17280                                        anchor.coord.col(),
17281                                    );
17282                                }
17283                            }
17284                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
17285                            if self.config.arrow_storage_enabled
17286                                && self.config.delta_overlay_enabled
17287                                && self.config.write_formula_overlay_enabled
17288                            {
17289                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17290                                self.mirror_value_to_computed_overlay(
17291                                    &sheet_name,
17292                                    anchor.coord.row() + 1,
17293                                    anchor.coord.col() + 1,
17294                                    &spill_val,
17295                                );
17296                            }
17297                            return Ok(spill_val);
17298                        }
17299                        let mut targets = Vec::new();
17300                        for r in 0..h {
17301                            for c in 0..w {
17302                                targets.push(self.graph.make_cell_ref_internal(
17303                                    sheet_id,
17304                                    anchor.coord.row() + r,
17305                                    anchor.coord.col() + c,
17306                                ));
17307                            }
17308                        }
17309
17310                        // Plan spill via spill manager shim
17311                        match self.spill_mgr.reserve(
17312                            vertex_id,
17313                            anchor,
17314                            SpillShape { rows: h, cols: w },
17315                            SpillMeta {
17316                                epoch: self.recalc_epoch,
17317                                config: self.config.spill,
17318                            },
17319                        ) {
17320                            Ok(()) => {
17321                                // Commit: write values to grid
17322                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
17323                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
17324                                if let Err(e) = self.commit_spill_and_mirror(
17325                                    vertex_id,
17326                                    &targets,
17327                                    rows.clone(),
17328                                    delta.as_deref_mut(),
17329                                    None,
17330                                ) {
17331                                    // If commit fails, mark as error
17332                                    self.clear_spill_projection_and_mirror(
17333                                        vertex_id,
17334                                        delta.as_deref_mut(),
17335                                    );
17336                                    if let Some(d) = delta.as_deref_mut() {
17337                                        let old = self
17338                                            .read_cell_value(
17339                                                self.graph.sheet_name(anchor.sheet_id),
17340                                                anchor.coord.row() + 1,
17341                                                anchor.coord.col() + 1,
17342                                            )
17343                                            .unwrap_or(LiteralValue::Empty);
17344                                        let new = LiteralValue::Error(e.clone());
17345                                        if old != new {
17346                                            d.record_cell(
17347                                                anchor.sheet_id,
17348                                                anchor.coord.row(),
17349                                                anchor.coord.col(),
17350                                            );
17351                                        }
17352                                    }
17353                                    let err_val = LiteralValue::Error(e.clone());
17354                                    self.graph.update_vertex_value(vertex_id, err_val.clone());
17355                                    if self.config.arrow_storage_enabled
17356                                        && self.config.delta_overlay_enabled
17357                                        && self.config.write_formula_overlay_enabled
17358                                    {
17359                                        let sheet_name =
17360                                            self.graph.sheet_name(anchor.sheet_id).to_string();
17361                                        self.mirror_value_to_computed_overlay(
17362                                            &sheet_name,
17363                                            anchor.coord.row() + 1,
17364                                            anchor.coord.col() + 1,
17365                                            &err_val,
17366                                        );
17367                                    }
17368                                    return Ok(err_val);
17369                                }
17370                                // Anchor shows the top-left value, like Excel
17371                                let top_left = rows
17372                                    .first()
17373                                    .and_then(|r| r.first())
17374                                    .cloned()
17375                                    .unwrap_or(LiteralValue::Empty);
17376                                self.graph.update_vertex_value(vertex_id, top_left.clone());
17377                                Ok(top_left)
17378                            }
17379                            Err(e) => {
17380                                self.clear_spill_projection_and_mirror(
17381                                    vertex_id,
17382                                    delta.as_deref_mut(),
17383                                );
17384                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17385                                    .with_message(
17386                                        e.message.unwrap_or_else(|| "Spill blocked".to_string()),
17387                                    )
17388                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17389                                        expected_rows: h,
17390                                        expected_cols: w,
17391                                    });
17392                                let spill_val = LiteralValue::Error(spill_err.clone());
17393                                if let Some(d) = delta.as_deref_mut() {
17394                                    let old = self
17395                                        .read_cell_value(
17396                                            self.graph.sheet_name(anchor.sheet_id),
17397                                            anchor.coord.row() + 1,
17398                                            anchor.coord.col() + 1,
17399                                        )
17400                                        .unwrap_or(LiteralValue::Empty);
17401                                    if old != spill_val {
17402                                        d.record_cell(
17403                                            anchor.sheet_id,
17404                                            anchor.coord.row(),
17405                                            anchor.coord.col(),
17406                                        );
17407                                    }
17408                                }
17409                                self.graph.update_vertex_value(vertex_id, spill_val.clone());
17410                                if self.config.arrow_storage_enabled
17411                                    && self.config.delta_overlay_enabled
17412                                    && self.config.write_formula_overlay_enabled
17413                                {
17414                                    let sheet_name =
17415                                        self.graph.sheet_name(anchor.sheet_id).to_string();
17416                                    self.mirror_value_to_computed_overlay(
17417                                        &sheet_name,
17418                                        anchor.coord.row() + 1,
17419                                        anchor.coord.col() + 1,
17420                                        &spill_val,
17421                                    );
17422                                }
17423                                Ok(spill_val)
17424                            }
17425                        }
17426                    }
17427                    other => {
17428                        // Scalar result: store value and ensure any previous spill is cleared
17429                        let spill_cells = self
17430                            .graph
17431                            .spill_cells_for_anchor(vertex_id)
17432                            .map(|cells| cells.to_vec())
17433                            .unwrap_or_default();
17434                        if let Some(d) = delta.as_deref_mut()
17435                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17436                        {
17437                            if spill_cells.is_empty() {
17438                                let old = self
17439                                    .read_cell_value(
17440                                        self.graph.sheet_name(anchor.sheet_id),
17441                                        anchor.coord.row() + 1,
17442                                        anchor.coord.col() + 1,
17443                                    )
17444                                    .unwrap_or(LiteralValue::Empty);
17445                                if old != other {
17446                                    d.record_cell(
17447                                        anchor.sheet_id,
17448                                        anchor.coord.row(),
17449                                        anchor.coord.col(),
17450                                    );
17451                                }
17452                            } else {
17453                                for cell in spill_cells.iter() {
17454                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
17455                                    let old = self
17456                                        .get_cell_value(
17457                                            sheet_name,
17458                                            cell.coord.row() + 1,
17459                                            cell.coord.col() + 1,
17460                                        )
17461                                        .unwrap_or(LiteralValue::Empty);
17462                                    let new = if cell.sheet_id == anchor.sheet_id
17463                                        && cell.coord.row() == anchor.coord.row()
17464                                        && cell.coord.col() == anchor.coord.col()
17465                                    {
17466                                        other.clone()
17467                                    } else {
17468                                        LiteralValue::Empty
17469                                    };
17470                                    Self::record_cell_if_changed(d, cell, &old, &new);
17471                                }
17472                            }
17473                        }
17474                        self.graph.clear_spill_region(vertex_id);
17475                        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17476                            self.record_formula_plane_structural_change(scope);
17477                        }
17478                        if self.config.arrow_storage_enabled
17479                            && self.config.delta_overlay_enabled
17480                            && self.config.write_formula_overlay_enabled
17481                        {
17482                            let empty = LiteralValue::Empty;
17483                            for cell in spill_cells.iter() {
17484                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17485                                self.mirror_value_to_computed_overlay(
17486                                    &sheet_name,
17487                                    cell.coord.row() + 1,
17488                                    cell.coord.col() + 1,
17489                                    &empty,
17490                                );
17491                            }
17492                        }
17493                        self.graph.update_vertex_value(vertex_id, other.clone());
17494                        // Optionally mirror into Arrow overlay for Arrow-backed reads
17495                        if self.config.arrow_storage_enabled
17496                            && self.config.delta_overlay_enabled
17497                            && self.config.write_formula_overlay_enabled
17498                        {
17499                            let anchor = self
17500                                .graph
17501                                .get_cell_ref(vertex_id)
17502                                .expect("cell ref for vertex");
17503                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17504                            self.mirror_value_to_computed_overlay(
17505                                &sheet_name,
17506                                anchor.coord.row() + 1,
17507                                anchor.coord.col() + 1,
17508                                &other,
17509                            );
17510                        }
17511                        Ok(other)
17512                    }
17513                }
17514            }
17515            Err(e) => {
17516                // Runtime Excel error: store as a cell value instead of propagating
17517                // as an exception so bulk eval paths don't fail the whole pass.
17518                let spill_cells = self
17519                    .graph
17520                    .spill_cells_for_anchor(vertex_id)
17521                    .map(|cells| cells.to_vec())
17522                    .unwrap_or_default();
17523                let err_val = LiteralValue::Error(e.clone());
17524                if let Some(d) = delta
17525                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17526                {
17527                    if spill_cells.is_empty() {
17528                        let old = self
17529                            .read_cell_value(
17530                                self.graph.sheet_name(anchor.sheet_id),
17531                                anchor.coord.row() + 1,
17532                                anchor.coord.col() + 1,
17533                            )
17534                            .unwrap_or(LiteralValue::Empty);
17535                        if old != err_val {
17536                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
17537                        }
17538                    } else {
17539                        for cell in spill_cells.iter() {
17540                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
17541                            let old = self
17542                                .get_cell_value(
17543                                    sheet_name,
17544                                    cell.coord.row() + 1,
17545                                    cell.coord.col() + 1,
17546                                )
17547                                .unwrap_or(LiteralValue::Empty);
17548                            let new = if cell.sheet_id == anchor.sheet_id
17549                                && cell.coord.row() == anchor.coord.row()
17550                                && cell.coord.col() == anchor.coord.col()
17551                            {
17552                                err_val.clone()
17553                            } else {
17554                                LiteralValue::Empty
17555                            };
17556                            Self::record_cell_if_changed(d, cell, &old, &new);
17557                        }
17558                    }
17559                }
17560                self.graph.clear_spill_region(vertex_id);
17561                if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17562                    self.record_formula_plane_structural_change(scope);
17563                }
17564                if self.config.arrow_storage_enabled
17565                    && self.config.delta_overlay_enabled
17566                    && self.config.write_formula_overlay_enabled
17567                {
17568                    let empty = LiteralValue::Empty;
17569                    for cell in spill_cells.iter() {
17570                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17571                        self.mirror_value_to_computed_overlay(
17572                            &sheet_name,
17573                            cell.coord.row() + 1,
17574                            cell.coord.col() + 1,
17575                            &empty,
17576                        );
17577                    }
17578                }
17579                self.graph.update_vertex_value(vertex_id, err_val.clone());
17580                if self.config.arrow_storage_enabled
17581                    && self.config.delta_overlay_enabled
17582                    && self.config.write_formula_overlay_enabled
17583                {
17584                    let anchor = self
17585                        .graph
17586                        .get_cell_ref(vertex_id)
17587                        .expect("cell ref for vertex");
17588                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17589                    self.mirror_value_to_computed_overlay(
17590                        &sheet_name,
17591                        anchor.coord.row() + 1,
17592                        anchor.coord.col() + 1,
17593                        &err_val,
17594                    );
17595                }
17596                Ok(err_val)
17597            }
17598        }
17599    }
17600
17601    fn evaluate_named_scalar(
17602        &mut self,
17603        vertex_id: VertexId,
17604        sheet_id: SheetId,
17605    ) -> Result<LiteralValue, ExcelError> {
17606        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17607            ExcelError::new(ExcelErrorKind::Name)
17608                .with_message("Named range metadata missing".to_string())
17609        })?;
17610
17611        match &named_range.definition {
17612            NamedDefinition::Cell(cell_ref) => {
17613                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17614                let row = cell_ref.coord.row() + 1;
17615                let col = cell_ref.coord.col() + 1;
17616
17617                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
17618                    && matches!(
17619                        self.graph.get_vertex_kind(dep_vertex),
17620                        VertexKind::FormulaScalar | VertexKind::FormulaArray
17621                    )
17622                {
17623                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
17624                    let value = self.evaluate_vertex(dep_vertex)?;
17625                    self.graph.update_vertex_value(vertex_id, value.clone());
17626                    Ok(value)
17627                } else {
17628                    let value = self
17629                        .get_cell_value(sheet_name, row, col)
17630                        .unwrap_or(LiteralValue::Empty);
17631                    self.graph.update_vertex_value(vertex_id, value.clone());
17632                    Ok(value)
17633                }
17634            }
17635            NamedDefinition::Literal(v) => {
17636                let out = v.clone();
17637                self.graph.update_vertex_value(vertex_id, out.clone());
17638                Ok(out)
17639            }
17640            NamedDefinition::Formula { ast, .. } => {
17641                let context_sheet = match named_range.scope {
17642                    NameScope::Sheet(id) => id,
17643                    NameScope::Workbook => sheet_id,
17644                };
17645                let sheet_name = self.graph.sheet_name(context_sheet);
17646                let cell_ref = self
17647                    .graph
17648                    .get_cell_ref(vertex_id)
17649                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17650                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17651                match interpreter.evaluate_ast(ast) {
17652                    Ok(cv) => {
17653                        let value = cv.into_literal();
17654                        match value {
17655                            LiteralValue::Array(_) => {
17656                                let err = ExcelError::new(ExcelErrorKind::NImpl)
17657                                    .with_message("Array result in scalar named range".to_string());
17658                                let err_val = LiteralValue::Error(err.clone());
17659                                self.graph.update_vertex_value(vertex_id, err_val.clone());
17660                                Ok(err_val)
17661                            }
17662                            other => {
17663                                self.graph.update_vertex_value(vertex_id, other.clone());
17664                                Ok(other)
17665                            }
17666                        }
17667                    }
17668                    Err(err) => {
17669                        let err_val = LiteralValue::Error(err.clone());
17670                        self.graph.update_vertex_value(vertex_id, err_val.clone());
17671                        Ok(err_val)
17672                    }
17673                }
17674            }
17675            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17676                .with_message("Range-valued name evaluated as scalar".to_string())),
17677        }
17678    }
17679
17680    fn evaluate_named_array(
17681        &mut self,
17682        vertex_id: VertexId,
17683        sheet_id: SheetId,
17684    ) -> Result<LiteralValue, ExcelError> {
17685        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17686            ExcelError::new(ExcelErrorKind::Name)
17687                .with_message("Named range metadata missing".to_string())
17688        })?;
17689
17690        let out = match &named_range.definition {
17691            NamedDefinition::Range(range_ref) => {
17692                if range_ref.start.sheet_id != range_ref.end.sheet_id {
17693                    return Err(ExcelError::new(ExcelErrorKind::Ref)
17694                        .with_message("Named range cannot span sheets".to_string()));
17695                }
17696
17697                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17698                let sr0 = range_ref.start.coord.row();
17699                let sc0 = range_ref.start.coord.col();
17700                let er0 = range_ref.end.coord.row();
17701                let ec0 = range_ref.end.coord.col();
17702                if sr0 > er0 || sc0 > ec0 {
17703                    return Err(ExcelError::new(ExcelErrorKind::Ref)
17704                        .with_message("Invalid named range bounds".to_string()));
17705                }
17706
17707                let h = (er0 - sr0 + 1) as usize;
17708                let w = (ec0 - sc0 + 1) as usize;
17709                let cell_count = (h as u64).saturating_mul(w as u64);
17710                if cell_count > self.config.spill.max_spill_cells as u64 {
17711                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17712                        "Named range too large to materialize as an array".to_string(),
17713                    ));
17714                }
17715
17716                let mut rows = Vec::with_capacity(h);
17717                for r0 in sr0..=er0 {
17718                    let mut row = Vec::with_capacity(w);
17719                    for c0 in sc0..=ec0 {
17720                        let v = self
17721                            .get_cell_value(sheet_name, r0 + 1, c0 + 1)
17722                            .unwrap_or(LiteralValue::Empty);
17723                        row.push(v);
17724                    }
17725                    rows.push(row);
17726                }
17727                LiteralValue::Array(rows)
17728            }
17729            NamedDefinition::Cell(cell_ref) => {
17730                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17731                let row = cell_ref.coord.row() + 1;
17732                let col = cell_ref.coord.col() + 1;
17733                let v = self
17734                    .get_cell_value(sheet_name, row, col)
17735                    .unwrap_or(LiteralValue::Empty);
17736                LiteralValue::Array(vec![vec![v]])
17737            }
17738            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
17739            NamedDefinition::Formula { ast, .. } => {
17740                let context_sheet = match named_range.scope {
17741                    NameScope::Sheet(id) => id,
17742                    NameScope::Workbook => sheet_id,
17743                };
17744                let sheet_name = self.graph.sheet_name(context_sheet);
17745                let cell_ref = self
17746                    .graph
17747                    .get_cell_ref(vertex_id)
17748                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17749                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17750                match interpreter.evaluate_ast(ast) {
17751                    Ok(cv) => {
17752                        let v = cv.into_literal();
17753                        match v {
17754                            LiteralValue::Array(_) => v,
17755                            other => LiteralValue::Array(vec![vec![other]]),
17756                        }
17757                    }
17758                    Err(err) => LiteralValue::Error(err),
17759                }
17760            }
17761        };
17762
17763        self.graph.update_vertex_value(vertex_id, out.clone());
17764        Ok(out)
17765    }
17766
17767    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
17768        crate::engine::ResourceLedgerError::Exhausted(
17769            formualizer_common::ResourceExhaustionDetail {
17770                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
17771                limit: limit as u64,
17772                observed: limit.saturating_add(1) as u64,
17773                request_id: self
17774                    .active_evaluation_resource_request
17775                    .as_ref()
17776                    .map(|stats| stats.request_id),
17777            },
17778        )
17779        .into_excel_error()
17780        .with_message(format!("{context} did not converge after {limit} replans"))
17781    }
17782
17783    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
17784        matches!(
17785            &error.extra,
17786            formualizer_common::ExcelErrorExtra::PreparationStale {
17787                reason: formualizer_common::PreparationStaleReason::Semantic
17788                    | formualizer_common::PreparationStaleReason::Provider
17789            }
17790        )
17791    }
17792
17793    fn prepare_graph_for_routed_evaluation(
17794        &mut self,
17795        targets: &[crate::engine::EvaluationTarget],
17796        options: &crate::engine::TargetEvalOptions<'_>,
17797    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
17798        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
17799        let mut retries = 0usize;
17800        loop {
17801            match self.prepare_graph_for_targets_unobserved(targets, options) {
17802                Err(error)
17803                    if Self::transient_target_preparation_stale(&error)
17804                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
17805                {
17806                    retries = retries.saturating_add(1);
17807                }
17808                result => return result,
17809            }
17810        }
17811    }
17812
17813    fn evaluate_mixed_targets(
17814        &mut self,
17815        targets: &[crate::engine::EvaluationTarget],
17816        delta: Option<&mut DeltaCollector>,
17817    ) -> Result<EvalResult, ExcelError> {
17818        let _source_cache = self.source_cache_session();
17819        let cancel = self.active_cancel_flag.clone();
17820        let options = crate::engine::TargetEvalOptions {
17821            request_id: self
17822                .active_evaluation_resource_request
17823                .as_ref()
17824                .map(|stats| stats.request_id),
17825            cancel,
17826            deadline: None,
17827            budgets: None,
17828            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
17829        };
17830        self.prepare_and_execute_target_recipe(targets, &options, delta)
17831    }
17832
17833    fn prepare_and_execute_target_recipe(
17834        &mut self,
17835        targets: &[crate::engine::EvaluationTarget],
17836        options: &crate::engine::TargetEvalOptions<'_>,
17837        delta: Option<&mut DeltaCollector>,
17838    ) -> Result<EvalResult, ExcelError> {
17839        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
17840        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
17841    }
17842
17843    fn execute_prepared_target_recipe(
17844        &mut self,
17845        targets: &[crate::engine::EvaluationTarget],
17846        scope: &crate::engine::PrepareScope,
17847        delta: Option<&mut DeltaCollector>,
17848    ) -> Result<EvalResult, ExcelError> {
17849        if matches!(scope, crate::engine::PrepareScope::Workbook)
17850            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
17851        {
17852            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
17853        }
17854        let mut roots = self.resolve_target_producers(targets)?;
17855        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
17856            let request_id = self
17857                .active_evaluation_resource_request
17858                .as_ref()
17859                .map(|request| request.request_id);
17860            let root_count = roots.len();
17861            let mut widened_roots =
17862                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
17863                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
17864            let sheet_ids = sheets
17865                .iter()
17866                .filter_map(|sheet| self.graph.sheet_id(sheet))
17867                .collect::<FxHashSet<_>>();
17868            for vertex in self.graph.formula_vertices() {
17869                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
17870                    widened_roots
17871                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
17872                            vertex,
17873                        ))
17874                        .map_err(|_| {
17875                            target_root_allocation_error(widened_roots.len() + 1, request_id)
17876                        })?;
17877                }
17878            }
17879            let authority = self.graph.formula_authority();
17880            for span_ref in authority.active_span_refs() {
17881                let Some(span) = authority.plane.spans.get(span_ref) else {
17882                    continue;
17883                };
17884                if sheet_ids.contains(&span.sheet_id) {
17885                    widened_roots
17886                        .push(crate::engine::target_preparation::TargetProducer::Span {
17887                            span_ref,
17888                            demanded: Region::from_domain(span.result_region.domain()),
17889                        })
17890                        .map_err(|_| {
17891                            target_root_allocation_error(widened_roots.len() + 1, request_id)
17892                        })?;
17893                }
17894            }
17895            roots = widened_roots.into_vec();
17896        }
17897        self.begin_evaluation_request();
17898        self.graph.flush_pending_edge_deltas();
17899        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
17900        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17901            && self.graph.formula_authority().active_span_count() > 0
17902        {
17903            if workbook_scope {
17904                self.evaluate_authoritative_formula_plane(None, delta)
17905            } else {
17906                self.evaluate_authoritative_formula_plane_targets(&roots, delta)
17907            }
17908        } else {
17909            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
17910                self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
17911            }
17912            if workbook_scope {
17913                if let Some(delta) = delta {
17914                    self.evaluate_all_with_delta_collector(delta)
17915                } else {
17916                    self.evaluate_all_legacy_impl()
17917                }
17918            } else {
17919                self.evaluate_legacy_target_roots(&roots, delta)
17920            }
17921        }
17922    }
17923
17924    fn legacy_coordinate_targets(
17925        &mut self,
17926        targets: &[(&str, u32, u32)],
17927    ) -> Vec<crate::engine::EvaluationTarget> {
17928        targets
17929            .iter()
17930            .map(|(sheet, row, col)| {
17931                // Compatibility APIs historically interned an unknown target sheet
17932                // and returned an empty value rather than rejecting the target.
17933                self.graph.sheet_id_mut(sheet);
17934                crate::engine::EvaluationTarget::Cell {
17935                    sheet: (*sheet).to_string(),
17936                    row: *row,
17937                    col: *col,
17938                }
17939            })
17940            .collect()
17941    }
17942
17943    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
17944    pub fn evaluate_targets(
17945        &mut self,
17946        targets: &[crate::engine::EvaluationTarget],
17947    ) -> Result<EvalResult, ExcelError> {
17948        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17949            engine.observe_function_semantic_epoch()?;
17950            engine.validate_deterministic_mode()?;
17951            engine.evaluate_mixed_targets(targets, None)
17952        })
17953    }
17954
17955    /// Evaluate typed targets with explicit preparation policy and request controls.
17956    pub fn evaluate_targets_with_options(
17957        &mut self,
17958        targets: &[crate::engine::EvaluationTarget],
17959        options: crate::engine::TargetEvalOptions<'_>,
17960    ) -> Result<EvalResult, ExcelError> {
17961        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17962            engine.active_cancel_flag = options.cancel.clone();
17963            engine.active_evaluation_deadline = options.deadline;
17964            let result = (|| {
17965                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
17966                engine.observe_function_semantic_epoch()?;
17967                engine.validate_deterministic_mode()?;
17968                let _source_cache = engine.source_cache_session();
17969                engine.prepare_and_execute_target_recipe(targets, &options, None)
17970            })();
17971            engine.active_cancel_flag = None;
17972            engine.active_evaluation_deadline = None;
17973            result
17974        })
17975    }
17976
17977    /// Evaluate typed targets and return the versioned run/region delta for the request.
17978    pub fn evaluate_targets_with_delta(
17979        &mut self,
17980        targets: &[crate::engine::EvaluationTarget],
17981    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
17982        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
17983            engine.observe_function_semantic_epoch()?;
17984            engine.validate_deterministic_mode()?;
17985            let mut collector = DeltaCollector::new(DeltaMode::Cells);
17986            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
17987            Ok((result, collector.finish_target()))
17988        })
17989    }
17990
17991    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
17992    pub fn evaluate_until(
17993        &mut self,
17994        targets: &[(&str, u32, u32)],
17995    ) -> Result<EvalResult, ExcelError> {
17996        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
17997            engine.evaluate_until_unobserved(targets)
17998        })
17999    }
18000
18001    fn evaluate_until_unobserved(
18002        &mut self,
18003        targets: &[(&str, u32, u32)],
18004    ) -> Result<EvalResult, ExcelError> {
18005        self.observe_function_semantic_epoch()?;
18006        let targets = self.legacy_coordinate_targets(targets);
18007        self.evaluate_mixed_targets(&targets, None)
18008    }
18009
18010    fn evaluate_until_with_delta_collector(
18011        &mut self,
18012        targets: &[(&str, u32, u32)],
18013        delta: &mut DeltaCollector,
18014    ) -> Result<EvalResult, ExcelError> {
18015        let targets = self.legacy_coordinate_targets(targets);
18016        self.evaluate_mixed_targets(&targets, Some(delta))
18017    }
18018
18019    fn evaluate_legacy_target_roots(
18020        &mut self,
18021        roots: &[crate::engine::target_preparation::TargetProducer],
18022        mut delta: Option<&mut DeltaCollector>,
18023    ) -> Result<EvalResult, ExcelError> {
18024        use crate::engine::target_preparation::TargetProducer;
18025        let start = crate::instant::FzInstant::now();
18026        let root_vertices = roots
18027            .iter()
18028            .filter_map(|root| match root {
18029                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
18030                TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
18031            })
18032            .collect::<Vec<_>>();
18033        let mut computed_vertices = 0usize;
18034        let mut cycle_errors = 0usize;
18035        let mut replans = 0usize;
18036        const MAX_REPLAN: usize = 5;
18037        loop {
18038            let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
18039            if precedents_to_eval.is_empty() {
18040                break;
18041            }
18042            let scheduler = Scheduler::new(&self.graph);
18043            let schedule =
18044                scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
18045            for &unit in &schedule.units {
18046                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
18047                match unit {
18048                    ScheduleUnit::Cycle(index) => {
18049                        if self.handle_cycle_unit(
18050                            schedule.unit_cycle(index),
18051                            delta.as_deref_mut(),
18052                            None,
18053                            None,
18054                        )? > 0
18055                        {
18056                            cycle_errors = cycle_errors.saturating_add(1);
18057                        }
18058                    }
18059                    ScheduleUnit::Layer(index) => {
18060                        let layer = schedule.unit_layer(index);
18061                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
18062                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18063                                self.evaluate_layer_parallel_with_delta(layer, delta)?
18064                            } else {
18065                                self.evaluate_layer_sequential_with_delta(layer, delta)?
18066                            }
18067                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18068                            self.evaluate_layer_parallel(layer)?
18069                        } else {
18070                            self.evaluate_layer_sequential(layer)?
18071                        };
18072                        computed_vertices = computed_vertices.saturating_add(evaluated);
18073                    }
18074                }
18075            }
18076            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
18077            self.resource_checkpoint(0)?;
18078            self.graph.clear_dirty_flags(&precedents_to_eval);
18079            for vertex in &changed {
18080                self.graph.set_dirty(*vertex, true);
18081            }
18082            if changed.is_empty() {
18083                break;
18084            }
18085            if replans >= MAX_REPLAN {
18086                return Err(self.replan_exhausted_error(
18087                    MAX_REPLAN,
18088                    "targeted legacy dynamic dependency evaluation",
18089                ));
18090            }
18091            replans = replans.saturating_add(1);
18092        }
18093        self.redirty_for_next_recalc();
18094        Ok(EvalResult {
18095            computed_vertices,
18096            cycle_errors,
18097            elapsed: start.elapsed(),
18098        })
18099    }
18100
18101    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
18102    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
18103        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
18104            return Err(
18105                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
18106                    "compatibility recalculation plans require all staged formulas to be prepared",
18107                ),
18108            );
18109        }
18110        let key = self.recalc_plan_key();
18111        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
18112        vertices.sort_unstable();
18113        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
18114        let schedule = if vertices.is_empty() {
18115            crate::engine::Schedule {
18116                units: Vec::new(),
18117                layers: Vec::new(),
18118                cycles: Vec::new(),
18119            }
18120        } else {
18121            self.create_evaluation_schedule_uncached(&vertices)?.0
18122        };
18123        self.validate_recalc_plan_key(&key)?;
18124        Ok(RecalcPlan {
18125            key,
18126            kind: RecalcPlanKind::CompatibilityFull {
18127                schedule,
18128                has_dynamic_refs,
18129            },
18130        })
18131    }
18132
18133    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
18134    pub fn build_recalc_plan_for_targets(
18135        &mut self,
18136        targets: &[crate::engine::EvaluationTarget],
18137    ) -> Result<RecalcPlan, ExcelError> {
18138        self.build_recalc_plan_for_targets_with_options(
18139            targets,
18140            crate::engine::TargetEvalOptions::default(),
18141        )
18142    }
18143
18144    pub fn build_recalc_plan_for_targets_with_options(
18145        &mut self,
18146        targets: &[crate::engine::EvaluationTarget],
18147        options: crate::engine::TargetEvalOptions<'_>,
18148    ) -> Result<RecalcPlan, ExcelError> {
18149        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18150            engine.observe_function_semantic_epoch()?;
18151            engine.validate_deterministic_mode()?;
18152            let _source_cache = engine.source_cache_session();
18153            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
18154            engine.graph.flush_pending_edge_deltas();
18155            let topology = if matches!(
18156                preparation.widened_scope,
18157                crate::engine::PrepareScope::Workbook
18158            ) {
18159                RecalcTopology::Workbook
18160            } else {
18161                RecalcTopology::RunLocalRecipe
18162            };
18163            Ok(RecalcPlan {
18164                key: engine.recalc_plan_key(),
18165                kind: RecalcPlanKind::Target {
18166                    targets: targets.to_vec(),
18167                    scope: preparation.widened_scope,
18168                    topology,
18169                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
18170                },
18171            })
18172        })
18173    }
18174
18175    /// Evaluate using a previously constructed compatibility or target plan.
18176    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
18177        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18178            engine.evaluate_recalc_plan_unobserved(plan)
18179        })
18180    }
18181
18182    pub fn evaluate_recalc_plan_with_controls(
18183        &mut self,
18184        plan: &RecalcPlan,
18185        cancel: Option<crate::engine::CancelToken>,
18186        deadline: Option<Instant>,
18187    ) -> Result<EvalResult, ExcelError> {
18188        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18189            engine.active_cancel_flag = cancel.clone();
18190            engine.active_evaluation_deadline = deadline;
18191            let result = engine.evaluate_recalc_plan_unobserved(plan);
18192            engine.active_cancel_flag = None;
18193            engine.active_evaluation_deadline = None;
18194            result
18195        })
18196    }
18197
18198    fn evaluate_recalc_plan_unobserved(
18199        &mut self,
18200        plan: &RecalcPlan,
18201    ) -> Result<EvalResult, ExcelError> {
18202        self.graph.flush_pending_edge_deltas();
18203        self.validate_recalc_plan_key(&plan.key)?;
18204        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
18205        self.validate_deterministic_mode()?;
18206
18207        match &plan.kind {
18208            RecalcPlanKind::Target {
18209                targets,
18210                scope,
18211                topology,
18212                dynamic_policy,
18213            } => {
18214                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
18215                debug_assert_eq!(
18216                    matches!(topology, RecalcTopology::Workbook),
18217                    matches!(scope, crate::engine::PrepareScope::Workbook)
18218                );
18219                let _source_cache = self.source_cache_session();
18220                self.execute_prepared_target_recipe(targets, scope, None)
18221            }
18222            RecalcPlanKind::CompatibilityFull {
18223                schedule,
18224                has_dynamic_refs,
18225            } => {
18226                let _source_cache = self.source_cache_session();
18227                self.begin_evaluation_request();
18228                if self.graph.formula_authority().active_span_count() > 0 {
18229                    return self.evaluate_authoritative_formula_plane_all();
18230                }
18231                if *has_dynamic_refs {
18232                    self.virtual_dep_fallback_activations =
18233                        self.virtual_dep_fallback_activations.saturating_add(1);
18234                    return self.evaluate_all_coordinator();
18235                }
18236
18237                let start = crate::instant::FzInstant::now();
18238                let dirty_vertices = self.graph.get_evaluation_vertices();
18239                if dirty_vertices.is_empty() {
18240                    return Ok(EvalResult {
18241                        computed_vertices: 0,
18242                        cycle_errors: 0,
18243                        elapsed: start.elapsed(),
18244                    });
18245                }
18246
18247                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
18248                let mut computed_vertices = 0;
18249                let mut cycle_errors = 0;
18250                for &unit in &schedule.units {
18251                    self.cancellation_checkpoint(
18252                        "Evaluation cancelled before recalculation plan schedule unit",
18253                    )?;
18254                    match unit {
18255                        ScheduleUnit::Cycle(i) => {
18256                            let stamped = self.handle_cycle_unit(
18257                                schedule.unit_cycle(i),
18258                                None,
18259                                Some(&dirty_set),
18260                                None,
18261                            )?;
18262                            if stamped > 0 {
18263                                cycle_errors += 1;
18264                            }
18265                        }
18266                        ScheduleUnit::Layer(i) => {
18267                            let work: Vec<VertexId> = schedule
18268                                .unit_layer(i)
18269                                .vertices
18270                                .iter()
18271                                .copied()
18272                                .filter(|v| dirty_set.contains(v))
18273                                .collect();
18274                            if work.is_empty() {
18275                                continue;
18276                            }
18277                            let temp_layer = crate::engine::scheduler::Layer { vertices: work };
18278                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
18279                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
18280                            } else {
18281                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
18282                            }
18283                        }
18284                    }
18285                }
18286
18287                self.resource_checkpoint(0)?;
18288                self.graph.clear_dirty_flags(&dirty_vertices);
18289                self.redirty_for_next_recalc();
18290                Ok(EvalResult {
18291                    computed_vertices,
18292                    cycle_errors,
18293                    elapsed: start.elapsed(),
18294                })
18295            }
18296        }
18297    }
18298    fn evaluate_all_legacy_and_ack_dirty(
18299        &mut self,
18300        formula_dirty: FormulaDirtyLease,
18301    ) -> Result<EvalResult, ExcelError> {
18302        let result = self.evaluate_all_legacy_impl()?;
18303        self.checked_ack_formula_dirty_observed(formula_dirty)?;
18304        Ok(result)
18305    }
18306
18307    fn evaluate_formula_plane_capacity_fallback(
18308        &mut self,
18309        mut formula_dirty: FormulaDirtyLease,
18310        owned_dirty_events: &[usize],
18311        selected_span_refs: &[FormulaSpanRef],
18312        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18313        delta: Option<&mut DeltaCollector>,
18314    ) -> Result<EvalResult, ExcelError> {
18315        let leased_len = formula_dirty.len();
18316        if !selected_span_refs.is_empty() {
18317            let materialization_started = crate::instant::FzInstant::now();
18318            let prepared = self
18319                .prepare_formula_span_demotion(selected_span_refs)
18320                .map_err(|error| {
18321                    if let FormulaSpanDemotionError::Resource(error) = error {
18322                        error
18323                    } else {
18324                        ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18325                            "FormulaPlane capacity fallback demotion preparation failed: {error}"
18326                        ))
18327                    }
18328                })?;
18329            let report = self
18330                .commit_prepared_formula_span_demotion(prepared)
18331                .map_err(|error| {
18332                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18333                        "FormulaPlane capacity fallback demotion commit failed: {error}"
18334                    ))
18335                })?;
18336            self.observe_materialization(
18337                report.placements_materialized,
18338                false,
18339                materialization_started.elapsed(),
18340            );
18341        }
18342        #[cfg(test)]
18343        {
18344            self.last_formula_plane_span_eval_report = None;
18345        }
18346        if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
18347            formula_dirty = extended;
18348        }
18349        let mut acknowledged = owned_dirty_events.to_vec();
18350        acknowledged.extend(leased_len..formula_dirty.len());
18351        let result = if let Some(target_roots) = target_roots {
18352            let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
18353            self.evaluate_legacy_target_roots(&refreshed, delta)?
18354        } else if let Some(delta) = delta {
18355            self.evaluate_all_with_delta_collector(delta)?
18356        } else {
18357            self.evaluate_all_legacy_impl()?
18358        };
18359        self.formula_plane_capacity_bailouts =
18360            self.formula_plane_capacity_bailouts.saturating_add(1);
18361        if target_roots.is_some() {
18362            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
18363        } else {
18364            self.checked_ack_formula_dirty_observed(formula_dirty)?;
18365        }
18366        Ok(result)
18367    }
18368
18369    fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
18370        self.evaluate_authoritative_formula_plane(None, None)
18371    }
18372
18373    fn evaluate_authoritative_formula_plane_targets(
18374        &mut self,
18375        roots: &[crate::engine::target_preparation::TargetProducer],
18376        delta: Option<&mut DeltaCollector>,
18377    ) -> Result<EvalResult, ExcelError> {
18378        self.evaluate_authoritative_formula_plane(Some(roots), delta)
18379    }
18380
18381    fn evaluate_authoritative_formula_plane(
18382        &mut self,
18383        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18384        mut delta: Option<&mut DeltaCollector>,
18385    ) -> Result<EvalResult, ExcelError> {
18386        // The public/request coordinators begin exactly once. Every primitive
18387        // below is deliberately non-beginning and non-finalising so the dirty
18388        // lease, iterative accumulator and §7.11 clock sample have one owner.
18389        let mut formula_dirty = self.graph.lease_formula_dirty();
18390        self.observe_dirty_lease_acquired(formula_dirty.is_empty());
18391
18392        // SingletonUnique formulas intentionally remain legacy graph vertices;
18393        // when no spans are active, execute through the private legacy primitive.
18394        if self.graph.formula_authority().active_span_count() == 0 {
18395            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
18396            #[cfg(test)]
18397            {
18398                self.last_formula_plane_span_eval_report = None;
18399            }
18400            if let Some(target_roots) = target_roots {
18401                let result =
18402                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18403                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18404                return Ok(result);
18405            }
18406            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18407        }
18408
18409        // With no graph-owned span/region dirtiness, only sparse legacy work
18410        // (including volatiles) can remain. Preserve the warm no-op fast path.
18411        if formula_dirty.is_empty() {
18412            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
18413            #[cfg(test)]
18414            {
18415                self.last_formula_plane_span_eval_report = None;
18416            }
18417            if let Some(target_roots) = target_roots {
18418                let result =
18419                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18420                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18421                return Ok(result);
18422            }
18423            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18424        }
18425
18426        let start = crate::instant::FzInstant::now();
18427        let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
18428        let mut workbook_exact_attempted = target_roots.is_none();
18429        // #CIRC stamps produced by demoting cyclic spans and resolving the
18430        // residual legacy-only cycle ahead of the mixed schedule (gotcha G8).
18431        let mut prepass_cycle_errors = 0usize;
18432        const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
18433        let mut cycle_demote_iters = 0usize;
18434        let mut include_dirty_regions = true;
18435        let mut retry_whole_spans = Vec::new();
18436        let mut computed_vertices = 0usize;
18437        let mut runtime_replan_iterations = 0usize;
18438        const MAX_RUNTIME_REPLAN: usize = 5;
18439        let mut virtual_telemetry = self
18440            .config
18441            .enable_virtual_dep_telemetry
18442            .then(|| self.start_virtual_dep_telemetry());
18443        'virtual_replan: loop {
18444            self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
18445            let (
18446                schedule,
18447                span_refs_by_id,
18448                plane_epoch,
18449                legacy_vertices,
18450                owned_dirty_events,
18451                island_plan,
18452            ) = loop {
18453                let (
18454                    schedule,
18455                    span_refs_by_id,
18456                    plane_epoch,
18457                    legacy_vertices,
18458                    owned_dirty_events,
18459                    island_plan,
18460                ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
18461                    self.build_formula_plane_target_schedule(
18462                        &formula_dirty,
18463                        &retry_whole_spans,
18464                        include_dirty_regions,
18465                        target_roots,
18466                    )?
18467                } else {
18468                    self.build_formula_plane_mixed_schedule(
18469                        &formula_dirty,
18470                        &retry_whole_spans,
18471                        include_dirty_regions,
18472                    )?
18473                };
18474                #[cfg(test)]
18475                let mut schedule = schedule;
18476                #[cfg(test)]
18477                if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
18478                    let producer = schedule
18479                        .layers
18480                        .iter()
18481                        .flat_map(|layer| layer.work.iter())
18482                        .map(|work| work.producer)
18483                        .next()
18484                        .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
18485                    schedule.fallbacks.push(
18486                        crate::formula_plane::scheduler::MixedScheduleFallback {
18487                            producer,
18488                            reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
18489                        },
18490                    );
18491                }
18492
18493                if schedule.is_authoritative_safe() {
18494                    break (
18495                        schedule,
18496                        span_refs_by_id,
18497                        plane_epoch,
18498                        legacy_vertices,
18499                        owned_dirty_events,
18500                        island_plan,
18501                    );
18502                }
18503
18504                // Non-cycle unsafe schedules cannot safely run legacy-only while a
18505                // scheduled span remains virtual. Refine exactly every span in this
18506                // request's scheduled work, then complete the request in one legacy
18507                // pass. Clean, unscheduled spans keep their authority and overlays.
18508                let has_cycle_fallback = schedule.stats.cycle_count > 0
18509                    || schedule
18510                        .fallbacks
18511                        .iter()
18512                        .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
18513                if !has_cycle_fallback {
18514                    let selected_span_refs = schedule
18515                        .layers
18516                        .iter()
18517                        .flat_map(|layer| layer.work.iter())
18518                        .filter_map(|work| match work.producer {
18519                            FormulaProducerId::Span(span_id) => {
18520                                span_refs_by_id.get(&span_id).copied()
18521                            }
18522                            FormulaProducerId::Legacy(_) => None,
18523                        })
18524                        .collect::<Vec<_>>();
18525
18526                    return self.evaluate_formula_plane_capacity_fallback(
18527                        formula_dirty,
18528                        &owned_dirty_events,
18529                        &selected_span_refs,
18530                        runtime_target_roots.as_deref(),
18531                        delta.as_deref_mut(),
18532                    );
18533                }
18534
18535                // Gotcha G8 (refs #112): a span whose member cell participates in a
18536                // statically-cyclic SCC must never be span-evaluated. Cross-cell
18537                // cycles that route through a span producer are invisible to the
18538                // legacy Tarjan pass (the span member has no graph vertex) and only
18539                // surface here, as `CycleDetected` fallbacks in the producer-bounded
18540                // mixed schedule. Demote the cyclic spans to legacy graph vertices
18541                // so the cycle members move onto the legacy SCC path, then resolve
18542                // the now legacy-only cycle ahead of the schedule and rebuild.
18543                // Spans that do not touch the cycle are left untouched.
18544                let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
18545                if !cyclic_spans.is_empty() {
18546                    self.demote_cyclic_spans(&cyclic_spans)?;
18547                }
18548
18549                if self.graph.formula_authority().active_span_count() == 0 {
18550                    // All spans demoted; nothing left for the FP coordinator. The
18551                    // legacy evaluator resolves the (now fully legacy) cycle.
18552                    if let Some(target_roots) = runtime_target_roots.as_deref() {
18553                        let refreshed =
18554                            self.resolve_target_producers_from_existing_roots(target_roots)?;
18555                        let result =
18556                            self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
18557                        self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18558                        return Ok(result);
18559                    }
18560                    return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18561                }
18562
18563                // Resolve the residual legacy-only cycle (`handle_cycle_unit`
18564                // honors Static vs Runtime) before rebuilding so the mixed schedule
18565                // is cycle-free and the surviving spans still get evaluated.
18566                prepass_cycle_errors =
18567                    prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
18568
18569                // Re-seed every surviving span explicitly through the graph-owned
18570                // dirty authority, then renew the lease so successful completion
18571                // acknowledges both the original prefix and retry seeds.
18572                retry_whole_spans = if runtime_target_roots.is_some() {
18573                    let active = self
18574                        .graph
18575                        .formula_authority()
18576                        .active_span_refs()
18577                        .into_iter()
18578                        .map(|span_ref| (span_ref.id, span_ref))
18579                        .collect::<BTreeMap<_, _>>();
18580                    let mut refs = schedule
18581                        .layers
18582                        .iter()
18583                        .flat_map(|layer| layer.work.iter())
18584                        .filter_map(|work| match work.producer {
18585                            FormulaProducerId::Span(id) => active.get(&id).copied(),
18586                            FormulaProducerId::Legacy(_) => None,
18587                        })
18588                        .collect::<Vec<_>>();
18589                    refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
18590                        match fallback.producer {
18591                            FormulaProducerId::Span(id) => active.get(&id).copied(),
18592                            FormulaProducerId::Legacy(_) => None,
18593                        }
18594                    }));
18595                    refs.sort_by_key(|span_ref| span_ref.id);
18596                    refs.dedup();
18597                    refs
18598                } else {
18599                    self.graph.formula_authority().active_span_refs()
18600                };
18601                self.graph.mark_formula_spans_dirty(
18602                    retry_whole_spans.iter().copied(),
18603                    WholeSpanDirtyReason::CycleRetry,
18604                );
18605                // Only the first renewal can enlarge this generation's owned
18606                // prefix. If another cycle iteration is required, its redundant
18607                // retry seeds stay pending rather than risking acknowledgement of
18608                // work recorded after the first renewal.
18609                if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
18610                {
18611                    formula_dirty = extended;
18612                    #[cfg(test)]
18613                    if std::mem::take(
18614                        &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
18615                    ) {
18616                        self.graph.mark_formula_spans_dirty(
18617                            retry_whole_spans.iter().copied().take(1),
18618                            WholeSpanDirtyReason::GlobalInvalidation,
18619                        );
18620                    }
18621                }
18622                include_dirty_regions = false;
18623
18624                cycle_demote_iters += 1;
18625                if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
18626                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18627                        "FormulaPlane cyclic demotion did not converge".to_string(),
18628                    ));
18629                }
18630            };
18631            self.observe_formula_plane_route(
18632                &island_plan,
18633                FormulaPlaneRoutePhase::Planned,
18634                FormulaPlaneRouteTransitionReason::ProvenIsolated,
18635                cycle_demote_iters,
18636                runtime_replan_iterations,
18637            );
18638            if !island_plan.dirty_vertices.is_empty() {
18639                self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
18640                let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
18641                    &island_plan.dirty_vertices,
18642                    delta.as_deref_mut(),
18643                )?;
18644                computed_vertices = computed_vertices.saturating_add(island_computed);
18645                prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
18646                self.observe_formula_plane_route(
18647                    &island_plan,
18648                    FormulaPlaneRoutePhase::Executed,
18649                    FormulaPlaneRouteTransitionReason::ProvenIsolated,
18650                    cycle_demote_iters,
18651                    runtime_replan_iterations,
18652                );
18653                self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
18654            }
18655            let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
18656            let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18657
18658            #[cfg(test)]
18659            {
18660                self.last_formula_plane_span_eval_report = None;
18661            }
18662            for layer in schedule.layers {
18663                self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
18664                let mut buffer = ComputedWriteBuffer::default();
18665                let mut sink = SpanComputedWriteSink::new(&mut buffer);
18666                let work_items = layer.work;
18667                let mut work_index = 0usize;
18668                while work_index < work_items.len() {
18669                    match work_items[work_index].producer {
18670                        FormulaProducerId::Span(span_id) => {
18671                            let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
18672                                ExcelError::new(ExcelErrorKind::NImpl)
18673                                    .with_message("FormulaPlane schedule referenced a stale span")
18674                            })?;
18675                            let sheet_id = {
18676                                let authority = self.graph.formula_authority();
18677                                let span =
18678                                    authority.plane.spans.get(span_ref).ok_or_else(|| {
18679                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
18680                                            "FormulaPlane schedule referenced a stale span",
18681                                        )
18682                                    })?;
18683                                span.sheet_id
18684                            };
18685                            let current_sheet = self.graph.sheet_name(sheet_id);
18686                            let authority = self.graph.formula_authority();
18687                            let evaluator = SpanEvaluator::new_with_cancel(
18688                                &authority.plane,
18689                                self,
18690                                current_sheet,
18691                                self.graph.data_store(),
18692                                self.graph.sheet_reg(),
18693                                self.active_cancel_flag.as_ref(),
18694                            );
18695                            #[cfg(test)]
18696                            let mut last_group_report = None;
18697                            let mut selected_group_work = 0_u64;
18698                            while work_index < work_items.len() {
18699                                let FormulaProducerId::Span(group_span_id) =
18700                                    work_items[work_index].producer
18701                                else {
18702                                    break;
18703                                };
18704                                let group_span_ref =
18705                                    *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
18706                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
18707                                            "FormulaPlane schedule referenced a stale span",
18708                                        )
18709                                    })?;
18710                                let group_sheet_id = {
18711                                    let authority = self.graph.formula_authority();
18712                                    let span =
18713                                        authority.plane.spans.get(group_span_ref).ok_or_else(
18714                                            || {
18715                                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
18716                                                    "FormulaPlane schedule referenced a stale span",
18717                                                )
18718                                            },
18719                                        )?;
18720                                    span.sheet_id
18721                                };
18722                                if group_sheet_id != sheet_id {
18723                                    break;
18724                                }
18725
18726                                let dirty = producer_dirty_to_span_dirty(
18727                                    work_items[work_index].dirty.clone(),
18728                                    group_span_ref,
18729                                );
18730                                let task = SpanEvalTask {
18731                                    span: group_span_ref,
18732                                    dirty,
18733                                    plane_epoch,
18734                                };
18735                                self.cancellation_checkpoint(
18736                                    "Evaluation cancelled during FormulaPlane span",
18737                                )?;
18738                                let report = evaluator.evaluate_task(&task, &mut sink).map_err(
18739                                    |err| {
18740                                        if err
18741                                            == crate::formula_plane::span_eval::SpanEvalError::Cancelled
18742                                        {
18743                                            ExcelError::new(ExcelErrorKind::Cancelled).with_message(
18744                                                "Evaluation cancelled during FormulaPlane span",
18745                                            )
18746                                        } else {
18747                                            ExcelError::new(ExcelErrorKind::NImpl).with_message(
18748                                                format!(
18749                                                    "FormulaPlane span evaluation failed: {err:?}"
18750                                                ),
18751                                            )
18752                                        }
18753                                    },
18754                                )?;
18755                                #[cfg(test)]
18756                                {
18757                                    last_group_report = Some(report.clone());
18758                                }
18759                                selected_group_work = selected_group_work
18760                                    .saturating_add(report.span_eval_placement_count);
18761                                computed_vertices = computed_vertices
18762                                    .saturating_add(report.span_eval_placement_count as usize);
18763                                work_index = work_index.saturating_add(1);
18764                            }
18765                            // Charge the exact selected placement regions before
18766                            // their transaction-local value buffer is published.
18767                            self.charge_bounded_work(selected_group_work)?;
18768                            #[cfg(test)]
18769                            {
18770                                if let Some(report) = last_group_report {
18771                                    self.last_formula_plane_span_eval_report = Some(report);
18772                                }
18773                            }
18774                        }
18775                        FormulaProducerId::Legacy(_) => {
18776                            // Batch the contiguous run of legacy work items into a
18777                            // synthetic layer and evaluate it through the same
18778                            // coalesced effects pipeline as the legacy scheduler.
18779                            // Items in one mixed layer have no edges between them
18780                            // (same invariant the legacy Kahn layers rely on), so
18781                            // batching preserves ordering semantics while
18782                            // amortizing per-write overlay mirroring that makes
18783                            // one-vertex-at-a-time evaluation ~30x slower.
18784                            let mut vertices = Vec::new();
18785                            while work_index < work_items.len() {
18786                                let FormulaProducerId::Legacy(vertex_id) =
18787                                    work_items[work_index].producer
18788                                else {
18789                                    break;
18790                                };
18791                                vertices.push(vertex_id);
18792                                work_index = work_index.saturating_add(1);
18793                            }
18794                            let legacy_layer = crate::engine::scheduler::Layer { vertices };
18795                            let evaluated = if let Some(delta) = delta.as_deref_mut() {
18796                                if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
18797                                    self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
18798                                } else {
18799                                    self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
18800                                }
18801                            } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
18802                            {
18803                                self.evaluate_layer_parallel(&legacy_layer)?
18804                            } else {
18805                                self.evaluate_layer_sequential(&legacy_layer)?
18806                            };
18807                            computed_vertices = computed_vertices.saturating_add(evaluated);
18808                        }
18809                    }
18810                }
18811                self.resource_checkpoint(0)?;
18812                self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
18813                if let Some(delta) = delta.as_deref_mut() {
18814                    self.record_computed_write_buffer_delta(&buffer, delta);
18815                }
18816                self.flush_computed_write_buffer(&mut buffer)?;
18817            }
18818
18819            let mut changed_virtual =
18820                self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
18821            let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18822            let mut dynamic_candidates = old_dynamic_regions
18823                .keys()
18824                .chain(new_dynamic_regions.keys())
18825                .copied()
18826                .collect::<FxHashSet<_>>();
18827            for vertex in dynamic_candidates.drain() {
18828                if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
18829                    && !changed_virtual.contains(&vertex)
18830                {
18831                    changed_virtual.push(vertex);
18832                }
18833            }
18834            if let Some(telemetry) = virtual_telemetry.as_mut() {
18835                telemetry.changed_vdeps_total = telemetry
18836                    .changed_vdeps_total
18837                    .saturating_add(changed_virtual.len());
18838            }
18839            self.resource_checkpoint(0)?;
18840            self.graph.clear_dirty_flags(&legacy_vertices);
18841            if !changed_virtual.is_empty() {
18842                for vertex in &changed_virtual {
18843                    self.graph.set_dirty(*vertex, true);
18844                }
18845                let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
18846                let can_attempt_workbook =
18847                    !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
18848                if can_replan || can_attempt_workbook {
18849                    if can_replan {
18850                        if let Some(roots) = runtime_target_roots.as_mut() {
18851                            self.extend_target_roots_with_dynamic_regions(
18852                                roots,
18853                                old_dynamic_regions
18854                                    .values()
18855                                    .flatten()
18856                                    .chain(new_dynamic_regions.values().flatten()),
18857                            )?;
18858                        }
18859                        runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
18860                    } else {
18861                        runtime_target_roots = None;
18862                        workbook_exact_attempted = true;
18863                    }
18864                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
18865                        if can_replan {
18866                            stats.runtime_replan_rounds =
18867                                stats.runtime_replan_rounds.saturating_add(1);
18868                        }
18869                        stats.runtime_widening_rounds =
18870                            stats.runtime_widening_rounds.saturating_add(1);
18871                        if can_attempt_workbook {
18872                            stats.workbook_exact_attempts = 1;
18873                        }
18874                    }
18875                    self.cached_mixed_topology = None;
18876                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
18877                        ledger
18878                            .account_mixed_cache(0)
18879                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
18880                    }
18881                    include_dirty_regions = true;
18882                    continue 'virtual_replan;
18883                }
18884                if let Some(telemetry) = virtual_telemetry.as_mut() {
18885                    telemetry.replan_iterations = runtime_replan_iterations;
18886                    telemetry.bailout_reason = Some("max_replan");
18887                    self.last_virtual_dep_telemetry = telemetry.clone();
18888                }
18889                return Err(self.replan_exhausted_error(
18890                    MAX_RUNTIME_REPLAN,
18891                    "mixed dynamic dependency evaluation",
18892                ));
18893            }
18894            if let Some(mut telemetry) = virtual_telemetry {
18895                telemetry.replan_iterations = runtime_replan_iterations;
18896                telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
18897                    "converged"
18898                } else {
18899                    "max_replan"
18900                });
18901                self.last_virtual_dep_telemetry = telemetry;
18902            }
18903            // Drop dirty flags on any newly-scheduled FP runtime cells whose graph
18904            // vertices weren't in the dirty subset (e.g. recently-introduced span
18905            // result cells); legacy clear_dirty_flags is safe over the full set.
18906            self.redirty_for_next_recalc();
18907            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
18908            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
18909            return Ok(EvalResult {
18910                computed_vertices,
18911                cycle_errors: prepass_cycle_errors,
18912                elapsed: start.elapsed(),
18913            });
18914        }
18915    }
18916
18917    fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
18918        let budgets = &self.evaluation_resource_budgets;
18919        let candidate_limit = budgets
18920            .optimization
18921            .mixed_cache_candidates
18922            .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
18923                limit.min(self.config.max_formula_plane_cache_candidates)
18924            });
18925        let edge_limit = budgets
18926            .optimization
18927            .mixed_cache_edges
18928            .map_or(self.config.max_formula_plane_cache_edges, |limit| {
18929                limit.min(self.config.max_formula_plane_cache_edges)
18930            });
18931        let mut byte_limit =
18932            u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
18933        if let Some(limit) = budgets.retained.total_bytes {
18934            byte_limit = byte_limit.min(limit);
18935        }
18936        if let Some(limit) = budgets.retained.mixed_cache_bytes {
18937            byte_limit = byte_limit.min(limit);
18938        }
18939        (
18940            candidate_limit,
18941            edge_limit,
18942            usize::try_from(byte_limit).unwrap_or(usize::MAX),
18943        )
18944    }
18945
18946    fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
18947        let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
18948        MixedTopologyCacheKey {
18949            engine_topology_epoch: self.topology_epoch,
18950            graph_topology_revision: self.graph.topology_revision(),
18951            authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
18952            legacy_island_revision: self.graph.topology_revision(),
18953            boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
18954            function_semantic_epoch: self.function_semantic_epoch_seen,
18955            function_provider_revision: self.function_provider_revision_seen,
18956            max_candidates,
18957            max_edges,
18958            max_memory_bytes,
18959        }
18960    }
18961
18962    fn mixed_topology_index_preflight_bytes(&self) -> u64 {
18963        use crate::formula_plane::region_index::AxisRange;
18964
18965        const FIXED_BYTES: u64 = 8 * 1024;
18966        const ENTRY_BYTES: u64 = 512;
18967        const INDEX_UNIT_BYTES: u64 = 1024;
18968        const BINDING_BYTES: u64 = 128;
18969        let index_units = |region: Region| -> u64 {
18970            match region.normalized().axis_ranges() {
18971                (AxisRange::Point(_), AxisRange::Point(_)) => 2,
18972                (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
18973                    let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
18974                    let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
18975                    row_buckets.saturating_mul(col_buckets).saturating_mul(2)
18976                }
18977                _ => 1,
18978            }
18979        };
18980
18981        let authority = self.graph.formula_authority();
18982        let span_refs = authority.active_span_refs();
18983        let legacy_vertices = self.graph.formula_vertices();
18984        let mut entry_count = 0_u64;
18985        let mut units = 0_u64;
18986        for span_ref in &span_refs {
18987            if let Some(span) = authority.plane.spans.get(*span_ref) {
18988                let result_region = Region::from_domain(span.result_region.domain());
18989                entry_count = entry_count.saturating_add(1);
18990                units = units.saturating_add(index_units(result_region));
18991                if let Some(summary_id) = span.read_summary_id
18992                    && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
18993                {
18994                    for dependency in &summary.dependencies {
18995                        entry_count = entry_count.saturating_add(1);
18996                        units = units.saturating_add(index_units(dependency.read_region));
18997                    }
18998                }
18999            }
19000        }
19001        for &vertex in &legacy_vertices {
19002            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
19003                let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19004                entry_count = entry_count.saturating_add(1);
19005                units = units.saturating_add(index_units(point));
19006            }
19007            for dependency in self.graph.get_dependencies(vertex) {
19008                if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
19009                    let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19010                    entry_count = entry_count.saturating_add(1);
19011                    units = units.saturating_add(index_units(point));
19012                }
19013            }
19014            if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
19015                for range in ranges {
19016                    if let Ok(Some(region)) = self.shared_range_to_region_pattern(range) {
19017                        entry_count = entry_count.saturating_add(1);
19018                        units = units.saturating_add(index_units(region));
19019                    }
19020                }
19021            }
19022        }
19023        FIXED_BYTES
19024            .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
19025            .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
19026            .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
19027    }
19028
19029    fn prove_sheet_disjoint_legacy_island(
19030        &self,
19031        span_results: &FormulaProducerResultIndex,
19032        span_reads: &FormulaConsumerReadIndex,
19033        legacy_vertices: &[VertexId],
19034    ) -> Option<LegacyIslandPlan> {
19035        if !self.legacy_island_structural_summaries_trusted
19036            || self.graph.named_ranges_iter().next().is_some()
19037            || self.graph.sheet_named_ranges_iter().next().is_some()
19038            || self.graph.spill_registry_counts() != (0, 0)
19039            || legacy_vertices.iter().copied().any(|vertex| {
19040                self.graph.is_dynamic(vertex)
19041                    || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19042            })
19043        {
19044            return None;
19045        }
19046        let span_result_sheets = span_results
19047            .entries()
19048            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19049            .map(|entry| entry.result_region.sheet_id())
19050            .collect::<FxHashSet<_>>();
19051        let span_read_sheets = span_reads
19052            .entries()
19053            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
19054            .map(|entry| entry.read_region.sheet_id())
19055            .collect::<FxHashSet<_>>();
19056
19057        let span_boundary_sheets = span_result_sheets
19058            .union(&span_read_sheets)
19059            .copied()
19060            .collect::<FxHashSet<_>>();
19061        for vertex in legacy_vertices {
19062            let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
19063            if span_boundary_sheets.contains(&cell.sheet_id) {
19064                return None;
19065            }
19066            for dependency in self.graph.get_dependencies(*vertex) {
19067                let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
19068                if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
19069                    return None;
19070                }
19071            }
19072            if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19073                for range in ranges {
19074                    let region = self.shared_range_to_region_pattern(range).ok()??;
19075                    if span_boundary_sheets.contains(&region.sheet_id()) {
19076                        return None;
19077                    }
19078                }
19079            }
19080        }
19081
19082        let mut membership = legacy_vertices.to_vec();
19083        membership.sort_unstable();
19084        if membership.is_empty() {
19085            return None;
19086        }
19087        let mut sheet_ids = membership
19088            .iter()
19089            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19090            .map(|cell| cell.sheet_id)
19091            .collect::<Vec<_>>();
19092        sheet_ids.sort_unstable();
19093        sheet_ids.dedup();
19094        let island_id = membership
19095            .iter()
19096            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19097                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19098            });
19099        let retained_bytes = membership
19100            .capacity()
19101            .saturating_mul(std::mem::size_of::<VertexId>())
19102            .saturating_add(
19103                sheet_ids
19104                    .capacity()
19105                    .saturating_mul(std::mem::size_of::<SheetId>()),
19106            );
19107        Some(LegacyIslandPlan {
19108            membership,
19109            dirty_vertices: Vec::new(),
19110            sheet_ids,
19111            island_id,
19112            retained_bytes,
19113            boundary_relationships: 0,
19114            omitted_relationships: legacy_vertices.len(),
19115            omitted_direct_relationships: 0,
19116        })
19117    }
19118
19119    fn contract_legacy_islands(
19120        &self,
19121        full_results: &FormulaProducerResultIndex,
19122        full_reads: &FormulaConsumerReadIndex,
19123        legacy_reads_complete: bool,
19124        candidate_cap: usize,
19125    ) -> Option<(
19126        FormulaProducerResultIndex,
19127        FormulaConsumerReadIndex,
19128        LegacyIslandPlan,
19129    )> {
19130        use crate::formula_plane::producer::ProjectionResult;
19131        use crate::formula_plane::region_index::BoundedRegionQueryResult;
19132
19133        if !self.legacy_island_structural_summaries_trusted
19134            || !legacy_reads_complete
19135            || self.graph.named_ranges_iter().next().is_some()
19136            || self.graph.sheet_named_ranges_iter().next().is_some()
19137            || self.graph.spill_registry_counts() != (0, 0)
19138        {
19139            return None;
19140        }
19141        let legacy_vertices = self.graph.formula_vertices();
19142        if legacy_vertices.iter().copied().any(|vertex| {
19143            self.graph.is_dynamic(vertex)
19144                || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19145        }) {
19146            return None;
19147        }
19148
19149        let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
19150        for entry in full_reads.entries() {
19151            if let FormulaProducerId::Legacy(vertex) = entry.consumer {
19152                reads_by_consumer
19153                    .entry(vertex)
19154                    .or_default()
19155                    .push(entry.read_region);
19156            }
19157        }
19158
19159        let mut connected = FxHashSet::default();
19160        let mut queue = VecDeque::new();
19161        let mut observed_candidates = 0usize;
19162        let mut boundary_relationships = 0usize;
19163        let add_connected = |vertex: VertexId,
19164                             connected: &mut FxHashSet<VertexId>,
19165                             queue: &mut VecDeque<VertexId>| {
19166            if connected.insert(vertex) {
19167                queue.push_back(vertex);
19168            }
19169        };
19170
19171        // Span results -> legacy consumers.
19172        for result in full_results.entries() {
19173            if !matches!(result.producer, FormulaProducerId::Span(_)) {
19174                continue;
19175            }
19176            let remaining = candidate_cap.saturating_sub(observed_candidates);
19177            let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
19178            let BoundedRegionQueryResult::Complete(query) = query else {
19179                return None;
19180            };
19181            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19182            for matched in query.matches {
19183                if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
19184                    match matched.value.dirty {
19185                        ProjectionResult::NoIntersection => {}
19186                        ProjectionResult::Unsupported(_) => return None,
19187                        ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19188                            boundary_relationships = boundary_relationships.saturating_add(1);
19189                            add_connected(vertex, &mut connected, &mut queue);
19190                        }
19191                    }
19192                }
19193            }
19194        }
19195
19196        // Span reads -> legacy producers.
19197        for read in full_reads.entries() {
19198            if !matches!(read.consumer, FormulaProducerId::Span(_)) {
19199                continue;
19200            }
19201            let remaining = candidate_cap.saturating_sub(observed_candidates);
19202            let query = full_results.query_bounded(read.read_region, remaining);
19203            let BoundedRegionQueryResult::Complete(query) = query else {
19204                return None;
19205            };
19206            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19207            for matched in query.matches {
19208                if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
19209                    boundary_relationships = boundary_relationships.saturating_add(1);
19210                    add_connected(vertex, &mut connected, &mut queue);
19211                }
19212            }
19213        }
19214
19215        // Undirected closure through legacy relationships. Every vertex in this
19216        // closure remains in the mixed scheduler, preserving the only detector
19217        // for cycles whose path crosses a virtual span member.
19218        while let Some(vertex) = queue.pop_front() {
19219            let producer = FormulaProducerId::Legacy(vertex);
19220            if let Some(result_region) = full_results.producer_result_region(producer) {
19221                let remaining = candidate_cap.saturating_sub(observed_candidates);
19222                let query = full_reads.query_changed_region_bounded(result_region, remaining);
19223                let BoundedRegionQueryResult::Complete(query) = query else {
19224                    return None;
19225                };
19226                observed_candidates =
19227                    observed_candidates.saturating_add(query.stats.candidate_count);
19228                for matched in query.matches {
19229                    if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
19230                        match matched.value.dirty {
19231                            ProjectionResult::NoIntersection => {}
19232                            ProjectionResult::Unsupported(_) => return None,
19233                            ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19234                                add_connected(consumer, &mut connected, &mut queue);
19235                            }
19236                        }
19237                    }
19238                }
19239            }
19240            for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
19241                let remaining = candidate_cap.saturating_sub(observed_candidates);
19242                let query = full_results.query_bounded(*read_region, remaining);
19243                let BoundedRegionQueryResult::Complete(query) = query else {
19244                    return None;
19245                };
19246                observed_candidates =
19247                    observed_candidates.saturating_add(query.stats.candidate_count);
19248                for matched in query.matches {
19249                    if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
19250                        add_connected(precedent, &mut connected, &mut queue);
19251                    }
19252                }
19253            }
19254        }
19255
19256        let mut membership = legacy_vertices
19257            .iter()
19258            .copied()
19259            .filter(|vertex| !connected.contains(vertex))
19260            .collect::<Vec<_>>();
19261        membership.sort_unstable();
19262        if membership.is_empty() {
19263            return None;
19264        }
19265
19266        let mut producer_results = FormulaProducerResultIndex::default();
19267        let mut consumer_reads = FormulaConsumerReadIndex::default();
19268        let retained_span_results = full_results
19269            .entries()
19270            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19271            .count();
19272        producer_results
19273            .try_reserve(retained_span_results.saturating_add(connected.len()))
19274            .ok()?;
19275        for entry in full_results.entries() {
19276            let retain = match entry.producer {
19277                FormulaProducerId::Span(_) => true,
19278                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19279            };
19280            if retain {
19281                producer_results.insert_producer(entry.producer, entry.result_region);
19282            }
19283        }
19284        for entry in full_reads.entries() {
19285            let retain = match entry.consumer {
19286                FormulaProducerId::Span(_) => true,
19287                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19288            };
19289            if retain {
19290                consumer_reads.try_reserve(1).ok()?;
19291                consumer_reads.insert_read(
19292                    entry.consumer,
19293                    entry.read_region,
19294                    entry.consumer_result_region,
19295                    entry.projection,
19296                );
19297            }
19298        }
19299
19300        let mut sheet_ids = membership
19301            .iter()
19302            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19303            .map(|cell| cell.sheet_id)
19304            .collect::<Vec<_>>();
19305        sheet_ids.sort_unstable();
19306        sheet_ids.dedup();
19307        let island_id = membership
19308            .iter()
19309            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19310                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19311            });
19312        let retained_bytes = membership
19313            .capacity()
19314            .saturating_mul(std::mem::size_of::<VertexId>())
19315            .saturating_add(
19316                sheet_ids
19317                    .capacity()
19318                    .saturating_mul(std::mem::size_of::<SheetId>()),
19319            );
19320        let omitted_relationships = full_reads
19321            .entries()
19322            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
19323            .count();
19324        let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
19325        let omitted_direct_relationships = membership
19326            .iter()
19327            .map(|vertex| {
19328                self.graph
19329                    .get_dependencies(*vertex)
19330                    .into_iter()
19331                    .filter(|dependency| membership_set.contains(dependency))
19332                    .count()
19333            })
19334            .sum();
19335        Some((
19336            producer_results,
19337            consumer_reads,
19338            LegacyIslandPlan {
19339                membership,
19340                dirty_vertices: Vec::new(),
19341                sheet_ids,
19342                island_id,
19343                retained_bytes,
19344                boundary_relationships,
19345                omitted_relationships,
19346                omitted_direct_relationships,
19347            },
19348        ))
19349    }
19350
19351    fn compile_formula_plane_mixed_topology(
19352        &mut self,
19353        key: MixedTopologyCacheKey,
19354    ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
19355        #[cfg(test)]
19356        {
19357            self.mixed_topology_index_builds_for_test =
19358                self.mixed_topology_index_builds_for_test.saturating_add(1);
19359        }
19360        let authority = self.graph.formula_authority();
19361        let mut producer_results = FormulaProducerResultIndex::default();
19362        let mut consumer_reads = FormulaConsumerReadIndex::default();
19363        let mut legacy_reads_complete = true;
19364        let span_refs = authority.active_span_refs();
19365        let legacy_vertices = self.graph.formula_vertices();
19366        let _producer_count = span_refs
19367            .len()
19368            .checked_add(legacy_vertices.len())
19369            .ok_or_else(|| {
19370                ExcelError::new(ExcelErrorKind::NImpl)
19371                    .with_message("FormulaPlane cache producer count overflow")
19372            })?;
19373        producer_results.try_reserve(_producer_count).map_err(|_| {
19374            ExcelError::new(ExcelErrorKind::NImpl)
19375                .with_message("FormulaPlane cache producer index reservation failed")
19376        })?;
19377        let span_refs_by_id = span_refs
19378            .iter()
19379            .copied()
19380            .map(|span_ref| (span_ref.id, span_ref))
19381            .collect::<BTreeMap<_, _>>();
19382        for span_ref in &span_refs {
19383            let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
19384                ExcelError::new(ExcelErrorKind::NImpl)
19385                    .with_message("FormulaPlane active span ref is stale")
19386            })?;
19387            let result_region = Region::from_domain(span.result_region.domain());
19388            producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
19389            let Some(read_summary_id) = span.read_summary_id else {
19390                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19391                    .with_message("FormulaPlane active span is missing read summary"));
19392            };
19393            let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
19394            else {
19395                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19396                    .with_message("FormulaPlane active span has stale read summary"));
19397            };
19398            if read_summary.result_region != result_region {
19399                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19400                    .with_message("FormulaPlane active span read summary is stale"));
19401            }
19402            for dependency in &read_summary.dependencies {
19403                consumer_reads.try_reserve(1).map_err(|_| {
19404                    ExcelError::new(ExcelErrorKind::NImpl)
19405                        .with_message("FormulaPlane cache read-index reservation failed")
19406                })?;
19407                consumer_reads.insert_read(
19408                    FormulaProducerId::Span(span.id),
19409                    dependency.read_region,
19410                    read_summary.result_region,
19411                    dependency.projection,
19412                );
19413            }
19414        }
19415
19416        let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
19417            &producer_results,
19418            &consumer_reads,
19419            &legacy_vertices,
19420        );
19421        if sheet_disjoint_island.is_none() {
19422            for vertex in &legacy_vertices {
19423                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19424                    continue;
19425                };
19426                producer_results.insert_producer(
19427                    FormulaProducerId::Legacy(*vertex),
19428                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
19429                );
19430            }
19431            for vertex in &legacy_vertices {
19432                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19433                    continue;
19434                };
19435                let result_region =
19436                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19437                let mut seen = rustc_hash::FxHashSet::default();
19438                for dep in self.graph.get_dependencies(*vertex) {
19439                    let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
19440                        // Named/table/source vertices do not have a trustworthy
19441                        // regional boundary summary. Preserve the global planner.
19442                        legacy_reads_complete = false;
19443                        continue;
19444                    };
19445                    let read_region = Region::point(
19446                        dep_cell.sheet_id,
19447                        dep_cell.coord.row(),
19448                        dep_cell.coord.col(),
19449                    );
19450                    if seen.insert(read_region) {
19451                        consumer_reads.try_reserve(1).map_err(|_| {
19452                            ExcelError::new(ExcelErrorKind::NImpl)
19453                                .with_message("FormulaPlane cache read-index reservation failed")
19454                        })?;
19455                        consumer_reads.insert_read(
19456                            FormulaProducerId::Legacy(*vertex),
19457                            read_region,
19458                            result_region,
19459                            DirtyProjectionRule::WholeResult,
19460                        );
19461                    }
19462                }
19463                if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19464                    for range in ranges {
19465                        let Some(read_region) = self.shared_range_to_region_pattern(range)? else {
19466                            legacy_reads_complete = false;
19467                            continue;
19468                        };
19469                        if seen.insert(read_region) {
19470                            consumer_reads.try_reserve(1).map_err(|_| {
19471                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
19472                                    "FormulaPlane cache read-index reservation failed",
19473                                )
19474                            })?;
19475                            consumer_reads.insert_read(
19476                                FormulaProducerId::Legacy(*vertex),
19477                                read_region,
19478                                result_region,
19479                                DirtyProjectionRule::WholeResult,
19480                            );
19481                        }
19482                    }
19483                }
19484                let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
19485                for dependency in virtual_dependencies
19486                    .get(vertex)
19487                    .into_iter()
19488                    .flatten()
19489                    .copied()
19490                {
19491                    let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
19492                        continue;
19493                    };
19494                    let read_region =
19495                        Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19496                    if seen.insert(read_region) {
19497                        consumer_reads.try_reserve(1).map_err(|_| {
19498                            ExcelError::new(ExcelErrorKind::NImpl)
19499                                .with_message("FormulaPlane cache virtual read reservation failed")
19500                        })?;
19501                        consumer_reads.insert_read(
19502                            FormulaProducerId::Legacy(*vertex),
19503                            read_region,
19504                            result_region,
19505                            DirtyProjectionRule::WholeResult,
19506                        );
19507                    }
19508                }
19509                for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
19510                {
19511                    if seen.insert(read_region) {
19512                        consumer_reads.try_reserve(1).map_err(|_| {
19513                            ExcelError::new(ExcelErrorKind::NImpl)
19514                                .with_message("FormulaPlane dynamic-region reservation failed")
19515                        })?;
19516                        consumer_reads.insert_read(
19517                            FormulaProducerId::Legacy(*vertex),
19518                            read_region,
19519                            result_region,
19520                            DirtyProjectionRule::WholeResult,
19521                        );
19522                    }
19523                }
19524            }
19525        }
19526
19527        // Boundary discovery uses complete, request-local indexes. Only the
19528        // span-connected closure is retained and compiled; disconnected legacy
19529        // membership is contracted into the legacy scheduler primitive.
19530        let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
19531        {
19532            (producer_results, consumer_reads, island)
19533        } else {
19534            self.contract_legacy_islands(
19535                &producer_results,
19536                &consumer_reads,
19537                legacy_reads_complete,
19538                key.max_candidates,
19539            )
19540            .unwrap_or((
19541                producer_results,
19542                consumer_reads,
19543                LegacyIslandPlan::default(),
19544            ))
19545        };
19546
19547        let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
19548            .checked_add(
19549                producer_results
19550                    .estimated_memory_bytes()
19551                    .unwrap_or(usize::MAX),
19552            )
19553            .and_then(|bytes| {
19554                bytes.checked_add(
19555                    consumer_reads
19556                        .estimated_memory_bytes()
19557                        .unwrap_or(usize::MAX),
19558                )
19559            })
19560            .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
19561            .and_then(|bytes| bytes.checked_add(island.retained_bytes))
19562            .unwrap_or(usize::MAX);
19563        let config = MixedTopologyConfig {
19564            max_candidates: key.max_candidates,
19565            max_edges: key.max_edges,
19566            max_memory_bytes: key.max_memory_bytes,
19567            retained_memory_bytes,
19568        };
19569        let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
19570        // `producers_observed` remains a discovery counter, not merely the
19571        // post-contraction mixed-node count. Preserve that established meaning
19572        // while exposing omissions separately in route telemetry.
19573        match &mut compiled {
19574            MixedTopologyCompileResult::Cached(topology) => {
19575                topology.stats.producers = topology
19576                    .stats
19577                    .producers
19578                    .saturating_add(island.membership.len());
19579                topology.stats.candidates = topology
19580                    .stats
19581                    .candidates
19582                    .saturating_add(island.omitted_direct_relationships);
19583                topology.stats.relationships = topology
19584                    .stats
19585                    .relationships
19586                    .saturating_add(island.omitted_direct_relationships);
19587            }
19588            MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
19589                observed.producers = observed.producers.saturating_add(island.membership.len());
19590                observed.candidates = observed
19591                    .candidates
19592                    .saturating_add(island.omitted_direct_relationships);
19593                observed.relationships = observed
19594                    .relationships
19595                    .saturating_add(island.omitted_direct_relationships);
19596            }
19597        }
19598        let plane_epoch = authority.plane.epoch().0;
19599        Ok(match compiled {
19600            MixedTopologyCompileResult::Cached(topology) => {
19601                FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
19602                    key,
19603                    topology,
19604                    producer_results,
19605                    consumer_reads,
19606                    span_refs_by_id,
19607                    plane_epoch,
19608                    island,
19609                })
19610            }
19611            MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
19612                FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
19613                    reason,
19614                    observed,
19615                    producer_results,
19616                    consumer_reads,
19617                    span_refs_by_id,
19618                    plane_epoch,
19619                    island,
19620                })
19621            }
19622        })
19623    }
19624
19625    fn build_formula_plane_mixed_schedule(
19626        &mut self,
19627        formula_dirty: &FormulaDirtyLease,
19628        retry_whole_spans: &[FormulaSpanRef],
19629        include_dirty_regions: bool,
19630    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19631        self.build_formula_plane_mixed_schedule_inner(
19632            formula_dirty,
19633            retry_whole_spans,
19634            include_dirty_regions,
19635            None,
19636        )
19637    }
19638
19639    fn build_formula_plane_target_schedule(
19640        &mut self,
19641        formula_dirty: &FormulaDirtyLease,
19642        retry_whole_spans: &[FormulaSpanRef],
19643        include_dirty_regions: bool,
19644        target_roots: &[crate::engine::target_preparation::TargetProducer],
19645    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19646        self.build_formula_plane_mixed_schedule_inner(
19647            formula_dirty,
19648            retry_whole_spans,
19649            include_dirty_regions,
19650            Some(target_roots),
19651        )
19652    }
19653
19654    fn build_formula_plane_mixed_schedule_inner(
19655        &mut self,
19656        formula_dirty: &FormulaDirtyLease,
19657        retry_whole_spans: &[FormulaSpanRef],
19658        include_dirty_regions: bool,
19659        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19660    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19661        let baseline = self.graph.baseline_stats();
19662        self.charge_bounded_work(
19663            baseline
19664                .graph_vertex_count
19665                .saturating_add(baseline.graph_edge_count) as u64,
19666        )?;
19667        let key = self.mixed_topology_cache_key();
19668        let active_refs = self
19669            .graph
19670            .formula_authority()
19671            .active_span_refs()
19672            .into_iter()
19673            .map(|span_ref| (span_ref.id, span_ref))
19674            .collect::<BTreeMap<_, _>>();
19675        let has_dynamic_legacy = self
19676            .graph
19677            .formula_vertices()
19678            .into_iter()
19679            .any(|vertex| self.graph.is_dynamic(vertex));
19680        let cache_hit = !has_dynamic_legacy
19681            && self
19682                .cached_mixed_topology
19683                .as_ref()
19684                .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
19685        let mut skipped_topology = None;
19686        let mut index_scratch_reserved = 0_u64;
19687        let mut demand_scratch_reserved = 0_u64;
19688        let topology_started = crate::instant::FzInstant::now();
19689        let (cache_outcome, strategy, compile_stats) = if cache_hit {
19690            self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
19691            self.mixed_topology_cache_skip_streak = 0;
19692            let cached = self
19693                .cached_mixed_topology
19694                .as_ref()
19695                .expect("cache hit has topology");
19696            let stats = cached.topology.stats.clone();
19697            let strategy = if cached.topology.is_complete() {
19698                FormulaPlaneTopologyStrategy::Cached
19699            } else {
19700                FormulaPlaneTopologyStrategy::ExactPagedIndexed
19701            };
19702            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19703                ledger
19704                    .account_mixed_cache(stats.estimated_memory_bytes as u64)
19705                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19706            }
19707            (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
19708        } else {
19709            // A key/cap miss makes the old retained topology stale. Drop it before building its
19710            // replacement so the request never accounts or owns both generations concurrently.
19711            self.cached_mixed_topology = None;
19712            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19713                ledger
19714                    .account_mixed_cache(0)
19715                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19716            }
19717
19718            let preflight = self.mixed_topology_index_preflight_bytes();
19719            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19720                ledger
19721                    .reserve_schedule_discovery(preflight)
19722                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19723            }
19724            index_scratch_reserved = preflight;
19725            let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
19726                Ok(compiled) => compiled,
19727                Err(error) => {
19728                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19729                        let _ = ledger.release_scratch(index_scratch_reserved);
19730                    }
19731                    return Err(error);
19732                }
19733            };
19734            let actual_index_bytes = match &compiled {
19735                FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
19736                    .producer_results
19737                    .estimated_memory_bytes()
19738                    .and_then(|bytes| {
19739                        bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
19740                    })
19741                    .and_then(|bytes| {
19742                        bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
19743                    })
19744                    .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
19745                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
19746                    .producer_results
19747                    .estimated_memory_bytes()
19748                    .and_then(|bytes| {
19749                        bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
19750                    })
19751                    .and_then(|bytes| {
19752                        bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
19753                    })
19754                    .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
19755            }
19756            .unwrap_or(usize::MAX) as u64;
19757            debug_assert!(
19758                actual_index_bytes <= index_scratch_reserved,
19759                "mixed topology index preflight underestimated actual allocation"
19760            );
19761            if actual_index_bytes > index_scratch_reserved {
19762                let additional = actual_index_bytes - index_scratch_reserved;
19763                let reserve_result = self
19764                    .active_resource_ledger
19765                    .as_mut()
19766                    .map_or(Ok(()), |ledger| {
19767                        ledger.reserve_schedule_discovery(additional)
19768                    });
19769                if let Err(error) = reserve_result {
19770                    drop(compiled);
19771                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19772                        let _ = ledger.release_scratch(index_scratch_reserved);
19773                    }
19774                    return Err(error.into_excel_error());
19775                }
19776            } else if actual_index_bytes < index_scratch_reserved
19777                && let Some(ledger) = self.active_resource_ledger.as_mut()
19778            {
19779                ledger
19780                    .release_scratch(index_scratch_reserved - actual_index_bytes)
19781                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19782            }
19783            index_scratch_reserved = actual_index_bytes;
19784
19785            self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
19786            match compiled {
19787                FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
19788                    let stats = compiled.topology.stats.clone();
19789                    self.mixed_topology_cache_skip_streak =
19790                        self.mixed_topology_cache_skip_streak.saturating_add(1);
19791                    skipped_topology = Some(SkippedMixedTopology {
19792                        reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
19793                        observed: stats.clone(),
19794                        producer_results: compiled.producer_results,
19795                        consumer_reads: compiled.consumer_reads,
19796                        span_refs_by_id: compiled.span_refs_by_id,
19797                        plane_epoch: compiled.plane_epoch,
19798                        island: compiled.island,
19799                    });
19800                    (
19801                        FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
19802                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19803                        stats,
19804                    )
19805                }
19806                FormulaPlaneTopologyCompileResult::Cached(compiled) => {
19807                    let stats = compiled.topology.stats.clone();
19808                    let strategy = if compiled.topology.is_complete() {
19809                        FormulaPlaneTopologyStrategy::CompiledAndCached
19810                    } else {
19811                        FormulaPlaneTopologyStrategy::ExactPagedIndexed
19812                    };
19813                    let retained_result = self
19814                        .active_resource_ledger
19815                        .as_mut()
19816                        .map_or(Ok(()), |ledger| {
19817                            ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
19818                        });
19819                    if let Err(error) = retained_result {
19820                        if let Some(ledger) = self.active_resource_ledger.as_mut() {
19821                            let _ = ledger.release_scratch(index_scratch_reserved);
19822                        }
19823                        return Err(error.into_excel_error());
19824                    }
19825                    self.mixed_topology_cache_skip_streak = 0;
19826                    self.cached_mixed_topology = Some(compiled);
19827                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19828                        ledger
19829                            .release_scratch(index_scratch_reserved)
19830                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19831                    }
19832                    index_scratch_reserved = 0;
19833                    (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
19834                }
19835                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
19836                    debug_assert!(self.cached_mixed_topology.is_none());
19837                    self.mixed_topology_cache_overflows =
19838                        self.mixed_topology_cache_overflows.saturating_add(1);
19839                    self.mixed_topology_cache_skip_streak =
19840                        self.mixed_topology_cache_skip_streak.saturating_add(1);
19841                    let stats = skipped.observed.clone();
19842                    skipped_topology = Some(skipped);
19843                    (
19844                        FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
19845                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19846                        stats,
19847                    )
19848                }
19849            }
19850        };
19851        self.observe_topology(
19852            cache_outcome,
19853            strategy,
19854            &compile_stats,
19855            topology_started.elapsed(),
19856        );
19857        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
19858            stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
19859            stats.topology.candidate_cap = Some(key.max_candidates as u64);
19860            stats.topology.edge_cap = Some(key.max_edges as u64);
19861            stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
19862            if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
19863                stats.topology.operator_guidance =
19864                    Some("dynamic legacy dependencies require an exact request topology rebuild");
19865            } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
19866                stats.topology.operator_guidance =
19867                    Some(if compile_stats.candidate_overflow_count > 0 {
19868                        "raise the mixed-cache candidate limit to avoid exact request rebuilds"
19869                    } else if compile_stats.edge_overflow_count > 0 {
19870                        "raise the mixed-cache edge limit to avoid exact request rebuilds"
19871                    } else {
19872                        "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
19873                    });
19874            }
19875        }
19876
19877        let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
19878        let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19879            let (
19880                producer_results,
19881                consumer_reads,
19882                span_refs_by_id,
19883                plane_epoch,
19884                cached_topology,
19885                partial_topology,
19886                island,
19887            ) = if let Some(skipped) = skipped_topology.as_ref() {
19888                (
19889                    &skipped.producer_results,
19890                    &skipped.consumer_reads,
19891                    &skipped.span_refs_by_id,
19892                    skipped.plane_epoch,
19893                    None,
19894                    None,
19895                    &skipped.island,
19896                )
19897            } else {
19898                let cached = self
19899                    .cached_mixed_topology
19900                    .as_ref()
19901                    .expect("mixed topology available for request");
19902                let (complete, partial) = if cached.topology.is_complete() {
19903                    (Some(&cached.topology), None)
19904                } else {
19905                    (None, Some(&cached.topology))
19906                };
19907                (
19908                    &cached.producer_results,
19909                    &cached.consumer_reads,
19910                    &cached.span_refs_by_id,
19911                    cached.plane_epoch,
19912                    complete,
19913                    partial,
19914                    &cached.island,
19915                )
19916            };
19917            let demand = if let Some(target_roots) = target_roots {
19918                use crate::engine::target_preparation::TargetProducer;
19919                let mut roots = Vec::new();
19920                let mut graph_roots = Vec::new();
19921                for root in target_roots {
19922                    match *root {
19923                        TargetProducer::Span { span_ref, demanded } => {
19924                            roots.push((FormulaProducerId::Span(span_ref.id), demanded))
19925                        }
19926                        TargetProducer::Legacy(vertex) => {
19927                            graph_roots.push(vertex);
19928                            if let Some(region) = producer_results
19929                                .producer_result_region(FormulaProducerId::Legacy(vertex))
19930                            {
19931                                roots.push((FormulaProducerId::Legacy(vertex), region));
19932                            }
19933                        }
19934                        TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
19935                        TargetProducer::ValueOnly(_) => {}
19936                    }
19937                }
19938                if !graph_roots.is_empty() {
19939                    let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
19940                    for vertex in legacy_demand {
19941                        if let Some(region) = producer_results
19942                            .producer_result_region(FormulaProducerId::Legacy(vertex))
19943                        {
19944                            roots.push((FormulaProducerId::Legacy(vertex), region));
19945                        }
19946                    }
19947                }
19948                let roots = roots;
19949                let closure_base = (producer_results.len() as u64)
19950                    .saturating_mul(128)
19951                    .saturating_add(4096);
19952                let scratch_estimates =
19953                    exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
19954                let native_allowed = native_exact_demand_allowed(
19955                    self.active_resource_ledger
19956                        .as_ref()
19957                        .and_then(ResourceLedger::disk_scratch_policy),
19958                    !cfg!(target_arch = "wasm32"),
19959                );
19960                let (selected_strategy, reservation) = if cached_topology.is_some() {
19961                    (None, closure_base)
19962                } else {
19963                    let (strategy, reservation) =
19964                        select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
19965                            self.can_reserve_topology_scratch(bytes)
19966                        });
19967                    (Some(strategy), reservation)
19968                };
19969                if let Some(ledger) = self.active_resource_ledger.as_mut() {
19970                    ledger
19971                        .reserve_schedule_discovery(reservation)
19972                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19973                }
19974                demand_scratch_reserved = reservation;
19975                #[allow(unused_mut)]
19976                let mut native_disk_bytes = 0_u64;
19977                let (demand, actual_strategy, passes) = match selected_strategy {
19978                    None => (
19979                        build_demand_closure_cached(
19980                            roots,
19981                            producer_results,
19982                            cached_topology.expect("cached demand strategy has topology"),
19983                        ),
19984                        None,
19985                        0,
19986                    ),
19987                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
19988                        let (demand, passes) = build_demand_closure_paged(
19989                            roots,
19990                            producer_results,
19991                            consumer_reads,
19992                            |units| {
19993                                Self::resource_loop_checkpoint(
19994                                    &mut self.active_resource_ledger,
19995                                    units,
19996                                )
19997                            },
19998                        )?;
19999                        (demand, selected_strategy, passes)
20000                    }
20001                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20002                        let (demand, passes) = build_demand_closure_in_memory_runs(
20003                            roots,
20004                            producer_results,
20005                            consumer_reads,
20006                            |units| {
20007                                Self::resource_loop_checkpoint(
20008                                    &mut self.active_resource_ledger,
20009                                    units,
20010                                )
20011                            },
20012                        )?;
20013                        (demand, selected_strategy, passes)
20014                    }
20015                    #[cfg(not(target_arch = "wasm32"))]
20016                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20017                        let request_id = self
20018                            .active_evaluation_resource_request
20019                            .as_ref()
20020                            .map_or(0, |stats| stats.request_id);
20021                        match NativeTopologyScratch::create(request_id) {
20022                            Ok(mut native) => match build_demand_closure_native(
20023                                roots.clone(),
20024                                producer_results,
20025                                consumer_reads,
20026                                native.file.as_mut().expect("native scratch file"),
20027                                native
20028                                    .auxiliary_file
20029                                    .as_mut()
20030                                    .expect("native auxiliary scratch file"),
20031                                |units| {
20032                                    Self::resource_loop_checkpoint(
20033                                        &mut self.active_resource_ledger,
20034                                        units,
20035                                    )
20036                                },
20037                            ) {
20038                                Ok((demand, passes, bytes)) => {
20039                                    native_disk_bytes = bytes;
20040                                    (demand, selected_strategy, passes)
20041                                }
20042                                Err(NativeExactDemandError::Work(error)) => return Err(error),
20043                                Err(NativeExactDemandError::Io(_)) => {
20044                                    let (demand, passes) = build_demand_closure_repeated_passes(
20045                                        roots,
20046                                        producer_results,
20047                                        consumer_reads,
20048                                        |units| {
20049                                            Self::resource_loop_checkpoint(
20050                                                &mut self.active_resource_ledger,
20051                                                units,
20052                                            )
20053                                        },
20054                                    )?;
20055                                    (
20056                                        demand,
20057                                        Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20058                                        passes,
20059                                    )
20060                                }
20061                            },
20062                            Err(_) => {
20063                                let (demand, passes) = build_demand_closure_repeated_passes(
20064                                    roots,
20065                                    producer_results,
20066                                    consumer_reads,
20067                                    |units| {
20068                                        Self::resource_loop_checkpoint(
20069                                            &mut self.active_resource_ledger,
20070                                            units,
20071                                        )
20072                                    },
20073                                )?;
20074                                (
20075                                    demand,
20076                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20077                                    passes,
20078                                )
20079                            }
20080                        }
20081                    }
20082                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20083                        let (demand, passes) = build_demand_closure_repeated_passes(
20084                            roots,
20085                            producer_results,
20086                            consumer_reads,
20087                            |units| {
20088                                Self::resource_loop_checkpoint(
20089                                    &mut self.active_resource_ledger,
20090                                    units,
20091                                )
20092                            },
20093                        )?;
20094                        (demand, selected_strategy, passes)
20095                    }
20096                    _ => unreachable!("exact demand strategy selection is exhaustive"),
20097                };
20098                let strategy_memory = match actual_strategy {
20099                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20100                        scratch_estimates.paged.saturating_sub(closure_base)
20101                    }
20102                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20103                        scratch_estimates.runs.saturating_sub(closure_base)
20104                    }
20105                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20106                        scratch_estimates.native.saturating_sub(closure_base)
20107                    }
20108                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20109                        scratch_estimates.repeated.saturating_sub(closure_base)
20110                    }
20111                    None => 0,
20112                    _ => 0,
20113                };
20114                let actual_scratch = demand
20115                    .estimated_memory_bytes()
20116                    .saturating_add(strategy_memory);
20117                if actual_scratch > demand_scratch_reserved {
20118                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
20119                        ledger
20120                            .reserve_schedule_discovery(
20121                                actual_scratch.saturating_sub(demand_scratch_reserved),
20122                            )
20123                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20124                    }
20125                } else if actual_scratch < demand_scratch_reserved
20126                    && let Some(ledger) = self.active_resource_ledger.as_mut()
20127                {
20128                    ledger
20129                        .release_scratch(demand_scratch_reserved - actual_scratch)
20130                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20131                }
20132                demand_scratch_reserved = actual_scratch;
20133                if let Some(strategy) = actual_strategy
20134                    && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20135                {
20136                    if strategy.severity() > stats.topology.strategy.severity() {
20137                        stats.topology.strategy = strategy;
20138                    }
20139                    stats.topology.exact_pass_count = stats
20140                        .topology
20141                        .exact_pass_count
20142                        .saturating_add(passes.max(1));
20143                    stats.topology.native_topology_disk_bytes = stats
20144                        .topology
20145                        .native_topology_disk_bytes
20146                        .saturating_add(native_disk_bytes);
20147                }
20148                Some(demand)
20149            } else {
20150                None
20151            };
20152
20153            let dirty_legacy = self.graph.get_evaluation_vertices();
20154            let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
20155            let island_target_demand = target_roots.map(|roots| {
20156                use crate::engine::target_preparation::TargetProducer;
20157                let graph_roots = roots
20158                    .iter()
20159                    .filter_map(|root| match *root {
20160                        TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
20161                            Some(vertex)
20162                        }
20163                        TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
20164                    })
20165                    .collect::<Vec<_>>();
20166                self.build_demand_subgraph(&graph_roots)
20167                    .0
20168                    .into_iter()
20169                    .collect::<FxHashSet<_>>()
20170            });
20171            let mut island_plan = island.clone();
20172            island_plan.dirty_vertices = island_plan
20173                .membership
20174                .iter()
20175                .copied()
20176                .filter(|vertex| dirty_legacy_set.contains(vertex))
20177                .filter(|vertex| {
20178                    island_target_demand
20179                        .as_ref()
20180                        .is_none_or(|demand| demand.contains(vertex))
20181                })
20182                .collect();
20183            let mut work = Vec::new();
20184            let mut scheduled_legacy_vertices = Vec::new();
20185            let demanded_dirty = |producer: FormulaProducerId,
20186                                  dirty: ProducerDirtyDomain|
20187             -> Option<ProducerDirtyDomain> {
20188                let Some(demand) = demand.as_ref() else {
20189                    return Some(dirty);
20190                };
20191                let result = producer_results.producer_result_region(producer)?;
20192                dirty.intersect_demanded(result, demand.regions(producer))
20193            };
20194            for (span_ref, region) in formula_dirty.span_regions() {
20195                let producer = FormulaProducerId::Span(span_ref.id);
20196                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20197                    && let Some(dirty) =
20198                        demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
20199                {
20200                    work.push(FormulaProducerWork { producer, dirty });
20201                }
20202            }
20203            for span_ref in formula_dirty
20204                .whole_spans()
20205                .chain(retry_whole_spans.iter().copied())
20206            {
20207                let producer = FormulaProducerId::Span(span_ref.id);
20208                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20209                    && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20210                {
20211                    work.push(FormulaProducerWork { producer, dirty });
20212                }
20213            }
20214            for vertex in dirty_legacy {
20215                let producer = FormulaProducerId::Legacy(vertex);
20216                if producer_results.producer_result_region(producer).is_some()
20217                    && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20218                {
20219                    scheduled_legacy_vertices.push(vertex);
20220                    work.push(FormulaProducerWork { producer, dirty });
20221                }
20222            }
20223
20224            let pending_changed_regions = formula_dirty
20225                .regions()
20226                .chain(formula_dirty.span_regions().map(|(_, region)| region))
20227                .collect::<Vec<_>>();
20228            if include_dirty_regions && !pending_changed_regions.is_empty() {
20229                use crate::formula_plane::producer::compute_dirty_closure_checked;
20230                let closure = compute_dirty_closure_checked(
20231                    consumer_reads,
20232                    pending_changed_regions.iter().copied(),
20233                    |producer| producer_results.producer_result_region(producer),
20234                    |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
20235                )?;
20236                if closure.incomplete {
20237                    // The fixed-point guard discards partial closure output. Complete conservatively
20238                    // in the same topology/schedule path; it must not become a new demotion route.
20239                    work.clear();
20240                    scheduled_legacy_vertices.clear();
20241                    if let Some(demand) = demand.as_ref() {
20242                        for producer in demand.producers() {
20243                            if let Some(dirty) =
20244                                demanded_dirty(producer, ProducerDirtyDomain::Whole)
20245                            {
20246                                if let FormulaProducerId::Legacy(vertex) = producer {
20247                                    scheduled_legacy_vertices.push(vertex);
20248                                }
20249                                work.push(FormulaProducerWork { producer, dirty });
20250                            }
20251                        }
20252                    } else {
20253                        work.extend(span_refs_by_id.keys().copied().map(|id| {
20254                            FormulaProducerWork {
20255                                producer: FormulaProducerId::Span(id),
20256                                dirty: ProducerDirtyDomain::Whole,
20257                            }
20258                        }));
20259                        for vertex in self.graph.formula_vertices() {
20260                            let producer = FormulaProducerId::Legacy(vertex);
20261                            if producer_results.producer_result_region(producer).is_some() {
20262                                scheduled_legacy_vertices.push(vertex);
20263                                work.push(FormulaProducerWork {
20264                                    producer,
20265                                    dirty: ProducerDirtyDomain::Whole,
20266                                });
20267                            }
20268                        }
20269                    }
20270                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20271                        stats.topology.incomplete_reason =
20272                            Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
20273                    }
20274                } else if let Some(demand) = demand.as_ref() {
20275                    for item in closure.work {
20276                        if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
20277                            work.push(FormulaProducerWork {
20278                                producer: item.producer,
20279                                dirty,
20280                            });
20281                        }
20282                    }
20283                } else {
20284                    work.extend(closure.work);
20285                }
20286                if !closure.incomplete && !closure.fallbacks.is_empty() {
20287                    let mut already_whole = work
20288                        .iter()
20289                        .filter_map(|item| match (item.producer, &item.dirty) {
20290                            (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
20291                            _ => None,
20292                        })
20293                        .collect::<rustc_hash::FxHashSet<_>>();
20294                    for fallback in closure.fallbacks {
20295                        if let FormulaProducerId::Span(id) = fallback.consumer
20296                            && already_whole.insert(id)
20297                        {
20298                            work.push(FormulaProducerWork {
20299                                producer: FormulaProducerId::Span(id),
20300                                dirty: ProducerDirtyDomain::Whole,
20301                            });
20302                        }
20303                    }
20304                }
20305            }
20306
20307            let owned_dirty_events = if let Some(demand) = demand.as_ref() {
20308                use crate::formula_plane::producer::compute_dirty_closure_checked;
20309                let mut owned = Vec::new();
20310                for (index, event) in formula_dirty.events() {
20311                    let (seed_regions, initial) = match event {
20312                        FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
20313                        FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
20314                            vec![region],
20315                            Some(FormulaProducerWork {
20316                                producer: FormulaProducerId::Span(span_ref.id),
20317                                dirty: ProducerDirtyDomain::Regions(vec![region]),
20318                            }),
20319                        ),
20320                        FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
20321                            let producer = FormulaProducerId::Span(span_ref.id);
20322                            let Some(result) = producer_results.producer_result_region(producer)
20323                            else {
20324                                continue;
20325                            };
20326                            (
20327                                vec![result],
20328                                Some(FormulaProducerWork {
20329                                    producer,
20330                                    dirty: ProducerDirtyDomain::Whole,
20331                                }),
20332                            )
20333                        }
20334                    };
20335                    let closure = compute_dirty_closure_checked(
20336                        consumer_reads,
20337                        seed_regions,
20338                        |producer| producer_results.producer_result_region(producer),
20339                        |units| {
20340                            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
20341                        },
20342                    )?;
20343                    if closure.incomplete || !closure.fallbacks.is_empty() {
20344                        continue;
20345                    }
20346                    let fully_demanded =
20347                        initial
20348                            .into_iter()
20349                            .chain(closure.work.into_iter())
20350                            .all(|item| {
20351                                let Some(result) =
20352                                    producer_results.producer_result_region(item.producer)
20353                                else {
20354                                    return false;
20355                                };
20356                                item.dirty
20357                                    .result_regions(result)
20358                                    .into_iter()
20359                                    .all(|dirty_region| {
20360                                        demand
20361                                            .regions(item.producer)
20362                                            .iter()
20363                                            .any(|demanded| demanded.contains_region(dirty_region))
20364                                    })
20365                            });
20366                    if fully_demanded {
20367                        owned.push(index);
20368                    }
20369                }
20370                owned
20371            } else {
20372                (0..formula_dirty.len()).collect()
20373            };
20374
20375            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
20376            let producer_count = producer_results.len();
20377            let dirty_units = work.iter().fold(0usize, |total, item| {
20378                total.saturating_add(match &item.dirty {
20379                    ProducerDirtyDomain::Whole => 0,
20380                    ProducerDirtyDomain::Cells(cells) => cells.len(),
20381                    ProducerDirtyDomain::Regions(regions) => regions.len(),
20382                })
20383            });
20384            let work_bytes = work
20385                .len()
20386                .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
20387                .saturating_add(
20388                    dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
20389                        std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
20390                    )),
20391                ) as u64;
20392            let binding_bytes =
20393                estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
20394            let schedule_bytes = work_bytes
20395                .saturating_mul(4)
20396                .saturating_add((producer_count as u64).saturating_mul(192))
20397                .saturating_add(binding_bytes)
20398                .saturating_add(1024);
20399            let maximum_edges =
20400                producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
20401            // Producer/read indexes are either retained by the cache or covered by the live index
20402            // preflight reservation. Strategy reservations therefore cover only additional scheduling
20403            // structures and must not charge those indexes a second time.
20404            let paged_bytes = schedule_bytes
20405                .saturating_add(maximum_edges.saturating_mul(160))
20406                .saturating_add(16 * 1024);
20407            let runs_bytes = schedule_bytes
20408                .saturating_add(maximum_edges.saturating_mul(48))
20409                .saturating_add(8192);
20410            let native_bytes = schedule_bytes
20411                .saturating_add((producer_count as u64).saturating_mul(128))
20412                .saturating_add(2048);
20413            let repeated_bytes = schedule_bytes
20414                .saturating_add((producer_count as u64).saturating_mul(96))
20415                .saturating_add(1024);
20416            let native_allowed = self
20417                .active_resource_ledger
20418                .as_ref()
20419                .and_then(ResourceLedger::disk_scratch_policy)
20420                == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
20421                && !cfg!(target_arch = "wasm32");
20422
20423            let cached_schedule_bytes = schedule_bytes
20424                .saturating_add(
20425                    cached_topology
20426                        .map_or(0, |topology| topology.stats.relationships as u64)
20427                        .saturating_mul(160),
20428                )
20429                .saturating_add(1024);
20430            let use_cached_schedule = cached_topology.is_some()
20431                && self.can_reserve_topology_scratch(cached_schedule_bytes);
20432            let (selected_strategy, scratch) = if use_cached_schedule {
20433                (None, cached_schedule_bytes)
20434            } else if self.can_reserve_topology_scratch(paged_bytes) {
20435                (
20436                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
20437                    paged_bytes,
20438                )
20439            } else if self.can_reserve_topology_scratch(runs_bytes) {
20440                (
20441                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
20442                    runs_bytes,
20443                )
20444            } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
20445                (
20446                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20447                    native_bytes,
20448                )
20449            } else if self.can_reserve_topology_scratch(repeated_bytes) {
20450                (
20451                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20452                    repeated_bytes,
20453                )
20454            } else {
20455                if let Some(ledger) = self.active_resource_ledger.as_mut() {
20456                    ledger
20457                        .reserve_schedule_discovery(repeated_bytes)
20458                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20459                }
20460                unreachable!("failed scratch reservation must return a typed error")
20461            };
20462            if let Some(ledger) = self.active_resource_ledger.as_mut() {
20463                ledger
20464                    .reserve_schedule_discovery(scratch)
20465                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20466            }
20467            let request_id = self
20468                .active_evaluation_resource_request
20469                .as_ref()
20470                .map_or(0, |stats| stats.request_id);
20471            let build_result = (|| -> Result<_, ExcelError> {
20472                let (schedule, actual_strategy, passes, native_disk_bytes) =
20473                    match (cached_topology, selected_strategy) {
20474                        (Some(topology), None) => (
20475                            schedule_dirty_work(work, producer_results, topology, 256),
20476                            None,
20477                            0,
20478                            0,
20479                        ),
20480                        (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
20481                            let (schedule, passes) = schedule_dirty_work_paged_hybrid(
20482                                work,
20483                                producer_results,
20484                                consumer_reads,
20485                                partial_topology.filter(|_| reuse_partial_topology),
20486                                256,
20487                                |units| {
20488                                    Self::resource_loop_checkpoint(
20489                                        &mut self.active_resource_ledger,
20490                                        units,
20491                                    )
20492                                },
20493                            )?;
20494                            (schedule, selected_strategy, passes.max(1), 0)
20495                        }
20496                        (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
20497                            let (schedule, passes) = schedule_dirty_work_in_memory_runs(
20498                                work,
20499                                producer_results,
20500                                consumer_reads,
20501                                256,
20502                                |units| {
20503                                    Self::resource_loop_checkpoint(
20504                                        &mut self.active_resource_ledger,
20505                                        units,
20506                                    )
20507                                },
20508                            )?;
20509                            (schedule, selected_strategy, passes.max(1), 0)
20510                        }
20511                        #[cfg(not(target_arch = "wasm32"))]
20512                        (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
20513                            let native = NativeTopologyScratch::create(request_id);
20514                            if let Ok(mut native) = native {
20515                                match schedule_dirty_work_native(
20516                                    work.clone(),
20517                                    producer_results,
20518                                    consumer_reads,
20519                                    256,
20520                                    native.file.as_mut().expect("native scratch file"),
20521                                    |units| {
20522                                        Self::resource_loop_checkpoint(
20523                                            &mut self.active_resource_ledger,
20524                                            units,
20525                                        )
20526                                    },
20527                                ) {
20528                                    Ok((schedule, passes, topology_bytes)) => (
20529                                        schedule,
20530                                        Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20531                                        passes.max(1),
20532                                        topology_bytes,
20533                                    ),
20534                                    Err(NativeExactScheduleError::Work(error)) => {
20535                                        return Err(error);
20536                                    }
20537                                    Err(NativeExactScheduleError::Io(_)) => {
20538                                        let (schedule, passes) =
20539                                            schedule_dirty_work_repeated_passes(
20540                                                work,
20541                                                producer_results,
20542                                                consumer_reads,
20543                                                256,
20544                                                |units| {
20545                                                    Self::resource_loop_checkpoint(
20546                                                        &mut self.active_resource_ledger,
20547                                                        units,
20548                                                    )
20549                                                },
20550                                            )?;
20551                                        (
20552                                            schedule,
20553                                            Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20554                                            passes.max(1),
20555                                            0,
20556                                        )
20557                                    }
20558                                }
20559                            } else {
20560                                let (schedule, passes) = schedule_dirty_work_repeated_passes(
20561                                    work,
20562                                    producer_results,
20563                                    consumer_reads,
20564                                    256,
20565                                    |units| {
20566                                        Self::resource_loop_checkpoint(
20567                                            &mut self.active_resource_ledger,
20568                                            units,
20569                                        )
20570                                    },
20571                                )?;
20572                                (
20573                                    schedule,
20574                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20575                                    passes.max(1),
20576                                    0,
20577                                )
20578                            }
20579                        }
20580                        (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
20581                            let (schedule, passes) = schedule_dirty_work_repeated_passes(
20582                                work,
20583                                producer_results,
20584                                consumer_reads,
20585                                256,
20586                                |units| {
20587                                    Self::resource_loop_checkpoint(
20588                                        &mut self.active_resource_ledger,
20589                                        units,
20590                                    )
20591                                },
20592                            )?;
20593                            (schedule, selected_strategy, passes.max(1), 0)
20594                        }
20595                        _ => unreachable!(
20596                            "topology cache and exact strategy selection are exhaustive"
20597                        ),
20598                    };
20599                Ok((
20600                    schedule,
20601                    actual_strategy,
20602                    passes,
20603                    native_disk_bytes,
20604                    span_refs_by_id.clone(),
20605                ))
20606            })();
20607            let release_result = self
20608                .active_resource_ledger
20609                .as_mut()
20610                .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
20611                .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20612            let (schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
20613                match build_result {
20614                    Ok(output) => {
20615                        release_result?;
20616                        output
20617                    }
20618                    Err(error) => {
20619                        let _ = release_result;
20620                        return Err(error);
20621                    }
20622                };
20623            if let Some(strategy) = exact_strategy
20624                && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20625            {
20626                if strategy.severity() > stats.topology.strategy.severity() {
20627                    stats.topology.strategy = strategy;
20628                }
20629                stats.topology.exact_pass_count = pass_count;
20630                stats.topology.native_topology_disk_bytes = stats
20631                    .topology
20632                    .native_topology_disk_bytes
20633                    .saturating_add(native_disk_bytes);
20634            }
20635            Ok((
20636                schedule,
20637                result_span_refs,
20638                plane_epoch,
20639                scheduled_legacy_vertices,
20640                owned_dirty_events,
20641                island_plan,
20642            ))
20643        })();
20644        let scratch_release_result = self
20645            .active_resource_ledger
20646            .as_mut()
20647            .map_or(Ok(()), |ledger| {
20648                ledger
20649                    .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
20650            })
20651            .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20652        match schedule_result {
20653            Ok(output) => {
20654                scratch_release_result?;
20655                Ok(output)
20656            }
20657            Err(error) => {
20658                let _ = scratch_release_result;
20659                Err(error)
20660            }
20661        }
20662    }
20663}
20664
20665impl<R> Engine<R>
20666where
20667    R: EvaluationContext,
20668{
20669    fn shared_range_to_region_pattern(
20670        &self,
20671        range: &crate::reference::SharedRangeRef<'static>,
20672    ) -> Result<Option<Region>, ExcelError> {
20673        use crate::reference::SharedSheetLocator;
20674        let sheet_id = match range.sheet {
20675            SharedSheetLocator::Id(id) => id,
20676            SharedSheetLocator::Current => self.graph.default_sheet_id(),
20677            SharedSheetLocator::Name(_) => return Ok(None),
20678        };
20679        match (
20680            range.start_row,
20681            range.end_row,
20682            range.start_col,
20683            range.end_col,
20684        ) {
20685            (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
20686                sheet_id, sr.index, er.index, sc.index, ec.index,
20687            ))),
20688            (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
20689                Ok(Some(Region::whole_col(sheet_id, sc.index)))
20690            }
20691            (Some(sr), Some(er), None, None) if sr.index == er.index => {
20692                Ok(Some(Region::whole_row(sheet_id, sr.index)))
20693            }
20694            _ => Ok(None),
20695        }
20696    }
20697
20698    /// Evaluate all dirty/volatile vertices
20699    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
20700        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
20701            engine.evaluate_all_unobserved()
20702        })
20703    }
20704
20705    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
20706        debug_assert!(
20707            !self.graph.deferred_dirty_active(),
20708            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
20709             was not balanced by end_deferred_dirty"
20710        );
20711        self.observe_function_semantic_epoch()?;
20712        self.lookup_index_cache.reset_counters();
20713        let _source_cache = self.source_cache_session();
20714        self.validate_deterministic_mode()?;
20715        if self.config.defer_graph_building {
20716            // Build graph for all staged formulas before evaluating
20717            self.build_graph_all()?;
20718        }
20719        self.evaluate_all_coordinator()
20720    }
20721
20722    /// Central FormulaPlane-aware coordinator for `evaluate_all`. In
20723    /// `AuthoritativeExperimental` mode every call enters the FormulaPlane
20724    /// coordinator; the coordinator itself composes with private legacy
20725    /// primitives for legacy-only work.
20726    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
20727        self.begin_evaluation_request();
20728        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
20729            return self.evaluate_authoritative_formula_plane_all();
20730        }
20731        self.evaluate_all_legacy_impl()
20732    }
20733
20734    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
20735    /// at its position and evaluate each layer (parallel when enabled).
20736    ///
20737    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
20738    /// number of Cycle units walked (the former `schedule.cycles.len()`).
20739    fn legacy_pass_run_units(
20740        &mut self,
20741        schedule: &crate::engine::scheduler::Schedule,
20742    ) -> Result<(usize, usize), ExcelError> {
20743        let mut computed_vertices = 0;
20744        let mut cycle_count = 0;
20745        for &unit in &schedule.units {
20746            match unit {
20747                ScheduleUnit::Cycle(i) => {
20748                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
20749                        cycle_count += 1;
20750                    }
20751                }
20752                ScheduleUnit::Layer(i) => {
20753                    let layer = schedule.unit_layer(i);
20754                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20755                        computed_vertices += self.evaluate_layer_parallel(layer)?;
20756                    } else {
20757                        computed_vertices += self.evaluate_layer_sequential(layer)?;
20758                    }
20759                }
20760            }
20761        }
20762        Ok((computed_vertices, cycle_count))
20763    }
20764
20765    /// Execute a proven span-disconnected legacy island without beginning or
20766    /// finalising an evaluation request. The FormulaPlane coordinator retains
20767    /// sole ownership of the dirty lease, volatile redirty, acknowledgement,
20768    /// clock sample, and recalc-epoch advance.
20769    fn evaluate_legacy_island_non_finalizing(
20770        &mut self,
20771        vertices: &[VertexId],
20772        mut delta: Option<&mut DeltaCollector>,
20773    ) -> Result<(usize, usize), ExcelError> {
20774        if vertices.is_empty() {
20775            return Ok((0, 0));
20776        }
20777        let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
20778        let mut computed = 0usize;
20779        let mut cycles = 0usize;
20780        for &unit in &schedule.units {
20781            self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
20782            match unit {
20783                ScheduleUnit::Cycle(index) => {
20784                    if self.handle_cycle_unit(
20785                        schedule.unit_cycle(index),
20786                        delta.as_deref_mut(),
20787                        None,
20788                        None,
20789                    )? > 0
20790                    {
20791                        cycles = cycles.saturating_add(1);
20792                    }
20793                }
20794                ScheduleUnit::Layer(index) => {
20795                    let layer = schedule.unit_layer(index);
20796                    let evaluated = if let Some(delta) = delta.as_deref_mut() {
20797                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20798                            self.evaluate_layer_parallel_with_delta(layer, delta)?
20799                        } else {
20800                            self.evaluate_layer_sequential_with_delta(layer, delta)?
20801                        }
20802                    } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20803                        self.evaluate_layer_parallel(layer)?
20804                    } else {
20805                        self.evaluate_layer_sequential(layer)?
20806                    };
20807                    computed = computed.saturating_add(evaluated);
20808                }
20809            }
20810        }
20811        self.graph.clear_dirty_flags(vertices);
20812        Ok((computed, cycles))
20813    }
20814
20815    /// Legacy `evaluate_all` body, reachable from the FormulaPlane coordinator
20816    /// when no active spans exist or FormulaPlane authority is not in
20817    /// `AuthoritativeExperimental` mode. This is now an internal primitive; it
20818    /// must not be invoked directly from public APIs.
20819    ///
20820    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
20821    /// per-recalc clock sample): the FormulaPlane coordinator composes this
20822    /// primitive *after* `evaluate_legacy_cycle_prepass` may have accumulated
20823    /// counts (G8 demotion path), and both sub-passes belong to ONE request /
20824    /// one clock sample; request begin happens at the public entry points /
20825    /// coordinators instead.
20826    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
20827        self.reset_virtual_dep_telemetry_if_disabled();
20828        #[cfg(feature = "tracing")]
20829        let _span_eval = tracing::info_span!("evaluate_all").entered();
20830        let start = crate::instant::FzInstant::now();
20831        let mut computed_vertices = 0;
20832        let mut cycle_errors = 0;
20833        let mut replan_iterations = 0;
20834        const MAX_REPLAN: usize = 5;
20835        let mut telemetry = self
20836            .config
20837            .enable_virtual_dep_telemetry
20838            .then(|| self.start_virtual_dep_telemetry());
20839
20840        loop {
20841            let to_evaluate = self.graph.get_evaluation_vertices();
20842            if to_evaluate.is_empty() {
20843                if let Some(t) = telemetry.as_mut()
20844                    && t.bailout_reason.is_none()
20845                {
20846                    t.bailout_reason = Some("no_work");
20847                }
20848                break;
20849            }
20850
20851            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20852            if let Some(t) = telemetry.as_mut() {
20853                Self::accumulate_schedule_meta(t, &meta);
20854            }
20855
20856            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
20857            computed_vertices += pass_computed;
20858            cycle_errors += pass_cycles;
20859
20860            // Check if dynamic dependencies changed
20861            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
20862            if let Some(t) = telemetry.as_mut() {
20863                t.changed_vdeps_total += changed_vertices.len();
20864            }
20865
20866            self.resource_checkpoint(0)?;
20867            self.graph.clear_dirty_flags(&to_evaluate);
20868            for v in &changed_vertices {
20869                self.graph.set_dirty(*v, true);
20870            }
20871
20872            if changed_vertices.is_empty() {
20873                if let Some(t) = telemetry.as_mut() {
20874                    t.bailout_reason = Some("converged");
20875                }
20876                break;
20877            }
20878            if replan_iterations >= MAX_REPLAN {
20879                if let Some(mut t) = telemetry.take() {
20880                    t.bailout_reason = Some("max_replan");
20881                    t.replan_iterations = replan_iterations;
20882                    self.last_virtual_dep_telemetry = t;
20883                }
20884                return Err(
20885                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
20886                );
20887            }
20888
20889            replan_iterations += 1;
20890        }
20891
20892        if let Some(mut t) = telemetry {
20893            t.replan_iterations = replan_iterations;
20894            self.last_virtual_dep_telemetry = t;
20895        }
20896
20897        // Re-dirty volatile vertices for the next evaluation cycle
20898        self.redirty_for_next_recalc();
20899
20900        // Advance recalc epoch after a full evaluation pass finishes
20901        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20902
20903        Ok(EvalResult {
20904            computed_vertices,
20905            cycle_errors,
20906            elapsed: start.elapsed(),
20907        })
20908    }
20909
20910    pub fn evaluate_all_with_target_delta(
20911        &mut self,
20912    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
20913        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20914            engine.observe_function_semantic_epoch()?;
20915            let mut collector = DeltaCollector::new(DeltaMode::Cells);
20916            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20917            Ok((result, collector.finish_target()))
20918        })
20919    }
20920
20921    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
20922        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
20923    }
20924
20925    pub fn evaluate_all_with_delta_policy(
20926        &mut self,
20927        policy: EvalDeltaCompatibilityPolicy,
20928    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
20929        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20930            engine.observe_function_semantic_epoch()?;
20931            let mut collector = DeltaCollector::new(DeltaMode::Cells);
20932            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20933            Ok((result, collector.finish_with_policy(policy)?))
20934        })
20935    }
20936
20937    fn evaluate_all_with_delta_collector(
20938        &mut self,
20939        delta: &mut DeltaCollector,
20940    ) -> Result<EvalResult, ExcelError> {
20941        self.begin_evaluation_request();
20942        let _source_cache = self.source_cache_session();
20943        if self.config.defer_graph_building {
20944            self.build_graph_all()?;
20945        }
20946        if self.graph.formula_authority().active_span_count() > 0 {
20947            return self.evaluate_authoritative_formula_plane(None, Some(delta));
20948        }
20949        self.reset_virtual_dep_telemetry_if_disabled();
20950        #[cfg(feature = "tracing")]
20951        let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
20952        let start = crate::instant::FzInstant::now();
20953        let mut computed_vertices = 0;
20954        let mut cycle_errors = 0;
20955
20956        let mut replan_iterations = 0;
20957        const MAX_REPLAN: usize = 5;
20958        let mut telemetry = self
20959            .config
20960            .enable_virtual_dep_telemetry
20961            .then(|| self.start_virtual_dep_telemetry());
20962
20963        loop {
20964            let to_evaluate = self.graph.get_evaluation_vertices();
20965            if to_evaluate.is_empty() {
20966                if let Some(t) = telemetry.as_mut()
20967                    && t.bailout_reason.is_none()
20968                {
20969                    t.bailout_reason = Some("no_work");
20970                }
20971                break;
20972            }
20973
20974            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20975            if let Some(t) = telemetry.as_mut() {
20976                Self::accumulate_schedule_meta(t, &meta);
20977            }
20978
20979            for &unit in &schedule.units {
20980                match unit {
20981                    ScheduleUnit::Cycle(i) => {
20982                        if self.handle_cycle_unit(
20983                            schedule.unit_cycle(i),
20984                            Some(delta),
20985                            None,
20986                            None,
20987                        )? > 0
20988                        {
20989                            cycle_errors += 1;
20990                        }
20991                    }
20992                    ScheduleUnit::Layer(i) => {
20993                        let layer = schedule.unit_layer(i);
20994                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20995                            computed_vertices +=
20996                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
20997                        } else {
20998                            computed_vertices +=
20999                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
21000                        }
21001                    }
21002                }
21003            }
21004
21005            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21006            if let Some(t) = telemetry.as_mut() {
21007                t.changed_vdeps_total += changed_vertices.len();
21008            }
21009            self.resource_checkpoint(0)?;
21010            self.graph.clear_dirty_flags(&to_evaluate);
21011            for v in &changed_vertices {
21012                self.graph.set_dirty(*v, true);
21013            }
21014
21015            if changed_vertices.is_empty() {
21016                if let Some(t) = telemetry.as_mut() {
21017                    t.bailout_reason = Some("converged");
21018                }
21019                break;
21020            }
21021            if replan_iterations >= MAX_REPLAN {
21022                if let Some(mut t) = telemetry.take() {
21023                    t.bailout_reason = Some("max_replan");
21024                    t.replan_iterations = replan_iterations;
21025                    self.last_virtual_dep_telemetry = t;
21026                }
21027                return Err(
21028                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21029                );
21030            }
21031            replan_iterations += 1;
21032        }
21033
21034        if let Some(mut t) = telemetry {
21035            t.replan_iterations = replan_iterations;
21036            self.last_virtual_dep_telemetry = t;
21037        }
21038
21039        self.redirty_for_next_recalc();
21040        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21041
21042        Ok(EvalResult {
21043            computed_vertices,
21044            cycle_errors,
21045            elapsed: start.elapsed(),
21046        })
21047    }
21048
21049    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
21050    ///
21051    /// This will evaluate only the minimal set of dirty / volatile precedents required
21052    /// to bring the target cell up-to-date (as if a user asked for that single value),
21053    /// rather than scheduling a full workbook recalc. If the cell is already clean and
21054    /// non-volatile, no vertices will be recomputed.
21055    ///
21056    /// Returns the (possibly newly computed) value stored for the cell afterwards.
21057    /// Empty cells return None. Errors are surfaced via the Result type.
21058    pub fn evaluate_cell(
21059        &mut self,
21060        sheet: &str,
21061        row: u32,
21062        col: u32,
21063    ) -> Result<Option<LiteralValue>, ExcelError> {
21064        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
21065            engine.evaluate_cell_unobserved(sheet, row, col)
21066        })
21067    }
21068
21069    fn evaluate_cell_unobserved(
21070        &mut self,
21071        sheet: &str,
21072        row: u32,
21073        col: u32,
21074    ) -> Result<Option<LiteralValue>, ExcelError> {
21075        if row == 0 || col == 0 {
21076            return Err(ExcelError::new(ExcelErrorKind::Ref)
21077                .with_message("Row and column must be >= 1".to_string()));
21078        }
21079
21080        let result = self.evaluate_cells(&[(sheet, row, col)])?;
21081
21082        match result.len() {
21083            0 => Ok(None),
21084            1 => {
21085                let v = result.into_iter().next().unwrap();
21086                Ok(v)
21087            }
21088            _ => unreachable!("evaluate_cells returned unexpected length"),
21089        }
21090    }
21091
21092    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
21093    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
21094    /// is computed once and evaluated, which is typically faster than calling
21095    /// `evaluate_cell` repeatedly for a related set of targets.
21096    ///
21097    /// Returns the resulting values for each requested target in the same order.
21098    pub fn evaluate_cells(
21099        &mut self,
21100        targets: &[(&str, u32, u32)],
21101    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21102        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
21103            engine.evaluate_cells_unobserved(targets)
21104        })
21105    }
21106
21107    fn evaluate_cells_unobserved(
21108        &mut self,
21109        targets: &[(&str, u32, u32)],
21110    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21111        self.observe_function_semantic_epoch()?;
21112        debug_assert!(
21113            !self.graph.deferred_dirty_active(),
21114            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
21115             was not balanced by end_deferred_dirty"
21116        );
21117        self.validate_deterministic_mode()?;
21118        if targets.is_empty() {
21119            return Ok(Vec::new());
21120        }
21121        let typed_targets = self.legacy_coordinate_targets(targets);
21122        self.evaluate_mixed_targets(&typed_targets, None)?;
21123        Ok(targets
21124            .iter()
21125            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21126            .collect())
21127    }
21128
21129    pub fn evaluate_cells_cancellable(
21130        &mut self,
21131        targets: &[(&str, u32, u32)],
21132        cancel: crate::engine::CancelToken,
21133    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21134        self.observe_evaluation_resource_request(
21135            EvaluationRequestKind::CellsCancellable,
21136            |engine| {
21137                engine.observe_function_semantic_epoch()?;
21138                engine.active_cancel_flag = Some(cancel.clone());
21139                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
21140                engine.active_cancel_flag = None;
21141                res
21142            },
21143        )
21144    }
21145
21146    fn evaluate_cells_cancellable_impl(
21147        &mut self,
21148        targets: &[(&str, u32, u32)],
21149        cancel_flag: &AtomicBool,
21150    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21151        self.validate_deterministic_mode()?;
21152        if targets.is_empty() {
21153            return Ok(Vec::new());
21154        }
21155        if cancel_flag.load(Ordering::Relaxed) {
21156            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21157                .with_message("Evaluation cancelled before target preparation"));
21158        }
21159        let typed_targets = self.legacy_coordinate_targets(targets);
21160        self.evaluate_mixed_targets(&typed_targets, None)?;
21161        Ok(targets
21162            .iter()
21163            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
21164            .collect())
21165    }
21166
21167    pub fn evaluate_cells_with_target_delta(
21168        &mut self,
21169        targets: &[(&str, u32, u32)],
21170    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
21171        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21172            engine.observe_function_semantic_epoch()?;
21173            engine.validate_deterministic_mode()?;
21174            if targets.is_empty() {
21175                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
21176            }
21177            let mut collector = DeltaCollector::new(DeltaMode::Cells);
21178            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
21179            let values = targets
21180                .iter()
21181                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
21182                .collect();
21183            Ok((values, collector.finish_target()))
21184        })
21185    }
21186
21187    pub fn evaluate_cells_with_delta(
21188        &mut self,
21189        targets: &[(&str, u32, u32)],
21190    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21191        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
21192    }
21193
21194    pub fn evaluate_cells_with_delta_policy(
21195        &mut self,
21196        targets: &[(&str, u32, u32)],
21197        policy: EvalDeltaCompatibilityPolicy,
21198    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21199        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21200            engine.evaluate_cells_with_delta_unobserved(targets, policy)
21201        })
21202    }
21203
21204    fn evaluate_cells_with_delta_unobserved(
21205        &mut self,
21206        targets: &[(&str, u32, u32)],
21207        policy: EvalDeltaCompatibilityPolicy,
21208    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21209        self.observe_function_semantic_epoch()?;
21210        self.validate_deterministic_mode()?;
21211        if targets.is_empty() {
21212            return Ok((Vec::new(), EvalDelta::default()));
21213        }
21214        let mut collector = DeltaCollector::new(DeltaMode::Cells);
21215        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
21216        let values = targets
21217            .iter()
21218            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21219            .collect();
21220        Ok((values, collector.finish_with_policy(policy)?))
21221    }
21222
21223    /// Get the evaluation plan for target cells without actually evaluating them
21224    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
21225        if targets.is_empty() {
21226            return Ok(EvalPlan {
21227                total_vertices_to_evaluate: 0,
21228                layers: Vec::new(),
21229                cycles_detected: 0,
21230                dirty_count: 0,
21231                volatile_count: 0,
21232                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21233                estimated_parallel_layers: 0,
21234                target_cells: Vec::new(),
21235            });
21236        }
21237        if self.config.defer_graph_building && self.has_staged_formulas() {
21238            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
21239                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
21240            ));
21241        }
21242
21243        // Convert targets to A1 notation for consistency
21244        let addresses: Vec<String> = targets
21245            .iter()
21246            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
21247            .collect();
21248
21249        // Parse target cell addresses
21250        let mut target_addrs = Vec::new();
21251        for (sheet, row, col) in targets {
21252            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
21253                let coord = Coord::from_excel(*row, *col, true, true);
21254                target_addrs.push(CellRef::new(sheet_id, coord));
21255            }
21256        }
21257
21258        // Find vertex IDs for targets
21259        let mut target_vertex_ids = Vec::new();
21260        for addr in &target_addrs {
21261            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
21262                target_vertex_ids.push(*vertex_id);
21263            }
21264        }
21265
21266        if target_vertex_ids.is_empty() {
21267            return Ok(EvalPlan {
21268                total_vertices_to_evaluate: 0,
21269                layers: Vec::new(),
21270                cycles_detected: 0,
21271                dirty_count: 0,
21272                volatile_count: 0,
21273                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21274                estimated_parallel_layers: 0,
21275                target_cells: addresses,
21276            });
21277        }
21278
21279        // Build demand subgraph with virtual edges (same as evaluate_until)
21280        let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
21281
21282        if precedents_to_eval.is_empty() {
21283            return Ok(EvalPlan {
21284                total_vertices_to_evaluate: 0,
21285                layers: Vec::new(),
21286                cycles_detected: 0,
21287                dirty_count: 0,
21288                volatile_count: 0,
21289                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21290                estimated_parallel_layers: 0,
21291                target_cells: addresses,
21292            });
21293        }
21294
21295        // Count dirty and volatile vertices
21296        let mut dirty_count = 0;
21297        let mut volatile_count = 0;
21298        for &vertex_id in &precedents_to_eval {
21299            if self.graph.is_dirty(vertex_id) {
21300                dirty_count += 1;
21301            }
21302            if self.graph.is_volatile(vertex_id) {
21303                volatile_count += 1;
21304            }
21305        }
21306
21307        // Create schedule for the minimal subgraph honoring virtual edges
21308        let scheduler = Scheduler::new(&self.graph);
21309        let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
21310
21311        // Build layer information
21312        let mut layers = Vec::new();
21313        let mut estimated_parallel_layers = 0;
21314        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
21315
21316        for layer in &schedule.layers {
21317            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
21318            if parallel_eligible {
21319                estimated_parallel_layers += 1;
21320            }
21321
21322            // Get sample cell addresses (up to 5)
21323            let sample_cells: Vec<String> = layer
21324                .vertices
21325                .iter()
21326                .take(5)
21327                .filter_map(|&vertex_id| {
21328                    self.graph
21329                        .get_cell_ref_for_vertex(vertex_id)
21330                        .map(|cell_ref| {
21331                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
21332                            format!(
21333                                "{}!{}{}",
21334                                sheet_name,
21335                                Self::col_to_letters(cell_ref.coord.col()),
21336                                cell_ref.coord.row() + 1
21337                            )
21338                        })
21339                })
21340                .collect();
21341
21342            layers.push(LayerInfo {
21343                vertex_count: layer.vertices.len(),
21344                parallel_eligible,
21345                sample_cells,
21346            });
21347        }
21348
21349        Ok(EvalPlan {
21350            total_vertices_to_evaluate: precedents_to_eval.len(),
21351            layers,
21352            cycles_detected: schedule.cycles.len(),
21353            dirty_count,
21354            volatile_count,
21355            parallel_enabled,
21356            estimated_parallel_layers,
21357            target_cells: addresses,
21358        })
21359    }
21360    /// Helper to create a schedule, integrating virtual dependencies automatically.
21361    fn create_evaluation_schedule(
21362        &mut self,
21363        to_evaluate: &[VertexId],
21364    ) -> Result<ScheduleBuildOutput, ExcelError> {
21365        // Fold pending edge deltas once per schedule build so traversal uses
21366        // the zero-allocation CSR slices (#125).
21367        self.graph.flush_pending_edge_deltas();
21368        if self.can_use_static_schedule_cache(to_evaluate) {
21369            if let Some(cached) = self.cached_static_schedule.as_ref()
21370                && cached.topology_epoch == self.topology_epoch
21371                && cached.candidate_vertices.as_slice() == to_evaluate
21372            {
21373                let meta = ScheduleBuildMeta {
21374                    candidate_vertices: to_evaluate.len(),
21375                    vdeps_vertices: 0,
21376                    vdeps_edges: 0,
21377                    builder_elapsed_ms: 0,
21378                    used_virtual_schedule: false,
21379                    schedule_cache_hit: true,
21380                    schedule_cache_eligible: true,
21381                };
21382                return Ok((cached.schedule.clone(), FxHashMap::default(), meta));
21383            }
21384
21385            let (schedule, vdeps, mut meta) =
21386                self.create_evaluation_schedule_uncached(to_evaluate)?;
21387            meta.schedule_cache_hit = false;
21388            meta.schedule_cache_eligible = true;
21389            if vdeps.is_empty() {
21390                self.cached_static_schedule = Some(CachedScheduleEntry {
21391                    topology_epoch: self.topology_epoch,
21392                    candidate_vertices: to_evaluate.to_vec(),
21393                    schedule: schedule.clone(),
21394                });
21395            }
21396            return Ok((schedule, vdeps, meta));
21397        }
21398
21399        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
21400        meta.schedule_cache_hit = false;
21401        meta.schedule_cache_eligible = false;
21402        Ok((schedule, vdeps, meta))
21403    }
21404
21405    fn create_evaluation_schedule_uncached(
21406        &self,
21407        to_evaluate: &[VertexId],
21408    ) -> Result<ScheduleBuildOutput, ExcelError> {
21409        let builder = VirtualDepBuilder::new(self);
21410        let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
21411            if self.config.enable_virtual_dep_telemetry {
21412                let build_started = crate::instant::FzInstant::now();
21413                let (vdeps, augmented) = builder.build(to_evaluate);
21414                let builder_elapsed_ms = build_started.elapsed().as_millis();
21415                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
21416                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
21417            } else {
21418                let (vdeps, augmented) = builder.build(to_evaluate);
21419                (vdeps, augmented, 0, 0)
21420            };
21421
21422        let mut final_evaluate = to_evaluate.to_vec();
21423        if !augmented.is_empty() {
21424            final_evaluate.extend(augmented);
21425            final_evaluate.sort_unstable();
21426            final_evaluate.dedup();
21427        }
21428
21429        let use_virtual = !vdeps.is_empty();
21430
21431        let scheduler = Scheduler::new(&self.graph);
21432        let schedule = if use_virtual {
21433            scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
21434        } else {
21435            scheduler.create_schedule(&final_evaluate)?
21436        };
21437
21438        let meta = ScheduleBuildMeta {
21439            candidate_vertices: to_evaluate.len(),
21440            vdeps_vertices: vdeps.len(),
21441            vdeps_edges,
21442            builder_elapsed_ms,
21443            used_virtual_schedule: use_virtual,
21444            schedule_cache_hit: false,
21445            schedule_cache_eligible: false,
21446        };
21447
21448        Ok((schedule, vdeps, meta))
21449    }
21450
21451    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
21452        !to_evaluate.is_empty()
21453            && to_evaluate.iter().copied().all(|v| {
21454                !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
21455            })
21456    }
21457
21458    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
21459        VirtualDepTelemetry {
21460            fallback_mode_activations: self.virtual_dep_fallback_activations,
21461            ..VirtualDepTelemetry::default()
21462        }
21463    }
21464
21465    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
21466        telemetry.candidate_vertices_total += meta.candidate_vertices;
21467        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
21468        telemetry.vdeps_edges_total += meta.vdeps_edges;
21469        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
21470        if meta.schedule_cache_eligible {
21471            if meta.schedule_cache_hit {
21472                telemetry.schedule_cache_hits += 1;
21473                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
21474            } else {
21475                telemetry.schedule_cache_misses += 1;
21476            }
21477        }
21478        if meta.used_virtual_schedule {
21479            telemetry.schedule_virtual_passes += 1;
21480        } else {
21481            telemetry.schedule_static_passes += 1;
21482        }
21483    }
21484
21485    fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
21486        vertices
21487            .iter()
21488            .copied()
21489            .filter(|vertex| self.graph.is_dynamic(*vertex))
21490            .filter_map(|vertex| {
21491                let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
21492                (!regions.is_empty()).then_some((vertex, regions))
21493            })
21494            .collect()
21495    }
21496
21497    fn extend_target_roots_with_dynamic_regions<'a>(
21498        &self,
21499        roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
21500        regions: impl Iterator<Item = &'a Region>,
21501    ) -> Result<(), ExcelError> {
21502        use crate::engine::target_preparation::TargetProducer;
21503        let request_id = self
21504            .active_evaluation_resource_request
21505            .as_ref()
21506            .map(|request| request.request_id);
21507        let root_count = roots.len();
21508        let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
21509            .map_err(|_| target_root_allocation_error(root_count, request_id))?;
21510        for region in regions {
21511            for vertex in self.graph.formula_vertices() {
21512                let key = crate::formula_plane::region_index::RegionKey {
21513                    sheet_id: self.graph.get_vertex_sheet_id(vertex),
21514                    row: self.graph.vertex_coord(vertex).row(),
21515                    col: self.graph.vertex_coord(vertex).col(),
21516                };
21517                if region.contains_key(key) {
21518                    extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
21519                        target_root_allocation_error(extended.len() + 1, request_id)
21520                    })?;
21521                }
21522            }
21523            let authority = self.graph.formula_authority();
21524            for span_ref in authority.active_span_refs() {
21525                let Some(span) = authority.plane.spans.get(span_ref) else {
21526                    continue;
21527                };
21528                let Some(demanded) =
21529                    Region::from_domain(span.result_region.domain()).intersection(*region)
21530                else {
21531                    continue;
21532                };
21533                extended
21534                    .push(TargetProducer::Span { span_ref, demanded })
21535                    .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
21536            }
21537        }
21538        *roots = extended.into_vec();
21539        Ok(())
21540    }
21541
21542    fn changed_virtual_dep_vertices(
21543        &mut self,
21544        to_evaluate: &[VertexId],
21545        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
21546    ) -> Vec<VertexId> {
21547        #[cfg(test)]
21548        if self.force_virtual_dep_changes_remaining_for_test > 0
21549            && let Some(vertex) = to_evaluate.first().copied()
21550        {
21551            self.force_virtual_dep_changes_remaining_for_test -= 1;
21552            return vec![vertex];
21553        }
21554        if !to_evaluate
21555            .iter()
21556            .copied()
21557            .any(|v| self.graph.is_dynamic(v))
21558        {
21559            return Vec::new();
21560        }
21561
21562        let builder = VirtualDepBuilder::new(self);
21563        let (new_vdeps, _) = builder.build(to_evaluate);
21564
21565        let mut candidates = FxHashSet::default();
21566        candidates.extend(old_vdeps.keys().copied());
21567        candidates.extend(new_vdeps.keys().copied());
21568
21569        let mut changed = Vec::new();
21570        for v in candidates {
21571            if old_vdeps.get(&v) != new_vdeps.get(&v) {
21572                changed.push(v);
21573            }
21574        }
21575        changed
21576    }
21577
21578    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
21579    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
21580    fn build_demand_subgraph(
21581        &self,
21582        target_vertices: &[VertexId],
21583    ) -> (
21584        Vec<VertexId>,
21585        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
21586    ) {
21587        #[cfg(feature = "tracing")]
21588        let _span =
21589            tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
21590        use rustc_hash::{FxHashMap, FxHashSet};
21591
21592        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
21593        let mut visited: FxHashSet<VertexId> = FxHashSet::default();
21594        let mut stack: Vec<VertexId> = Vec::new();
21595        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); // incoming deps per vertex
21596
21597        for &t in target_vertices {
21598            stack.push(t);
21599        }
21600
21601        while let Some(v) = stack.pop() {
21602            if !visited.insert(v) {
21603                continue;
21604            }
21605            if !self.graph.vertex_exists(v) {
21606                continue;
21607            }
21608            // Schedule dirty/volatile formulas. Also schedule pass-through
21609            // Named*/Range vertices so the scheduler honours the
21610            // topological position of any formula cells that sit underneath
21611            // them — without these in `vertex_set` the scheduler skips the
21612            // edges that route a target through a named-range vertex into
21613            // its underlying cells, and the underlying cells then end up
21614            // in the same (or an earlier) layer as the target.
21615            match self.graph.get_vertex_kind(v) {
21616                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
21617                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
21618                        to_evaluate.insert(v);
21619                    }
21620                }
21621                VertexKind::NamedScalar
21622                | VertexKind::NamedArray
21623                | VertexKind::Range
21624                | VertexKind::InfiniteRange => {
21625                    to_evaluate.insert(v);
21626                }
21627                _ => {}
21628            }
21629
21630            // Explicit dependencies (graph edges). We push *every* dep onto
21631            // the stack — not just formulas — because intermediate vertices
21632            // (NamedScalar, NamedArray, Range) are pass-through nodes whose
21633            // own dependencies point at the actual formula cells. Filtering
21634            // by kind here previously caused DN-range refs to be dropped
21635            // from the demand subgraph, so a target like
21636            // ``=SUM(named_range_pointing_at_dirty_cells)`` would evaluate
21637            // using stale values for those cells. The kind check at the top
21638            // of the loop still gates which vertices end up in
21639            // ``to_evaluate``; only Formula vertices are scheduled.
21640            if let Some(dependencies) = self.graph.dependencies_slice(v) {
21641                for &dep in dependencies {
21642                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21643                        stack.push(dep);
21644                    }
21645                }
21646            } else {
21647                for dep in self.graph.get_dependencies(v) {
21648                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21649                        stack.push(dep);
21650                    }
21651                }
21652            } // Virtual dependencies (compressed ranges + dynamic like INDIRECT)
21653            let builder = VirtualDepBuilder::new(self);
21654            let (vdeps_map, _) = builder.build(&[v]);
21655            if let Some(deps) = vdeps_map.get(&v) {
21656                for &u in deps {
21657                    vdeps.entry(v).or_default().push(u);
21658                    if !visited.contains(&u) {
21659                        stack.push(u);
21660                    }
21661                }
21662            }
21663        }
21664
21665        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
21666        result.sort_unstable();
21667        // Dedup virtual deps
21668        for deps in vdeps.values_mut() {
21669            deps.sort_unstable();
21670            deps.dedup();
21671        }
21672        (result, vdeps)
21673    }
21674
21675    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
21676    fn col_to_letters(col: u32) -> String {
21677        col_letters_from_1based(col).expect("column index must be >= 1")
21678    }
21679
21680    /// Evaluate all dirty/volatile vertices with cancellation support
21681    pub fn evaluate_all_cancellable(
21682        &mut self,
21683        cancel: crate::engine::CancelToken,
21684    ) -> Result<EvalResult, ExcelError> {
21685        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
21686            engine.observe_function_semantic_epoch()?;
21687            engine.active_cancel_flag = Some(cancel.clone());
21688            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
21689            engine.active_cancel_flag = None;
21690            res
21691        })
21692    }
21693
21694    fn evaluate_all_cancellable_impl(
21695        &mut self,
21696        cancel_flag: &AtomicBool,
21697    ) -> Result<EvalResult, ExcelError> {
21698        self.begin_evaluation_request();
21699        let _source_cache = self.source_cache_session();
21700        self.validate_deterministic_mode()?;
21701        if self.config.defer_graph_building {
21702            self.build_graph_all()?;
21703        }
21704        if self.graph.formula_authority().active_span_count() > 0 {
21705            if cancel_flag.load(Ordering::Relaxed) {
21706                return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21707                    "Evaluation cancelled before FormulaPlane scheduling".to_string(),
21708                ));
21709            }
21710            return self.evaluate_authoritative_formula_plane_all();
21711        }
21712        self.reset_virtual_dep_telemetry_if_disabled();
21713        let start = crate::instant::FzInstant::now();
21714        let mut computed_vertices = 0;
21715        let mut cycle_errors = 0;
21716
21717        let mut replan_iterations = 0;
21718        const MAX_REPLAN: usize = 5;
21719        let mut telemetry = self
21720            .config
21721            .enable_virtual_dep_telemetry
21722            .then(|| self.start_virtual_dep_telemetry());
21723
21724        loop {
21725            if cancel_flag.load(Ordering::Relaxed) {
21726                if let Some(mut t) = telemetry {
21727                    t.bailout_reason = Some("cancelled");
21728                    t.replan_iterations = replan_iterations;
21729                    self.last_virtual_dep_telemetry = t;
21730                }
21731                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21732                    .with_message("Evaluation cancelled before scheduling".to_string()));
21733            }
21734
21735            let to_evaluate = self.graph.get_evaluation_vertices();
21736            if to_evaluate.is_empty() {
21737                if let Some(t) = telemetry.as_mut()
21738                    && t.bailout_reason.is_none()
21739                {
21740                    t.bailout_reason = Some("no_work");
21741                }
21742                break;
21743            }
21744
21745            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21746            if let Some(t) = telemetry.as_mut() {
21747                Self::accumulate_schedule_meta(t, &meta);
21748            }
21749
21750            // Walk units in condensation order, checking cancellation between
21751            // units (formerly between cycles and between layers).
21752            for &unit in &schedule.units {
21753                match unit {
21754                    ScheduleUnit::Cycle(i) => {
21755                        // Check cancellation between cycles
21756                        if cancel_flag.load(Ordering::Relaxed) {
21757                            if let Some(mut t) = telemetry {
21758                                t.bailout_reason = Some("cancelled");
21759                                t.replan_iterations = replan_iterations;
21760                                self.last_virtual_dep_telemetry = t;
21761                            }
21762                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21763                                "Evaluation cancelled during cycle handling".to_string(),
21764                            ));
21765                        }
21766
21767                        if self.handle_cycle_unit(
21768                            schedule.unit_cycle(i),
21769                            None,
21770                            None,
21771                            Some(cancel_flag),
21772                        )? > 0
21773                        {
21774                            cycle_errors += 1;
21775                        }
21776                    }
21777                    ScheduleUnit::Layer(i) => {
21778                        let layer = schedule.unit_layer(i);
21779                        // Check cancellation between layers
21780                        if cancel_flag.load(Ordering::Relaxed) {
21781                            if let Some(mut t) = telemetry {
21782                                t.bailout_reason = Some("cancelled");
21783                                t.replan_iterations = replan_iterations;
21784                                self.last_virtual_dep_telemetry = t;
21785                            }
21786                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21787                                .with_message("Evaluation cancelled between layers".to_string()));
21788                        }
21789
21790                        // Evaluate vertices in this layer (parallel or sequential)
21791                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21792                            computed_vertices +=
21793                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
21794                        } else {
21795                            computed_vertices +=
21796                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
21797                        }
21798                    }
21799                }
21800            }
21801
21802            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21803            if let Some(t) = telemetry.as_mut() {
21804                t.changed_vdeps_total += changed_vertices.len();
21805            }
21806            self.resource_checkpoint(0)?;
21807            self.graph.clear_dirty_flags(&to_evaluate);
21808            for v in &changed_vertices {
21809                self.graph.set_dirty(*v, true);
21810            }
21811
21812            if changed_vertices.is_empty() {
21813                if let Some(t) = telemetry.as_mut() {
21814                    t.bailout_reason = Some("converged");
21815                }
21816                break;
21817            }
21818            if replan_iterations >= MAX_REPLAN {
21819                if let Some(mut t) = telemetry.take() {
21820                    t.bailout_reason = Some("max_replan");
21821                    t.replan_iterations = replan_iterations;
21822                    self.last_virtual_dep_telemetry = t;
21823                }
21824                return Err(
21825                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21826                );
21827            }
21828            replan_iterations += 1;
21829        }
21830
21831        if let Some(mut t) = telemetry {
21832            t.replan_iterations = replan_iterations;
21833            self.last_virtual_dep_telemetry = t;
21834        }
21835
21836        // Re-dirty volatile vertices for the next evaluation cycle
21837        self.redirty_for_next_recalc();
21838        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21839
21840        Ok(EvalResult {
21841            computed_vertices,
21842            cycle_errors,
21843            elapsed: start.elapsed(),
21844        })
21845    }
21846
21847    /// Evaluate only the necessary precedents for specific target cells with cancellation support
21848    pub fn evaluate_until_cancellable(
21849        &mut self,
21850        targets: &[&str],
21851        cancel: crate::engine::CancelToken,
21852    ) -> Result<EvalResult, ExcelError> {
21853        self.observe_evaluation_resource_request(
21854            EvaluationRequestKind::TargetedCancellable,
21855            |engine| {
21856                engine.observe_function_semantic_epoch()?;
21857                engine.active_cancel_flag = Some(cancel.clone());
21858                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
21859                engine.active_cancel_flag = None;
21860                res
21861            },
21862        )
21863    }
21864
21865    fn evaluate_until_cancellable_impl(
21866        &mut self,
21867        targets: &[&str],
21868        cancel_flag: &AtomicBool,
21869    ) -> Result<EvalResult, ExcelError> {
21870        if cancel_flag.load(Ordering::Relaxed) {
21871            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21872                .with_message("Evaluation cancelled before target preparation"));
21873        }
21874        let mut typed_targets = Vec::with_capacity(targets.len());
21875        for target in targets {
21876            let (sheet, row, col) = self.parse_a1_notation(target)?;
21877            self.graph.sheet_id_mut(&sheet);
21878            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
21879        }
21880        self.evaluate_mixed_targets(&typed_targets, None)
21881    }
21882
21883    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
21884        let mut quoted = false;
21885        let mut separator = None;
21886        let bytes = address.as_bytes();
21887        let mut index = 0usize;
21888        while index < bytes.len() {
21889            match bytes[index] {
21890                b'\'' => {
21891                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
21892                        index = index.saturating_add(1);
21893                    } else {
21894                        quoted = !quoted;
21895                    }
21896                }
21897                b'!' if !quoted => separator = Some(index),
21898                _ => {}
21899            }
21900            index = index.saturating_add(1);
21901        }
21902        if quoted {
21903            return Err(ExcelError::new(ExcelErrorKind::Ref)
21904                .with_message(format!("Invalid quoted sheet reference `{address}`")));
21905        }
21906        let (sheet, cell_part) = match separator {
21907            Some(separator) => {
21908                let raw_sheet = &address[..separator];
21909                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
21910                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
21911                } else {
21912                    raw_sheet.to_string()
21913                };
21914                (sheet, &address[separator + 1..])
21915            }
21916            None => (self.default_sheet_name().to_string(), address),
21917        };
21918
21919        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
21920            ExcelError::new(ExcelErrorKind::Ref)
21921                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
21922        })?;
21923
21924        Ok((sheet, row, col))
21925    }
21926
21927    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
21928    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
21929        use formualizer_parse::parser::ASTNodeType;
21930        match &ast.node_type {
21931            ASTNodeType::Function { name, args, .. } => {
21932                if let Some(func) = self
21933                    .get_function("", name)
21934                    .or_else(|| crate::function_registry::get("", name))
21935                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
21936                {
21937                    return true;
21938                }
21939                args.iter()
21940                    .any(|arg| self.is_ast_volatile_with_provider(arg))
21941            }
21942            ASTNodeType::BinaryOp { left, right, .. } => {
21943                self.is_ast_volatile_with_provider(left)
21944                    || self.is_ast_volatile_with_provider(right)
21945            }
21946            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
21947            ASTNodeType::Array(rows) => rows.iter().any(|row| {
21948                row.iter()
21949                    .any(|cell| self.is_ast_volatile_with_provider(cell))
21950            }),
21951            _ => false,
21952        }
21953    }
21954
21955    /// Find dirty precedents that need evaluation for the given target vertices
21956    fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
21957        let mut to_evaluate = FxHashSet::default();
21958        let mut visited = FxHashSet::default();
21959        let mut stack = Vec::new();
21960
21961        // Start reverse traversal from target vertices
21962        for &target in target_vertices {
21963            stack.push(target);
21964        }
21965
21966        while let Some(vertex_id) = stack.pop() {
21967            if !visited.insert(vertex_id) {
21968                continue; // Already processed
21969            }
21970
21971            if self.graph.vertex_exists(vertex_id) {
21972                // Check if this vertex needs evaluation
21973                let kind = self.graph.get_vertex_kind(vertex_id);
21974                let needs_eval = match kind {
21975                    super::vertex::VertexKind::FormulaScalar
21976                    | super::vertex::VertexKind::FormulaArray => {
21977                        self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
21978                    }
21979                    _ => false, // Values and empty cells don't need evaluation
21980                };
21981
21982                if needs_eval {
21983                    to_evaluate.insert(vertex_id);
21984                }
21985
21986                // Continue traversal to dependencies (precedents)
21987                if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
21988                    for &dep_id in dependencies {
21989                        if !visited.contains(&dep_id) {
21990                            stack.push(dep_id);
21991                        }
21992                    }
21993                } else {
21994                    let dependencies = self.graph.get_dependencies(vertex_id);
21995                    for dep_id in dependencies {
21996                        if !visited.contains(&dep_id) {
21997                            stack.push(dep_id);
21998                        }
21999                    }
22000                }
22001            }
22002        }
22003
22004        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
22005        result.sort_unstable();
22006        result
22007    }
22008
22009    /// Evaluate a layer sequentially
22010    fn evaluate_layer_sequential(
22011        &mut self,
22012        layer: &super::scheduler::Layer,
22013    ) -> Result<usize, ExcelError> {
22014        self.resource_checkpoint(layer.vertices.len() as u64)?;
22015        self.evaluate_layer_sequential_effects(layer)
22016    }
22017
22018    fn update_vertex_value_with_delta(
22019        &mut self,
22020        vertex_id: VertexId,
22021        new_value: LiteralValue,
22022        delta: &mut DeltaCollector,
22023    ) {
22024        if delta.mode != DeltaMode::Off
22025            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
22026        {
22027            let sheet_name = self.graph.sheet_name(cell.sheet_id);
22028            let old = self
22029                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22030                .unwrap_or(LiteralValue::Empty);
22031            if old != new_value {
22032                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
22033            }
22034        }
22035        self.graph.update_vertex_value(vertex_id, new_value.clone());
22036        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
22037    }
22038
22039    fn evaluate_layer_sequential_with_delta(
22040        &mut self,
22041        layer: &super::scheduler::Layer,
22042        delta: &mut DeltaCollector,
22043    ) -> Result<usize, ExcelError> {
22044        self.resource_checkpoint(layer.vertices.len() as u64)?;
22045        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
22046    }
22047
22048    /// Evaluate a layer sequentially with cancellation support
22049    fn evaluate_layer_sequential_cancellable(
22050        &mut self,
22051        layer: &super::scheduler::Layer,
22052        cancel_flag: &AtomicBool,
22053    ) -> Result<usize, ExcelError> {
22054        self.resource_checkpoint(layer.vertices.len() as u64)?;
22055        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
22056    }
22057
22058    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
22059    fn evaluate_layer_sequential_cancellable_demand_driven(
22060        &mut self,
22061        layer: &super::scheduler::Layer,
22062        cancel_flag: &AtomicBool,
22063    ) -> Result<usize, ExcelError> {
22064        self.resource_checkpoint(layer.vertices.len() as u64)?;
22065        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
22066    }
22067
22068    /// Evaluate a layer in parallel using the thread pool
22069    fn evaluate_layer_parallel(
22070        &mut self,
22071        layer: &super::scheduler::Layer,
22072    ) -> Result<usize, ExcelError> {
22073        self.resource_checkpoint(layer.vertices.len() as u64)?;
22074        self.evaluate_layer_parallel_effects(layer)
22075    }
22076
22077    fn evaluate_layer_parallel_with_delta(
22078        &mut self,
22079        layer: &super::scheduler::Layer,
22080        delta: &mut DeltaCollector,
22081    ) -> Result<usize, ExcelError> {
22082        self.resource_checkpoint(layer.vertices.len() as u64)?;
22083        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
22084    }
22085
22086    /// Evaluate a layer in parallel with cancellation support
22087    fn evaluate_layer_parallel_cancellable(
22088        &mut self,
22089        layer: &super::scheduler::Layer,
22090        cancel_flag: &AtomicBool,
22091    ) -> Result<usize, ExcelError> {
22092        self.resource_checkpoint(layer.vertices.len() as u64)?;
22093        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
22094    }
22095
22096    /// Apply a computed result produced by `evaluate_vertex_immutable()`.
22097    ///
22098    /// This is the parallel equivalent of the "apply" portion of `evaluate_vertex_impl`.
22099    /// We keep apply sequential for correctness (spill commit is inherently stateful).
22100    fn apply_parallel_vertex_result(
22101        &mut self,
22102        vertex_id: VertexId,
22103        result: LiteralValue,
22104        mut delta: Option<&mut DeltaCollector>,
22105        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22106    ) -> Result<(), ExcelError> {
22107        // If this vertex's cell is currently covered by a spill from a different anchor,
22108        // ignore the computed result. The spill's committed values own the grid.
22109        if let Some(cell) = self.graph.get_cell_ref(vertex_id)
22110            && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
22111            && owner != vertex_id
22112        {
22113            return Ok(());
22114        }
22115
22116        let kind = self.graph.get_vertex_kind(vertex_id);
22117
22118        // Only formula vertices spill dynamic arrays into the grid.
22119        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
22120        if is_formula {
22121            match result {
22122                LiteralValue::Array(rows) => {
22123                    self.apply_array_result_from_parallel(
22124                        vertex_id,
22125                        rows,
22126                        delta.as_deref_mut(),
22127                        overwritable_formulas,
22128                    )?;
22129                }
22130                other => {
22131                    self.apply_non_array_result_from_parallel(
22132                        vertex_id,
22133                        other,
22134                        delta.as_deref_mut(),
22135                    );
22136                }
22137            }
22138            return Ok(());
22139        }
22140
22141        // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
22142        if let Some(d) = delta {
22143            self.update_vertex_value_with_delta(vertex_id, result, d);
22144        } else {
22145            self.graph.update_vertex_value(vertex_id, result.clone());
22146            self.mirror_vertex_value_to_overlay(vertex_id, &result);
22147        }
22148        Ok(())
22149    }
22150
22151    fn apply_non_array_result_from_parallel(
22152        &mut self,
22153        vertex_id: VertexId,
22154        value: LiteralValue,
22155        delta: Option<&mut DeltaCollector>,
22156    ) {
22157        // Scalar/error result: store value and ensure any previous spill is cleared.
22158        // This mirrors the sequential behavior in `evaluate_vertex_impl`.
22159        let spill_cells = self
22160            .graph
22161            .spill_cells_for_anchor(vertex_id)
22162            .map(|cells| cells.to_vec())
22163            .unwrap_or_default();
22164
22165        if let Some(d) = delta
22166            && d.mode != DeltaMode::Off
22167            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
22168        {
22169            if spill_cells.is_empty() {
22170                let old = self
22171                    .read_cell_value(
22172                        self.graph.sheet_name(anchor.sheet_id),
22173                        anchor.coord.row() + 1,
22174                        anchor.coord.col() + 1,
22175                    )
22176                    .unwrap_or(LiteralValue::Empty);
22177                if old != value {
22178                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22179                }
22180            } else {
22181                for cell in spill_cells.iter() {
22182                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
22183                    let old = self
22184                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22185                        .unwrap_or(LiteralValue::Empty);
22186                    let new = if cell.sheet_id == anchor.sheet_id
22187                        && cell.coord.row() == anchor.coord.row()
22188                        && cell.coord.col() == anchor.coord.col()
22189                    {
22190                        value.clone()
22191                    } else {
22192                        LiteralValue::Empty
22193                    };
22194                    Self::record_cell_if_changed(d, cell, &old, &new);
22195                }
22196            }
22197        }
22198
22199        self.graph.clear_spill_region(vertex_id);
22200        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
22201            self.record_formula_plane_structural_change(scope);
22202        }
22203
22204        if self.config.arrow_storage_enabled
22205            && self.config.delta_overlay_enabled
22206            && self.config.write_formula_overlay_enabled
22207        {
22208            let empty = LiteralValue::Empty;
22209            for cell in spill_cells.iter() {
22210                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
22211                self.mirror_value_to_computed_overlay(
22212                    &sheet_name,
22213                    cell.coord.row() + 1,
22214                    cell.coord.col() + 1,
22215                    &empty,
22216                );
22217            }
22218        }
22219
22220        self.graph.update_vertex_value(vertex_id, value.clone());
22221        self.mirror_vertex_value_to_overlay(vertex_id, &value);
22222    }
22223
22224    fn apply_array_result_from_parallel(
22225        &mut self,
22226        vertex_id: VertexId,
22227        rows: Vec<Vec<LiteralValue>>,
22228        mut delta: Option<&mut DeltaCollector>,
22229        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22230    ) -> Result<(), ExcelError> {
22231        // Keep behavior consistent with the sequential spill path in `evaluate_vertex_impl`.
22232        self.graph
22233            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
22234
22235        let anchor = self
22236            .graph
22237            .get_cell_ref(vertex_id)
22238            .expect("cell ref for vertex");
22239        let sheet_id = anchor.sheet_id;
22240        let h = rows.len() as u32;
22241        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
22242
22243        // Hard cap to avoid vertex explosion from huge dynamic arrays.
22244        let spill_cells = (h as u64).saturating_mul(w as u64);
22245        if spill_cells > self.config.spill.max_spill_cells as u64 {
22246            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22247            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22248                .with_message("SpillTooLarge")
22249                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22250                    expected_rows: h,
22251                    expected_cols: w,
22252                });
22253            let spill_val = LiteralValue::Error(spill_err.clone());
22254            if let Some(d) = delta.as_deref_mut()
22255                && d.mode != DeltaMode::Off
22256            {
22257                let old = self
22258                    .read_cell_value(
22259                        self.graph.sheet_name(anchor.sheet_id),
22260                        anchor.coord.row() + 1,
22261                        anchor.coord.col() + 1,
22262                    )
22263                    .unwrap_or(LiteralValue::Empty);
22264                if old != spill_val {
22265                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22266                }
22267            }
22268            self.graph.update_vertex_value(vertex_id, spill_val.clone());
22269            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22270            return Ok(());
22271        }
22272
22273        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
22274        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
22275        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
22276        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
22277        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
22278        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
22279            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22280            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22281                .with_message("Spill exceeds sheet bounds")
22282                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22283                    expected_rows: h,
22284                    expected_cols: w,
22285                });
22286            let spill_val = LiteralValue::Error(spill_err.clone());
22287            if let Some(d) = delta.as_deref_mut()
22288                && d.mode != DeltaMode::Off
22289            {
22290                let old = self
22291                    .read_cell_value(
22292                        self.graph.sheet_name(anchor.sheet_id),
22293                        anchor.coord.row() + 1,
22294                        anchor.coord.col() + 1,
22295                    )
22296                    .unwrap_or(LiteralValue::Empty);
22297                if old != spill_val {
22298                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22299                }
22300            }
22301            self.graph.update_vertex_value(vertex_id, spill_val.clone());
22302            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22303            return Ok(());
22304        }
22305
22306        let mut targets = Vec::new();
22307        for r in 0..h {
22308            for c in 0..w {
22309                targets.push(self.graph.make_cell_ref_internal(
22310                    sheet_id,
22311                    anchor.coord.row() + r,
22312                    anchor.coord.col() + c,
22313                ));
22314            }
22315        }
22316
22317        match self.spill_mgr.reserve(
22318            vertex_id,
22319            anchor,
22320            SpillShape { rows: h, cols: w },
22321            SpillMeta {
22322                epoch: self.recalc_epoch,
22323                config: self.config.spill,
22324            },
22325        ) {
22326            Ok(()) => {
22327                if let Err(e) = self.commit_spill_and_mirror(
22328                    vertex_id,
22329                    &targets,
22330                    rows.clone(),
22331                    delta.as_deref_mut(),
22332                    overwritable_formulas,
22333                ) {
22334                    self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22335                    let err_val = LiteralValue::Error(e.clone());
22336                    if let Some(d) = delta.as_deref_mut()
22337                        && d.mode != DeltaMode::Off
22338                    {
22339                        let old = self
22340                            .read_cell_value(
22341                                self.graph.sheet_name(anchor.sheet_id),
22342                                anchor.coord.row() + 1,
22343                                anchor.coord.col() + 1,
22344                            )
22345                            .unwrap_or(LiteralValue::Empty);
22346                        if old != err_val {
22347                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22348                        }
22349                    }
22350                    self.graph.update_vertex_value(vertex_id, err_val.clone());
22351                    self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
22352                    return Ok(());
22353                }
22354
22355                // Anchor shows the top-left value, like Excel
22356                let top_left = rows
22357                    .first()
22358                    .and_then(|r| r.first())
22359                    .cloned()
22360                    .unwrap_or(LiteralValue::Empty);
22361                self.graph.update_vertex_value(vertex_id, top_left.clone());
22362                self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
22363                Ok(())
22364            }
22365            Err(e) => {
22366                self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22367                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22368                    .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
22369                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22370                        expected_rows: h,
22371                        expected_cols: w,
22372                    });
22373                let spill_val = LiteralValue::Error(spill_err.clone());
22374                if let Some(d) = delta
22375                    && d.mode != DeltaMode::Off
22376                {
22377                    let old = self
22378                        .read_cell_value(
22379                            self.graph.sheet_name(anchor.sheet_id),
22380                            anchor.coord.row() + 1,
22381                            anchor.coord.col() + 1,
22382                        )
22383                        .unwrap_or(LiteralValue::Empty);
22384                    if old != spill_val {
22385                        d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22386                    }
22387                }
22388                self.graph.update_vertex_value(vertex_id, spill_val.clone());
22389                self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22390                Ok(())
22391            }
22392        }
22393    }
22394
22395    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
22396    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
22397        // Check if vertex exists
22398        if !self.graph.vertex_exists(vertex_id) {
22399            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
22400                .with_message(format!("Vertex not found: {vertex_id:?}")));
22401        }
22402
22403        // Get vertex kind and check if it needs evaluation
22404        let kind = self.graph.get_vertex_kind(vertex_id);
22405        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
22406
22407        let ast_id = match kind {
22408            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
22409                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
22410                    ast_id
22411                } else {
22412                    return Ok(LiteralValue::Number(0.0));
22413                }
22414            }
22415            VertexKind::Empty | VertexKind::Cell => {
22416                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
22417                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22418                    let row = cell_ref.coord.row() + 1;
22419                    let col = cell_ref.coord.col() + 1;
22420                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
22421                        return Ok(v);
22422                    }
22423                }
22424                return Ok(LiteralValue::Number(0.0));
22425            }
22426            VertexKind::NamedScalar => {
22427                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22428                    ExcelError::new(ExcelErrorKind::Name)
22429                        .with_message("Named range metadata missing".to_string())
22430                })?;
22431
22432                return match &named_range.definition {
22433                    NamedDefinition::Cell(cell_ref) => {
22434                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22435                        Ok(self
22436                            .get_cell_value(
22437                                sheet_name,
22438                                cell_ref.coord.row() + 1,
22439                                cell_ref.coord.col() + 1,
22440                            )
22441                            .unwrap_or(LiteralValue::Empty))
22442                    }
22443                    NamedDefinition::Literal(v) => Ok(v.clone()),
22444                    NamedDefinition::Formula { ast, .. } => {
22445                        let context_sheet = match named_range.scope {
22446                            NameScope::Sheet(id) => id,
22447                            NameScope::Workbook => sheet_id,
22448                        };
22449                        let sheet_name = self.graph.sheet_name(context_sheet);
22450                        let cell_ref = self
22451                            .graph
22452                            .get_cell_ref(vertex_id)
22453                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22454                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22455                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
22456                    }
22457                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
22458                        .with_message("Range-valued name evaluated as scalar".to_string())),
22459                };
22460            }
22461            VertexKind::NamedArray => {
22462                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22463                    ExcelError::new(ExcelErrorKind::Name)
22464                        .with_message("Named range metadata missing".to_string())
22465                })?;
22466
22467                return match &named_range.definition {
22468                    NamedDefinition::Range(range_ref) => {
22469                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
22470                            return Err(ExcelError::new(ExcelErrorKind::Ref)
22471                                .with_message("Named range cannot span sheets".to_string()));
22472                        }
22473                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
22474                        let sr0 = range_ref.start.coord.row();
22475                        let sc0 = range_ref.start.coord.col();
22476                        let er0 = range_ref.end.coord.row();
22477                        let ec0 = range_ref.end.coord.col();
22478                        if sr0 > er0 || sc0 > ec0 {
22479                            return Err(ExcelError::new(ExcelErrorKind::Ref)
22480                                .with_message("Invalid named range bounds".to_string()));
22481                        }
22482
22483                        let h = (er0 - sr0 + 1) as usize;
22484                        let w = (ec0 - sc0 + 1) as usize;
22485                        let cell_count = (h as u64).saturating_mul(w as u64);
22486                        if cell_count > self.config.spill.max_spill_cells as u64 {
22487                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
22488                                "Named range too large to materialize as an array".to_string(),
22489                            ));
22490                        }
22491
22492                        let mut rows = Vec::with_capacity(h);
22493                        for r0 in sr0..=er0 {
22494                            let mut row = Vec::with_capacity(w);
22495                            for c0 in sc0..=ec0 {
22496                                let v = self
22497                                    .get_cell_value(sheet_name, r0 + 1, c0 + 1)
22498                                    .unwrap_or(LiteralValue::Empty);
22499                                row.push(v);
22500                            }
22501                            rows.push(row);
22502                        }
22503                        Ok(LiteralValue::Array(rows))
22504                    }
22505                    NamedDefinition::Cell(cell_ref) => {
22506                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22507                        let row = cell_ref.coord.row() + 1;
22508                        let col = cell_ref.coord.col() + 1;
22509                        let v = self
22510                            .get_cell_value(sheet_name, row, col)
22511                            .unwrap_or(LiteralValue::Empty);
22512                        Ok(LiteralValue::Array(vec![vec![v]]))
22513                    }
22514                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
22515                    NamedDefinition::Formula { ast, .. } => {
22516                        let context_sheet = match named_range.scope {
22517                            NameScope::Sheet(id) => id,
22518                            NameScope::Workbook => sheet_id,
22519                        };
22520                        let sheet_name = self.graph.sheet_name(context_sheet);
22521                        let cell_ref = self
22522                            .graph
22523                            .get_cell_ref(vertex_id)
22524                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22525                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22526                        match interpreter.evaluate_ast(ast) {
22527                            Ok(cv) => {
22528                                let v = cv.into_literal();
22529                                match v {
22530                                    LiteralValue::Array(_) => Ok(v),
22531                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
22532                                }
22533                            }
22534                            Err(err) => Ok(LiteralValue::Error(err)),
22535                        }
22536                    }
22537                };
22538            }
22539            VertexKind::InfiniteRange
22540            | VertexKind::Range
22541            | VertexKind::External
22542            | VertexKind::Table => {
22543                // Not directly evaluatable here.
22544                return Ok(LiteralValue::Number(0.0));
22545            }
22546        };
22547
22548        // The interpreter uses a reference to the engine as the context
22549        let sheet_name = self.graph.sheet_name(sheet_id);
22550        let cell_ref = self
22551            .graph
22552            .get_cell_ref(vertex_id)
22553            .expect("cell ref for vertex");
22554        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22555
22556        interpreter
22557            .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
22558            .map(|cv| {
22559                crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
22560            })
22561    }
22562
22563    /// Get access to the shared thread pool for parallel evaluation
22564    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
22565        self.thread_pool.as_ref()
22566    }
22567}
22568
22569#[derive(Default)]
22570struct RowBoundsCache {
22571    snapshot: u64,
22572    // key: (sheet_id, col_idx)
22573    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
22574}
22575
22576impl RowBoundsCache {
22577    fn new(snapshot: u64) -> Self {
22578        Self {
22579            snapshot,
22580            map: Default::default(),
22581        }
22582    }
22583    fn get_row_bounds(
22584        &self,
22585        sheet_id: SheetId,
22586        col_idx: usize,
22587        snapshot: u64,
22588    ) -> Option<(Option<u32>, Option<u32>)> {
22589        if self.snapshot != snapshot {
22590            return None;
22591        }
22592        self.map.get(&(sheet_id as u32, col_idx)).copied()
22593    }
22594    fn put_row_bounds(
22595        &mut self,
22596        sheet_id: SheetId,
22597        col_idx: usize,
22598        snapshot: u64,
22599        bounds: (Option<u32>, Option<u32>),
22600    ) {
22601        if self.snapshot != snapshot {
22602            self.snapshot = snapshot;
22603            self.map.clear();
22604        }
22605        self.map.insert((sheet_id as u32, col_idx), bounds);
22606    }
22607}
22608
22609struct UsedAxisBoundsCache {
22610    snapshot: u64,
22611    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22612    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22613    #[cfg(test)]
22614    row_hits: std::sync::atomic::AtomicUsize,
22615    #[cfg(test)]
22616    row_misses: std::sync::atomic::AtomicUsize,
22617    #[cfg(test)]
22618    col_hits: std::sync::atomic::AtomicUsize,
22619    #[cfg(test)]
22620    col_misses: std::sync::atomic::AtomicUsize,
22621}
22622
22623impl UsedAxisBoundsCache {
22624    fn new(snapshot: u64) -> Self {
22625        Self {
22626            snapshot,
22627            row_bounds_by_col_span: Default::default(),
22628            col_bounds_by_row_span: Default::default(),
22629            #[cfg(test)]
22630            row_hits: std::sync::atomic::AtomicUsize::new(0),
22631            #[cfg(test)]
22632            row_misses: std::sync::atomic::AtomicUsize::new(0),
22633            #[cfg(test)]
22634            col_hits: std::sync::atomic::AtomicUsize::new(0),
22635            #[cfg(test)]
22636            col_misses: std::sync::atomic::AtomicUsize::new(0),
22637        }
22638    }
22639
22640    fn reset_for_snapshot(&mut self, snapshot: u64) {
22641        if self.snapshot != snapshot {
22642            self.snapshot = snapshot;
22643            self.row_bounds_by_col_span.clear();
22644            self.col_bounds_by_row_span.clear();
22645        }
22646    }
22647
22648    fn get_row_bounds(
22649        &self,
22650        sheet_id: SheetId,
22651        start_col: u32,
22652        end_col: u32,
22653        snapshot: u64,
22654    ) -> Option<Option<(u32, u32)>> {
22655        if self.snapshot != snapshot {
22656            return None;
22657        }
22658        let cached = self
22659            .row_bounds_by_col_span
22660            .get(&(sheet_id, start_col, end_col))
22661            .copied();
22662        #[cfg(test)]
22663        if cached.is_some() {
22664            self.row_hits.fetch_add(1, Ordering::Relaxed);
22665        }
22666        cached
22667    }
22668
22669    fn put_row_bounds(
22670        &mut self,
22671        sheet_id: SheetId,
22672        start_col: u32,
22673        end_col: u32,
22674        snapshot: u64,
22675        bounds: Option<(u32, u32)>,
22676    ) {
22677        self.reset_for_snapshot(snapshot);
22678        self.row_bounds_by_col_span
22679            .insert((sheet_id, start_col, end_col), bounds);
22680        #[cfg(test)]
22681        self.row_misses.fetch_add(1, Ordering::Relaxed);
22682    }
22683
22684    fn get_col_bounds(
22685        &self,
22686        sheet_id: SheetId,
22687        start_row: u32,
22688        end_row: u32,
22689        snapshot: u64,
22690    ) -> Option<Option<(u32, u32)>> {
22691        if self.snapshot != snapshot {
22692            return None;
22693        }
22694        let cached = self
22695            .col_bounds_by_row_span
22696            .get(&(sheet_id, start_row, end_row))
22697            .copied();
22698        #[cfg(test)]
22699        if cached.is_some() {
22700            self.col_hits.fetch_add(1, Ordering::Relaxed);
22701        }
22702        cached
22703    }
22704
22705    fn put_col_bounds(
22706        &mut self,
22707        sheet_id: SheetId,
22708        start_row: u32,
22709        end_row: u32,
22710        snapshot: u64,
22711        bounds: Option<(u32, u32)>,
22712    ) {
22713        self.reset_for_snapshot(snapshot);
22714        self.col_bounds_by_row_span
22715            .insert((sheet_id, start_row, end_row), bounds);
22716        #[cfg(test)]
22717        self.col_misses.fetch_add(1, Ordering::Relaxed);
22718    }
22719}
22720
22721// Phase 2 shim: in-process spill manager delegating to current graph methods.
22722#[derive(Default)]
22723pub struct ShimSpillManager {
22724    region_locks: RegionLockManager,
22725    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
22726}
22727
22728impl ShimSpillManager {
22729    pub(crate) fn reserve(
22730        &mut self,
22731        owner: VertexId,
22732        anchor_cell: CellRef,
22733        shape: SpillShape,
22734        _meta: SpillMeta,
22735    ) -> Result<(), ExcelError> {
22736        // Derive region from anchor + shape; enforce in-flight exclusivity only.
22737        let region = crate::engine::spill::Region {
22738            sheet_id: anchor_cell.sheet_id as u32,
22739            row_start: anchor_cell.coord.row(),
22740            row_end: anchor_cell
22741                .coord
22742                .row()
22743                .saturating_add(shape.rows)
22744                .saturating_sub(1),
22745            col_start: anchor_cell.coord.col(),
22746            col_end: anchor_cell
22747                .coord
22748                .col()
22749                .saturating_add(shape.cols)
22750                .saturating_sub(1),
22751        };
22752        match self.region_locks.reserve(region, owner) {
22753            Ok(id) => {
22754                if id != 0 {
22755                    self.active_locks.insert(owner, id);
22756                }
22757                Ok(())
22758            }
22759            Err(e) => Err(e),
22760        }
22761    }
22762
22763    /// Release any in-flight region reservation still held for `owner`.
22764    ///
22765    /// Reservations are normally released on commit/rollback, but if an anchor is
22766    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
22767    /// stale reservation could remain. This is a no-op when nothing is held.
22768    pub(crate) fn release_owner(&mut self, owner: VertexId) {
22769        if let Some(id) = self.active_locks.remove(&owner) {
22770            self.region_locks.release(id);
22771        }
22772    }
22773
22774    pub(crate) fn commit_array_with_value_probe<F>(
22775        &mut self,
22776        graph: &mut DependencyGraph,
22777        anchor_vertex: VertexId,
22778        targets: &[CellRef],
22779        rows: Vec<Vec<LiteralValue>>,
22780        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22781        mut value_probe: F,
22782    ) -> Result<(), ExcelError>
22783    where
22784        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
22785    {
22786        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
22787
22788        // Re-run plan on concrete targets before committing to respect blockers.
22789        // This plan checks formula/spill ownership in the graph, but when the graph value cache
22790        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
22791        let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
22792            anchor_vertex,
22793            targets,
22794            overwritable_formulas,
22795        );
22796        if let Err(e) = plan_res {
22797            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22798                self.region_locks.release(id);
22799            }
22800            return Err(e);
22801        }
22802
22803        if !graph.value_cache_enabled() {
22804            // Compute expected spill shape from the target rectangle for diagnostics.
22805            let (expected_rows, expected_cols) = if targets.is_empty() {
22806                (0u32, 0u32)
22807            } else {
22808                let mut min_r = u32::MAX;
22809                let mut max_r = 0u32;
22810                let mut min_c = u32::MAX;
22811                let mut max_c = 0u32;
22812                for cell in targets {
22813                    let r = cell.coord.row();
22814                    let c = cell.coord.col();
22815                    min_r = min_r.min(r);
22816                    max_r = max_r.max(r);
22817                    min_c = min_c.min(c);
22818                    max_c = max_c.max(c);
22819                }
22820                (
22821                    max_r.saturating_sub(min_r).saturating_add(1),
22822                    max_c.saturating_sub(min_c).saturating_add(1),
22823                )
22824            };
22825
22826            let anchor_cell = graph
22827                .get_cell_ref(anchor_vertex)
22828                .expect("anchor cell ref for spill commit");
22829
22830            for cell in targets {
22831                // Never treat the anchor as a blocker.
22832                if *cell == anchor_cell {
22833                    continue;
22834                }
22835                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
22836                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
22837                    continue;
22838                }
22839                // Skip formula vertices in the target region; plan() handled them (or allowed).
22840                if let Some(&vid) = graph.get_vertex_id_for_address(cell)
22841                    && vid != anchor_vertex
22842                {
22843                    match graph.get_vertex_kind(vid) {
22844                        crate::engine::vertex::VertexKind::FormulaScalar
22845                        | crate::engine::vertex::VertexKind::FormulaArray => {
22846                            // plan() already approved allowed overwrites.
22847                            continue;
22848                        }
22849                        _ => {}
22850                    }
22851                }
22852
22853                if let Some(v) = value_probe(graph, cell)
22854                    && !matches!(v, LiteralValue::Empty)
22855                {
22856                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22857                        self.region_locks.release(id);
22858                    }
22859                    return Err(ExcelError::new(ExcelErrorKind::Spill)
22860                        .with_message("BlockedByValue")
22861                        .with_extra(ExcelErrorExtra::Spill {
22862                            expected_rows,
22863                            expected_cols,
22864                        }));
22865                }
22866            }
22867        }
22868
22869        let commit_res = graph.commit_spill_region_atomic_with_fault(
22870            anchor_vertex,
22871            targets.to_vec(),
22872            rows,
22873            None,
22874        );
22875        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22876            self.region_locks.release(id);
22877        }
22878        commit_res.map(|_| ())
22879    }
22880
22881    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
22882    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
22883        &mut self,
22884        engine: &mut Engine<R>,
22885        anchor_vertex: VertexId,
22886        targets: &[CellRef],
22887        rows: Vec<Vec<LiteralValue>>,
22888        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22889    ) -> Result<(), ExcelError> {
22890        // Re-run plan on concrete targets before committing to respect blockers.
22891        let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
22892            anchor_vertex,
22893            targets,
22894            overwritable_formulas,
22895        );
22896        if let Err(e) = plan_res {
22897            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22898                self.region_locks.release(id);
22899            }
22900            return Err(e);
22901        }
22902
22903        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
22904            anchor_vertex,
22905            targets.to_vec(),
22906            rows.clone(),
22907            None,
22908        );
22909        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22910            self.region_locks.release(id);
22911        }
22912        commit_res.map(|_| ())?;
22913
22914        // Mirror into Arrow overlay when enabled
22915        if engine.config.arrow_storage_enabled
22916            && engine.config.delta_overlay_enabled
22917            && engine.config.write_formula_overlay_enabled
22918        {
22919            // Expect targets to be a contiguous rectangle row-major starting at some anchor
22920            for (idx, cell) in targets.iter().enumerate() {
22921                let (r_off, c_off) = {
22922                    if rows.is_empty() || rows[0].is_empty() {
22923                        (0usize, 0usize)
22924                    } else {
22925                        let width = rows[0].len();
22926                        (idx / width, idx % width)
22927                    }
22928                };
22929                let v = rows
22930                    .get(r_off)
22931                    .and_then(|r| r.get(c_off))
22932                    .cloned()
22933                    .unwrap_or(LiteralValue::Empty);
22934                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
22935                engine.mirror_value_to_computed_overlay(
22936                    &sheet_name,
22937                    cell.coord.row() + 1,
22938                    cell.coord.col() + 1,
22939                    &v,
22940                );
22941            }
22942        }
22943        Ok(())
22944    }
22945}
22946
22947impl<R> Engine<R>
22948where
22949    R: EvaluationContext,
22950{
22951    fn resolve_shared_ref(
22952        &self,
22953        reference: &ReferenceType,
22954        current_sheet: &str,
22955    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
22956        use formualizer_common::{
22957            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
22958            SheetRef as SharedRef,
22959        };
22960
22961        // Preserve anchor flags from the parsed reference when possible.
22962        let sr = match reference {
22963            ReferenceType::Cell {
22964                sheet,
22965                row,
22966                col,
22967                row_abs,
22968                col_abs,
22969            } => {
22970                let row0 = row
22971                    .checked_sub(1)
22972                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
22973                let col0 = col
22974                    .checked_sub(1)
22975                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
22976                let sheet_loc = match sheet.as_deref() {
22977                    Some(name) => SheetLocator::from_name(name),
22978                    None => SheetLocator::Current,
22979                };
22980                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
22981                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
22982            }
22983            ReferenceType::Range {
22984                sheet,
22985                start_row,
22986                start_col,
22987                end_row,
22988                end_col,
22989                start_row_abs,
22990                start_col_abs,
22991                end_row_abs,
22992                end_col_abs,
22993            } => {
22994                let sheet_loc = match sheet.as_deref() {
22995                    Some(name) => SheetLocator::from_name(name),
22996                    None => SheetLocator::Current,
22997                };
22998                let sr = start_row
22999                    .map(|r| {
23000                        r.checked_sub(1)
23001                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23002                    })
23003                    .transpose()?;
23004                let sc = start_col
23005                    .map(|c| {
23006                        c.checked_sub(1)
23007                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23008                    })
23009                    .transpose()?;
23010                let er = end_row
23011                    .map(|r| {
23012                        r.checked_sub(1)
23013                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23014                    })
23015                    .transpose()?;
23016                let ec = end_col
23017                    .map(|c| {
23018                        c.checked_sub(1)
23019                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23020                    })
23021                    .transpose()?;
23022                let range = SharedRangeRef::from_parts(
23023                    sheet_loc,
23024                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
23025                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
23026                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
23027                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
23028                )
23029                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
23030                SharedRef::Range(range)
23031            }
23032            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23033        };
23034
23035        let current_id = self
23036            .graph
23037            .sheet_id(current_sheet)
23038            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23039
23040        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
23041            match loc {
23042                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
23043                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
23044                SheetLocator::Name(name) => {
23045                    let n = name.as_ref();
23046                    self.graph
23047                        .sheet_id(n)
23048                        .map(SheetLocator::Id)
23049                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23050                }
23051            }
23052        };
23053
23054        match sr {
23055            SharedRef::Cell(cell) => {
23056                let owned = cell.into_owned();
23057                let sheet = resolve_loc(owned.sheet)?;
23058                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
23059            }
23060            SharedRef::Range(range) => {
23061                let owned = range.into_owned();
23062                let sheet = resolve_loc(owned.sheet)?;
23063                Ok(SharedRef::Range(SharedRangeRef {
23064                    sheet,
23065                    start_row: owned.start_row,
23066                    start_col: owned.start_col,
23067                    end_row: owned.end_row,
23068                    end_col: owned.end_col,
23069                }))
23070            }
23071        }
23072    }
23073}
23074
23075// Implement the resolver traits for the Engine.
23076// This allows the interpreter to resolve references by querying the engine's graph.
23077impl<R> crate::traits::ReferenceResolver for Engine<R>
23078where
23079    R: EvaluationContext,
23080{
23081    fn resolve_cell_reference(
23082        &self,
23083        sheet: Option<&str>,
23084        row: u32,
23085        col: u32,
23086    ) -> Result<LiteralValue, ExcelError> {
23087        // This context-free trait method has no knowledge of the formula's
23088        // current sheet, so an unqualified (`None`) reference cannot be resolved
23089        // here. Previously this fell back to `default_sheet_name()`, which leaked
23090        // the reference onto an unrelated sheet (issue #110). Interpreter paths
23091        // already qualify references with the current sheet before reaching this
23092        // method (see `Interpreter::implicit_intersection_from_reference`), and
23093        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
23094        // with an explicit `current_sheet`. Returning #REF! for an unqualified
23095        // reference here surfaces the missing context instead of silently
23096        // returning data from the wrong sheet.
23097        let Some(sheet_name) = sheet else {
23098            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23099                "Unqualified cell reference resolved without sheet context".to_string(),
23100            ));
23101        };
23102        // Prefer engine's unified accessor which consults Arrow store for base values
23103        // and falls back to graph for formulas and stored values.
23104        if let Some(v) = self.get_cell_value(sheet_name, row, col) {
23105            Ok(v)
23106        } else {
23107            // Excel semantics: empty cell coerces to 0 in numeric contexts
23108            Ok(LiteralValue::Number(0.0))
23109        }
23110    }
23111}
23112
23113impl<R> crate::traits::RangeResolver for Engine<R>
23114where
23115    R: EvaluationContext,
23116{
23117    fn resolve_range_reference(
23118        &self,
23119        sheet: Option<&str>,
23120        sr: Option<u32>,
23121        sc: Option<u32>,
23122        er: Option<u32>,
23123        ec: Option<u32>,
23124    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
23125        // For now, delegate range resolution to the external resolver.
23126        // A future optimization could be to handle this within the graph.
23127        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
23128    }
23129}
23130
23131impl<R> crate::traits::NamedRangeResolver for Engine<R>
23132where
23133    R: EvaluationContext,
23134{
23135    fn resolve_named_range_reference(
23136        &self,
23137        name: &str,
23138    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
23139        self.resolver.resolve_named_range_reference(name)
23140    }
23141}
23142
23143impl<R> crate::traits::TableResolver for Engine<R>
23144where
23145    R: EvaluationContext,
23146{
23147    fn resolve_table_reference(
23148        &self,
23149        tref: &formualizer_parse::parser::TableReference,
23150    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23151        self.resolver.resolve_table_reference(tref)
23152    }
23153}
23154
23155impl<R> crate::traits::SourceResolver for Engine<R>
23156where
23157    R: EvaluationContext,
23158{
23159    fn source_scalar_version(&self, name: &str) -> Option<u64> {
23160        self.resolver.source_scalar_version(name)
23161    }
23162
23163    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
23164        self.resolver.resolve_source_scalar(name)
23165    }
23166
23167    fn source_table_version(&self, name: &str) -> Option<u64> {
23168        self.resolver.source_table_version(name)
23169    }
23170
23171    fn resolve_source_table(
23172        &self,
23173        name: &str,
23174    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23175        self.resolver.resolve_source_table(name)
23176    }
23177}
23178
23179// The Engine is a Resolver because it implements the constituent traits.
23180impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
23181
23182// The Engine provides functions by delegating to its internal resolver.
23183impl<R> crate::traits::FunctionProvider for Engine<R>
23184where
23185    R: EvaluationContext,
23186{
23187    fn planning_semantic_revision(&self) -> Option<u64> {
23188        self.resolver.planning_semantic_revision()
23189    }
23190
23191    fn get_function(
23192        &self,
23193        prefix: &str,
23194        name: &str,
23195    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23196        self.resolver.get_function(prefix, name)
23197    }
23198
23199    fn get_function_for_planning(
23200        &self,
23201        prefix: &str,
23202        name: &str,
23203    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23204        self.resolver.get_function_for_planning(prefix, name)
23205    }
23206}
23207
23208impl<R> Engine<R>
23209where
23210    R: EvaluationContext,
23211{
23212    /// Semantic used coordinates exclude graph-only dependency placeholders.
23213    ///
23214    /// Non-empty base/overlay/computed cells come from Arrow storage, while
23215    /// scalar and array formulas come from graph formula kinds even before
23216    /// their results are materialized. The legacy graph fallback is omitted:
23217    /// `load_packed_to_vertex` entries are either represented by those sources
23218    /// or are `Empty` dependency placeholders, not a third value authority.
23219    pub(crate) fn semantic_used_rows_for_columns(
23220        &self,
23221        sheet: &str,
23222        start_col: u32,
23223        end_col: u32,
23224    ) -> Option<(u32, u32)> {
23225        let arrow_bounds = self
23226            .sheet_store()
23227            .sheet(sheet)
23228            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23229        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23230        Self::union_used_bounds(arrow_bounds, formula_bounds)
23231    }
23232
23233    pub(crate) fn semantic_used_cols_for_rows(
23234        &self,
23235        sheet: &str,
23236        start_row: u32,
23237        end_row: u32,
23238    ) -> Option<(u32, u32)> {
23239        let arrow_bounds = self
23240            .sheet_store()
23241            .sheet(sheet)
23242            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23243        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23244        Self::union_used_bounds(arrow_bounds, formula_bounds)
23245    }
23246}
23247
23248// Override EvaluationContext to provide thread pool access
23249impl<R> crate::traits::EvaluationContext for Engine<R>
23250where
23251    R: EvaluationContext,
23252{
23253    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
23254        &self.clock
23255    }
23256
23257    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
23258        self.thread_pool.as_ref()
23259    }
23260
23261    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
23262        self.active_cancel_flag.clone()
23263    }
23264
23265    fn chunk_hint(&self) -> Option<usize> {
23266        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
23267        let hint =
23268            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
23269        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
23270    }
23271
23272    fn volatile_level(&self) -> crate::traits::VolatileLevel {
23273        self.config.volatile_level
23274    }
23275
23276    fn workbook_seed(&self) -> u64 {
23277        self.config.workbook_seed
23278    }
23279
23280    fn recalc_epoch(&self) -> u64 {
23281        self.recalc_epoch
23282    }
23283
23284    fn workbook_sheet_count(&self) -> Option<usize> {
23285        Some(self.graph.sheet_reg().active_len())
23286    }
23287
23288    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
23289        self.graph.sheet_reg().active_position(sheet)
23290    }
23291
23292    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
23293        self.sheet_index_by_name(current_sheet)
23294    }
23295
23296    fn inspect_reference(
23297        &self,
23298        reference: &ReferenceType,
23299        current_sheet: &str,
23300    ) -> Result<Option<ReferenceInfo>, ExcelError> {
23301        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
23302            let sheet_id = self
23303                .graph
23304                .sheet_id(sheet_name)
23305                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23306            let sheet_index = self
23307                .graph
23308                .sheet_reg()
23309                .active_position_by_id(sheet_id)
23310                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23311            Ok((sheet_id, sheet_index))
23312        };
23313
23314        let cell_info =
23315            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
23316                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23317                let row0 = row
23318                    .checked_sub(1)
23319                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23320                let col0 = col
23321                    .checked_sub(1)
23322                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23323                Ok(ReferenceInfo {
23324                    first_sheet_index: Some(sheet_index),
23325                    sheet_count: Some(1),
23326                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23327                })
23328            };
23329
23330        let range_info = |sheet_name: &str,
23331                          start_row: Option<u32>,
23332                          start_col: Option<u32>|
23333         -> Result<ReferenceInfo, ExcelError> {
23334            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23335            let row = start_row.unwrap_or(1);
23336            let col = start_col.unwrap_or(1);
23337            let row0 = row
23338                .checked_sub(1)
23339                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23340            let col0 = col
23341                .checked_sub(1)
23342                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23343            Ok(ReferenceInfo {
23344                first_sheet_index: Some(sheet_index),
23345                sheet_count: Some(1),
23346                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23347            })
23348        };
23349
23350        let info = match reference {
23351            ReferenceType::Cell {
23352                sheet, row, col, ..
23353            } => {
23354                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23355                cell_info(sheet_name, *row, *col)?
23356            }
23357            ReferenceType::Range {
23358                sheet,
23359                start_row,
23360                start_col,
23361                ..
23362            } => {
23363                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23364                range_info(sheet_name, *start_row, *start_col)?
23365            }
23366            ReferenceType::Cell3D {
23367                sheet_first,
23368                sheet_last,
23369                row,
23370                col,
23371                ..
23372            } => {
23373                let first = cell_info(sheet_first, *row, *col)?;
23374                ReferenceInfo {
23375                    first_sheet_index: first.first_sheet_index,
23376                    sheet_count: self
23377                        .graph
23378                        .sheet_reg()
23379                        .active_span_len(sheet_first, sheet_last),
23380                    first_cell: first.first_cell,
23381                }
23382            }
23383            ReferenceType::Range3D {
23384                sheet_first,
23385                sheet_last,
23386                start_row,
23387                start_col,
23388                ..
23389            } => {
23390                let first = range_info(sheet_first, *start_row, *start_col)?;
23391                ReferenceInfo {
23392                    first_sheet_index: first.first_sheet_index,
23393                    sheet_count: self
23394                        .graph
23395                        .sheet_reg()
23396                        .active_span_len(sheet_first, sheet_last),
23397                    first_cell: first.first_cell,
23398                }
23399            }
23400            ReferenceType::NamedRange(name) => {
23401                let current_id = self
23402                    .graph
23403                    .sheet_id(current_sheet)
23404                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23405                let named = self
23406                    .graph
23407                    .resolve_name_entry(name, current_id)
23408                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23409                match &named.definition {
23410                    NamedDefinition::Cell(cell) => ReferenceInfo {
23411                        first_sheet_index: self
23412                            .graph
23413                            .sheet_reg()
23414                            .active_position_by_id(cell.sheet_id),
23415                        sheet_count: Some(1),
23416                        first_cell: Some(*cell),
23417                    },
23418                    NamedDefinition::Range(range) => ReferenceInfo {
23419                        first_sheet_index: self
23420                            .graph
23421                            .sheet_reg()
23422                            .active_position_by_id(range.start.sheet_id),
23423                        sheet_count: Some(1),
23424                        first_cell: Some(range.start),
23425                    },
23426                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
23427                        ReferenceInfo {
23428                            first_sheet_index: None,
23429                            sheet_count: None,
23430                            first_cell: None,
23431                        }
23432                    }
23433                }
23434            }
23435            ReferenceType::Table(tref) => {
23436                let table = self
23437                    .graph
23438                    .resolve_table_entry(&tref.name)
23439                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23440                ReferenceInfo {
23441                    first_sheet_index: self
23442                        .graph
23443                        .sheet_reg()
23444                        .active_position_by_id(table.range.start.sheet_id),
23445                    sheet_count: Some(1),
23446                    first_cell: Some(table.range.start),
23447                }
23448            }
23449            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23450        };
23451
23452        Ok(Some(info))
23453    }
23454
23455    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
23456        let sheet_name = self.graph.sheet_name(cell.sheet_id);
23457        if sheet_name.is_empty() {
23458            return Err(ExcelError::new(ExcelErrorKind::Ref));
23459        }
23460        let row = cell.coord.row() + 1;
23461        let col = cell.coord.col() + 1;
23462
23463        if let Some(entries) = self.staged_formulas.get(sheet_name)
23464            && let Some(text) = entries.get(row, col)
23465        {
23466            return Ok(Some(if text.starts_with('=') {
23467                text.to_owned()
23468            } else {
23469                format!("={text}")
23470            }));
23471        }
23472
23473        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
23474            return Ok(None);
23475        };
23476        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
23477    }
23478
23479    fn used_rows_for_columns(
23480        &self,
23481        sheet: &str,
23482        start_col: u32,
23483        end_col: u32,
23484    ) -> Option<(u32, u32)> {
23485        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
23486        let sheet_id = self.graph.sheet_id(sheet)?;
23487        let snap = self.data_snapshot_id();
23488        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23489            guard
23490                .as_ref()
23491                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
23492        }) {
23493            return cached;
23494        }
23495
23496        let arrow_bounds = self
23497            .sheet_store()
23498            .sheet(sheet)
23499            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23500        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23501        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23502            Some(bounds)
23503        } else {
23504            let sc0 = start_col.saturating_sub(1);
23505            let ec0 = end_col.saturating_sub(1);
23506            self.graph
23507                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
23508                .map(|(a0, b0)| (a0 + 1, b0 + 1))
23509        };
23510
23511        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23512            guard
23513                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23514                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
23515        }
23516
23517        computed
23518    }
23519
23520    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
23521        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
23522        let sheet_id = self.graph.sheet_id(sheet)?;
23523        let snap = self.data_snapshot_id();
23524        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23525            guard
23526                .as_ref()
23527                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
23528        }) {
23529            return cached;
23530        }
23531
23532        let arrow_bounds = self
23533            .sheet_store()
23534            .sheet(sheet)
23535            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23536        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23537        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23538            Some(bounds)
23539        } else {
23540            let sr0 = start_row.saturating_sub(1);
23541            let er0 = end_row.saturating_sub(1);
23542            self.graph
23543                .used_col_bounds_for_rows(sheet_id, sr0, er0)
23544                .map(|(a0, b0)| (a0 + 1, b0 + 1))
23545        };
23546
23547        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23548            guard
23549                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23550                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
23551        }
23552
23553        computed
23554    }
23555
23556    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
23557        let _ = self.graph.sheet_id(sheet)?;
23558        // Excel-like upper bounds; we expose something finite but large.
23559        // Backends may override with real bounds.
23560        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
23561    }
23562
23563    fn data_snapshot_id(&self) -> u64 {
23564        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
23565    }
23566
23567    fn backend_caps(&self) -> crate::traits::BackendCaps {
23568        crate::traits::BackendCaps {
23569            streaming: true,
23570            used_region: true,
23571            write: false,
23572            tables: false,
23573            async_stream: false,
23574        }
23575    }
23576
23577    fn build_lookup_index(
23578        &self,
23579        view: &RangeView<'_>,
23580        axis: LookupAxis,
23581    ) -> Option<Arc<LookupIndex>> {
23582        self.build_lookup_index_impl(view, axis)
23583    }
23584
23585    // Flats removed
23586
23587    fn date_system(&self) -> crate::engine::DateSystem {
23588        self.config.date_system
23589    }
23590    /// New: resolve a reference into a RangeView (Phase 2 API)
23591    fn resolve_range_view<'c>(
23592        &'c self,
23593        reference: &ReferenceType,
23594        current_sheet: &str,
23595    ) -> Result<RangeView<'c>, ExcelError> {
23596        match reference {
23597            ReferenceType::External(ext) => {
23598                let name = ext.raw.as_str();
23599                match ext.kind {
23600                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
23601                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
23602                            return Err(ExcelError::new(ExcelErrorKind::Name)
23603                                .with_message(format!("Undefined name: {name}")));
23604                        };
23605                        let version = source
23606                            .version
23607                            .or_else(|| self.resolver.source_scalar_version(name));
23608                        let v = self.resolve_source_scalar_cached(name, version)?;
23609                        Ok(RangeView::from_owned_rows(
23610                            vec![vec![v]],
23611                            self.config.date_system,
23612                        ))
23613                    }
23614                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
23615                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
23616                            return Err(ExcelError::new(ExcelErrorKind::Name)
23617                                .with_message(format!("Undefined table: {name}")));
23618                        };
23619                        let version = source
23620                            .version
23621                            .or_else(|| self.resolver.source_table_version(name));
23622                        let table = self.resolve_source_table_cached(name, version)?;
23623                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
23624                        self.source_table_to_range_view(table.as_ref(), &spec)
23625                    }
23626                }
23627            }
23628            ReferenceType::Range { .. } => {
23629                let shared = self.resolve_shared_ref(reference, current_sheet)?;
23630                let formualizer_common::SheetRef::Range(range) = shared else {
23631                    return Err(ExcelError::new(ExcelErrorKind::Ref));
23632                };
23633                let sheet_id = match range.sheet {
23634                    formualizer_common::SheetLocator::Id(id) => id,
23635                    _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23636                };
23637                let sheet_name = self.graph.sheet_name(sheet_id);
23638
23639                let bounded_range = if range.start_row.is_some()
23640                    && range.start_col.is_some()
23641                    && range.end_row.is_some()
23642                    && range.end_col.is_some()
23643                {
23644                    Some(RangeRef::try_from_shared(range.as_ref())?)
23645                } else {
23646                    None
23647                };
23648
23649                let sr = bounded_range
23650                    .as_ref()
23651                    .map(|r| r.start.coord.row() + 1)
23652                    .or_else(|| range.start_row.map(|b| b.index + 1));
23653                let sc = bounded_range
23654                    .as_ref()
23655                    .map(|r| r.start.coord.col() + 1)
23656                    .or_else(|| range.start_col.map(|b| b.index + 1));
23657                let er = bounded_range
23658                    .as_ref()
23659                    .map(|r| r.end.coord.row() + 1)
23660                    .or_else(|| range.end_row.map(|b| b.index + 1));
23661                let ec = bounded_range
23662                    .as_ref()
23663                    .map(|r| r.end.coord.col() + 1)
23664                    .or_else(|| range.end_col.map(|b| b.index + 1));
23665
23666                let extent = resolve_used_extent_with_fallback(
23667                    OpenRangeBounds {
23668                        start_row: sr,
23669                        start_column: sc,
23670                        end_row: er,
23671                        end_column: ec,
23672                    },
23673                    ExtentPolicy::EvaluationCompat {
23674                        fallback_row: None,
23675                        fallback_column: None,
23676                    },
23677                    || {
23678                        self.sheet_bounds(sheet_name)
23679                            .map(|_| self.config.max_open_ended_rows)
23680                    },
23681                    || {
23682                        self.sheet_bounds(sheet_name)
23683                            .map(|_| self.config.max_open_ended_cols)
23684                    },
23685                    |first, last| self.used_rows_for_columns(sheet_name, first, last),
23686                    |first, last| self.used_cols_for_rows(sheet_name, first, last),
23687                );
23688                let (sr, sc, er, ec) = extent
23689                    .map(|extent| {
23690                        (
23691                            extent.start_row,
23692                            extent.start_column,
23693                            extent.end_row,
23694                            extent.end_column,
23695                        )
23696                    })
23697                    .unwrap_or((1, 1, 0, 0));
23698
23699                if self.force_materialize_range_views {
23700                    if er < sr || ec < sc {
23701                        return Ok(RangeView::from_owned_rows(
23702                            Vec::new(),
23703                            self.config.date_system,
23704                        ));
23705                    }
23706                    let h = (er - sr + 1) as u64;
23707                    let w = (ec - sc + 1) as u64;
23708                    let cell_count = h.saturating_mul(w);
23709                    if cell_count <= self.config.spill.max_spill_cells as u64 {
23710                        let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
23711                        for r in sr..=er {
23712                            let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
23713                            for c in sc..=ec {
23714                                rowv.push(
23715                                    self.get_cell_value(sheet_name, r, c)
23716                                        .unwrap_or(LiteralValue::Empty),
23717                                );
23718                            }
23719                            rows.push(rowv);
23720                        }
23721                        return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
23722                    }
23723                }
23724
23725                let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
23726                    return Ok(RangeView::from_owned_rows(
23727                        Vec::new(),
23728                        self.config.date_system,
23729                    ));
23730                };
23731
23732                let rv = if er < sr || ec < sc {
23733                    asheet.range_view(1, 1, 0, 0)
23734                } else {
23735                    let sr0 = sr.saturating_sub(1) as usize;
23736                    let sc0 = sc.saturating_sub(1) as usize;
23737                    let er0 = er.saturating_sub(1) as usize;
23738                    let ec0 = ec.saturating_sub(1) as usize;
23739                    asheet.range_view(sr0, sc0, er0, ec0)
23740                };
23741
23742                Ok(rv)
23743            }
23744            ReferenceType::Cell { .. } => {
23745                let shared = self.resolve_shared_ref(reference, current_sheet)?;
23746                let formualizer_common::SheetRef::Cell(cell) = shared else {
23747                    return Err(ExcelError::new(ExcelErrorKind::Ref));
23748                };
23749                let addr = CellRef::try_from_shared(cell)?;
23750                let sheet_id = addr.sheet_id;
23751                let sheet_name = self.graph.sheet_name(sheet_id);
23752                let row = addr.coord.row() + 1;
23753                let col = addr.coord.col() + 1;
23754
23755                if self.force_materialize_range_views {
23756                    let v = self
23757                        .get_cell_value(sheet_name, row, col)
23758                        .unwrap_or(LiteralValue::Empty);
23759                    return Ok(RangeView::from_owned_rows(
23760                        vec![vec![v]],
23761                        self.config.date_system,
23762                    ));
23763                }
23764
23765                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
23766                    let r0 = row.saturating_sub(1) as usize;
23767                    let c0 = col.saturating_sub(1) as usize;
23768                    let rv = asheet.range_view(r0, c0, r0, c0);
23769                    Ok(rv)
23770                } else {
23771                    let v = self
23772                        .get_cell_value(sheet_name, row, col)
23773                        .unwrap_or(LiteralValue::Empty);
23774                    Ok(RangeView::from_owned_rows(
23775                        vec![vec![v]],
23776                        self.config.date_system,
23777                    ))
23778                }
23779            }
23780            ReferenceType::NamedRange(name) => {
23781                if let Some(current_id) = self.graph.sheet_id(current_sheet)
23782                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
23783                {
23784                    match &named.definition {
23785                        NamedDefinition::Cell(cell_ref) => {
23786                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23787                            if self.force_materialize_range_views {
23788                                let v = self
23789                                    .get_cell_value(
23790                                        sheet_name,
23791                                        cell_ref.coord.row() + 1,
23792                                        cell_ref.coord.col() + 1,
23793                                    )
23794                                    .unwrap_or(LiteralValue::Empty);
23795                                return Ok(RangeView::from_owned_rows(
23796                                    vec![vec![v]],
23797                                    self.config.date_system,
23798                                ));
23799                            } else {
23800                                let asheet = self
23801                                    .sheet_store()
23802                                    .sheet(sheet_name)
23803                                    .expect("Arrow sheet missing for named cell");
23804                                let r0 = cell_ref.coord.row() as usize;
23805                                let c0 = cell_ref.coord.col() as usize;
23806                                let rv = asheet.range_view(r0, c0, r0, c0);
23807                                return Ok(rv);
23808                            }
23809                        }
23810                        NamedDefinition::Range(range_ref) => {
23811                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
23812                            let sr = range_ref.start.coord.row() + 1;
23813                            let sc = range_ref.start.coord.col() + 1;
23814                            let er = range_ref.end.coord.row() + 1;
23815                            let ec = range_ref.end.coord.col() + 1;
23816                            if self.force_materialize_range_views {
23817                                let h = (er.saturating_sub(sr) + 1) as u64;
23818                                let w = (ec.saturating_sub(sc) + 1) as u64;
23819                                let cell_count = h.saturating_mul(w);
23820                                if cell_count <= self.config.spill.max_spill_cells as u64 {
23821                                    let mut rows: Vec<Vec<LiteralValue>> =
23822                                        Vec::with_capacity(h as usize);
23823                                    for r in sr..=er {
23824                                        let mut rowv: Vec<LiteralValue> =
23825                                            Vec::with_capacity(w as usize);
23826                                        for c in sc..=ec {
23827                                            rowv.push(
23828                                                self.get_cell_value(sheet_name, r, c)
23829                                                    .unwrap_or(LiteralValue::Empty),
23830                                            );
23831                                        }
23832                                        rows.push(rowv);
23833                                    }
23834                                    return Ok(RangeView::from_owned_rows(
23835                                        rows,
23836                                        self.config.date_system,
23837                                    ));
23838                                }
23839                            }
23840                            let asheet = self
23841                                .sheet_store()
23842                                .sheet(sheet_name)
23843                                .expect("Arrow sheet missing for named range");
23844                            let sr0 = range_ref.start.coord.row() as usize;
23845                            let sc0 = range_ref.start.coord.col() as usize;
23846                            let er0 = range_ref.end.coord.row() as usize;
23847                            let ec0 = range_ref.end.coord.col() as usize;
23848                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
23849                            return Ok(rv);
23850                        }
23851                        NamedDefinition::Literal(v) => {
23852                            return Ok(RangeView::from_owned_rows(
23853                                vec![vec![v.clone()]],
23854                                self.config.date_system,
23855                            ));
23856                        }
23857                        NamedDefinition::Formula { .. } => {
23858                            if let Some(value) = self.graph.get_value(named.vertex) {
23859                                return Ok(RangeView::from_owned_rows(
23860                                    vec![vec![value]],
23861                                    self.config.date_system,
23862                                ));
23863                            }
23864                        }
23865                    }
23866                }
23867
23868                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
23869                    let version = source
23870                        .version
23871                        .or_else(|| self.resolver.source_scalar_version(name));
23872                    let v = self.resolve_source_scalar_cached(name, version)?;
23873                    return Ok(RangeView::from_owned_rows(
23874                        vec![vec![v]],
23875                        self.config.date_system,
23876                    ));
23877                }
23878
23879                let data = self.resolver.resolve_named_range_reference(name)?;
23880                Ok(RangeView::from_owned_rows(data, self.config.date_system))
23881            }
23882            ReferenceType::Table(tref) => {
23883                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
23884                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
23885                    let asheet = self
23886                        .sheet_store()
23887                        .sheet(sheet_name)
23888                        .expect("Arrow sheet missing for table reference");
23889
23890                    let sr0 = table.range.start.coord.row() as usize;
23891                    let sc0 = table.range.start.coord.col() as usize;
23892                    let er0 = table.range.end.coord.row() as usize;
23893                    let ec0 = table.range.end.coord.col() as usize;
23894
23895                    let has_totals = table.totals_row;
23896                    let has_headers = table.header_row;
23897                    let data_sr = if has_headers {
23898                        sr0.saturating_add(1)
23899                    } else {
23900                        sr0
23901                    };
23902                    let data_er = if has_totals {
23903                        er0.saturating_sub(1)
23904                    } else {
23905                        er0
23906                    };
23907
23908                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
23909                        if sr > er || sc > ec {
23910                            asheet.range_view(1, 1, 0, 0)
23911                        } else {
23912                            asheet.range_view(sr, sc, er, ec)
23913                        }
23914                    };
23915
23916                    let av = match &tref.specifier {
23917                        None => {
23918                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23919                                "Table reference without specifier is unsupported".to_string(),
23920                            ));
23921                        }
23922                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
23923                            let Some(idx) = table.col_index(col) else {
23924                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23925                                    "Column refers to unknown table column".to_string(),
23926                                ));
23927                            };
23928                            let c0 = sc0 + idx;
23929                            select(data_sr, c0, data_er, c0)
23930                        }
23931                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
23932                            start,
23933                            end,
23934                        )) => {
23935                            let Some(si) = table.col_index(start) else {
23936                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23937                                    "Column range refers to unknown column(s)".to_string(),
23938                                ));
23939                            };
23940                            let Some(ei) = table.col_index(end) else {
23941                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23942                                    "Column range refers to unknown column(s)".to_string(),
23943                                ));
23944                            };
23945                            let (mut a, mut b) = (si, ei);
23946                            if a > b {
23947                                std::mem::swap(&mut a, &mut b);
23948                            }
23949                            let c_start = sc0 + a;
23950                            let c_end = sc0 + b;
23951                            select(data_sr, c_start, data_er, c_end)
23952                        }
23953                        Some(formualizer_parse::parser::TableSpecifier::All)
23954                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23955                            formualizer_parse::parser::SpecialItem::All,
23956                        )) => select(sr0, sc0, er0, ec0),
23957                        Some(formualizer_parse::parser::TableSpecifier::Data)
23958                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23959                            formualizer_parse::parser::SpecialItem::Data,
23960                        )) => select(data_sr, sc0, data_er, ec0),
23961                        Some(formualizer_parse::parser::TableSpecifier::Headers)
23962                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23963                            formualizer_parse::parser::SpecialItem::Headers,
23964                        )) => {
23965                            if !has_headers {
23966                                asheet.range_view(1, 1, 0, 0)
23967                            } else {
23968                                select(sr0, sc0, sr0, ec0)
23969                            }
23970                        }
23971                        Some(formualizer_parse::parser::TableSpecifier::Totals)
23972                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23973                            formualizer_parse::parser::SpecialItem::Totals,
23974                        )) => {
23975                            if !has_totals {
23976                                asheet.range_view(1, 1, 0, 0)
23977                            } else {
23978                                select(er0, sc0, er0, ec0)
23979                            }
23980                        }
23981                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
23982                            formualizer_parse::parser::SpecialItem::ThisRow,
23983                        )) => {
23984                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23985                                "@ (This Row) requires table-aware context; not yet supported"
23986                                    .to_string(),
23987                            ));
23988                        }
23989                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
23990                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
23991                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23992                                "Complex structured references not yet supported".to_string(),
23993                            ));
23994                        }
23995                    };
23996
23997                    return Ok(av);
23998                }
23999
24000                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
24001                    let version = source
24002                        .version
24003                        .or_else(|| self.resolver.source_table_version(&tref.name));
24004                    let table = self.resolve_source_table_cached(&tref.name, version)?;
24005                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
24006                }
24007
24008                // Fallback: materialize via Resolver::resolve_range_like tranche 1
24009                let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
24010                let owned = boxed.materialise().into_owned();
24011                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
24012            }
24013            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
24014                Err(ExcelError::new(ExcelErrorKind::NImpl)
24015                    .with_message("3D references are not yet supported".to_string()))
24016            }
24017        }
24018    }
24019
24020    fn resolve_cell_reference_value(
24021        &self,
24022        sheet: Option<&str>,
24023        row: u32,
24024        col: u32,
24025        current_sheet: &str,
24026    ) -> Result<LiteralValue, ExcelError> {
24027        let sheet_name = sheet.unwrap_or(current_sheet);
24028        if self.graph.sheet_id(sheet_name).is_none() {
24029            return Err(ExcelError::new(ExcelErrorKind::Ref));
24030        }
24031        Ok(self
24032            .get_cell_value(sheet_name, row, col)
24033            .unwrap_or(LiteralValue::Empty))
24034    }
24035
24036    fn build_criteria_mask(
24037        &self,
24038        view: &RangeView<'_>,
24039        col_in_view: usize,
24040        pred: &crate::args::CriteriaPredicate,
24041    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24042        if view.dims().1 == 0 {
24043            return None;
24044        }
24045        // If the view is logically open-ended but the backing sheet has no physical rows,
24046        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
24047        let sheet_rows = view.sheet().nrows as usize;
24048        if sheet_rows == 0 || view.start_row() >= sheet_rows {
24049            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
24050        }
24051        compute_criteria_mask(view, col_in_view, pred)
24052    }
24053
24054    fn build_row_visibility_mask(
24055        &self,
24056        view: &RangeView<'_>,
24057        mode: VisibilityMaskMode,
24058    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24059        self.build_row_visibility_mask_for_view(view, mode)
24060    }
24061}
24062
24063impl<R> Engine<R>
24064where
24065    R: EvaluationContext,
24066{
24067    fn clear_spill_projection_and_mirror(
24068        &mut self,
24069        anchor_vertex: VertexId,
24070        delta: Option<&mut DeltaCollector>,
24071    ) {
24072        let spill_cells = self
24073            .graph
24074            .spill_cells_for_anchor(anchor_vertex)
24075            .map(|cells| cells.to_vec())
24076            .unwrap_or_default();
24077        if spill_cells.is_empty() {
24078            return;
24079        }
24080
24081        if let Some(delta) = delta
24082            && delta.mode != DeltaMode::Off
24083        {
24084            let empty = LiteralValue::Empty;
24085            for cell in spill_cells.iter() {
24086                let sheet_name = self.graph.sheet_name(cell.sheet_id);
24087                let old = self
24088                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24089                    .unwrap_or(LiteralValue::Empty);
24090                if old != empty {
24091                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24092                }
24093            }
24094        }
24095
24096        self.graph.clear_spill_region(anchor_vertex);
24097        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24098            self.record_formula_plane_structural_change(scope);
24099        }
24100
24101        if self.config.arrow_storage_enabled
24102            && self.config.delta_overlay_enabled
24103            && self.config.write_formula_overlay_enabled
24104        {
24105            let empty = LiteralValue::Empty;
24106            for cell in spill_cells.iter() {
24107                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24108                self.mirror_value_to_computed_overlay(
24109                    &sheet_name,
24110                    cell.coord.row() + 1,
24111                    cell.coord.col() + 1,
24112                    &empty,
24113                );
24114            }
24115        }
24116    }
24117
24118    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
24119    /// (optionally filtered) members via `stamp_cycle_error`.
24120    ///
24121    /// This is the single per-SCC application point used by every schedule
24122    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
24123    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
24124    ///
24125    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
24126    /// only members present in the set are stamped.
24127    ///
24128    /// Returns the number of vertices stamped (0 when a filter excludes every
24129    /// member), so callers can keep their site-specific `cycle_errors`
24130    /// accounting.
24131    fn apply_cycle_outcome(
24132        &mut self,
24133        cycle: &[VertexId],
24134        mut delta: Option<&mut DeltaCollector>,
24135        dirty_filter: Option<&FxHashSet<VertexId>>,
24136    ) -> usize {
24137        let circ_error = LiteralValue::Error(
24138            ExcelError::new(ExcelErrorKind::Circ)
24139                .with_message("Circular dependency detected".to_string()),
24140        );
24141        let mut stamped = 0usize;
24142        for &vertex_id in cycle {
24143            if let Some(filter) = dirty_filter
24144                && !filter.contains(&vertex_id)
24145            {
24146                continue;
24147            }
24148            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
24149            stamped += 1;
24150        }
24151        stamped
24152    }
24153
24154    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
24155    ///
24156    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
24157    /// vertex previously anchored: it clears the spilled cells, releases the graph
24158    /// spill registry, drops any lingering region reservation, and mirrors the
24159    /// cleared cells into the computed overlay — the same teardown a normal scalar/
24160    /// error result performs (see `apply_non_array_result_from_parallel` /
24161    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
24162    /// spill anchor would leave stale spilled values and a reserved region behind
24163    /// (issue #111).
24164    ///
24165    /// When `delta` is provided, the cleared spill cells are recorded (by
24166    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
24167    /// recorded here, matching how other result paths emit deltas.
24168    fn stamp_cycle_error(
24169        &mut self,
24170        vertex_id: VertexId,
24171        circ_error: &LiteralValue,
24172        mut delta: Option<&mut DeltaCollector>,
24173    ) {
24174        // Tear down any previous spill projection/region before overwriting the anchor.
24175        if self.graph.spill_registry_has_anchor(vertex_id) {
24176            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24177        }
24178        // Drop any reservation that was never committed (defensive; normally released
24179        // on the prior successful commit).
24180        self.spill_mgr.release_owner(vertex_id);
24181
24182        // Record the anchor's own #CIRC delta, like other result paths.
24183        if let Some(d) = delta
24184            && d.mode != DeltaMode::Off
24185            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24186        {
24187            let sheet_name = self.graph.sheet_name(cell.sheet_id);
24188            let old = self
24189                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24190                .unwrap_or(LiteralValue::Empty);
24191            if old != *circ_error {
24192                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24193            }
24194        }
24195
24196        self.graph
24197            .update_vertex_value(vertex_id, circ_error.clone());
24198        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
24199    }
24200
24201    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
24202    ///
24203    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
24204    ///   `#CIRC!` on the (optionally dirty-filtered) members.
24205    /// * `CycleDetection::Runtime` — evaluate the SCC via
24206    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
24207    ///   no dirty member → skip the task entirely (values stand); any dirty
24208    ///   member → the whole SCC evaluates (an SCC cannot be partially
24209    ///   evaluated).
24210    ///
24211    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
24212    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
24213    fn handle_cycle_unit(
24214        &mut self,
24215        cycle: &[VertexId],
24216        mut delta: Option<&mut DeltaCollector>,
24217        dirty_filter: Option<&FxHashSet<VertexId>>,
24218        cancel_flag: Option<&AtomicBool>,
24219    ) -> Result<usize, ExcelError> {
24220        self.resource_checkpoint(cycle.len() as u64)?;
24221        match self.config.cycle.detection {
24222            CycleDetection::Static => {
24223                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
24224            }
24225            CycleDetection::Runtime => {
24226                if let Some(filter) = dirty_filter
24227                    && !cycle.iter().any(|v| filter.contains(v))
24228                {
24229                    return Ok(0);
24230                }
24231                // Both policies share `evaluate_scc_unit`; they differ only
24232                // in the settle loop's live-cycle arm (Error stamps,
24233                // Iterate keeps passing — RFC #113).
24234                self.evaluate_scc_unit(cycle, delta, cancel_flag)
24235            }
24236        }
24237    }
24238
24239    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
24240    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
24241    /// spec §3; Iterate policy arm per RFC #113).
24242    ///
24243    /// Phantom SCCs (live-acyclic) produce ordinary values under both
24244    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
24245    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
24246    /// (converge per spec §6 or cap at `max_iterations` passes) under
24247    /// `CyclePolicy::Iterate`. Runs sequentially on the
24248    /// coordinating thread; commits are write-through per member (no
24249    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
24250    /// always flushed before a Cycle unit runs, G1), so later members' scalar
24251    /// *and* range reads observe earlier members' results through the overlay
24252    /// cascade. Deltas are recorded once per member at end of task (G11).
24253    ///
24254    /// Returns the number of vertices stamped `#CIRC!`.
24255    ///
24256    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
24257    /// that ingest-time cycle rejection makes unreachable via public edits.
24258    pub(crate) fn evaluate_scc_unit(
24259        &mut self,
24260        cycle: &[VertexId],
24261        mut delta: Option<&mut DeltaCollector>,
24262        cancel_flag: Option<&AtomicBool>,
24263    ) -> Result<usize, ExcelError> {
24264        struct SccMember {
24265            vertex: VertexId,
24266            cell: Option<CellRef>,
24267        }
24268
24269        let task_start = crate::instant::FzInstant::now();
24270
24271        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
24272        // name vertices after, lexicographic by folded canonical name; any
24273        // other vertex kind (defensive — `get_evaluation_vertices` only emits
24274        // formula/name kinds) last by id, never evaluated.
24275        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
24276        let mut name_members: Vec<(VertexId, String)> = Vec::new();
24277        let mut other_members: Vec<VertexId> = Vec::new();
24278        for &v in cycle {
24279            match self.graph.get_vertex_kind(v) {
24280                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24281                    match self.graph.get_cell_ref(v) {
24282                        Some(cell) => cell_members.push((v, cell)),
24283                        None => other_members.push(v),
24284                    }
24285                }
24286                VertexKind::NamedScalar | VertexKind::NamedArray => {
24287                    match self.graph.name_key_for_vertex(v) {
24288                        Some(key) => name_members.push((v, key)),
24289                        None => other_members.push(v),
24290                    }
24291                }
24292                _ => other_members.push(v),
24293            }
24294        }
24295        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
24296        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
24297        other_members.sort_unstable();
24298
24299        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
24300        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
24301        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
24302        for (v, c) in &cell_members {
24303            members.push(SccMember {
24304                vertex: *v,
24305                cell: Some(*c),
24306            });
24307        }
24308        for (v, _) in &name_members {
24309            members.push(SccMember {
24310                vertex: *v,
24311                cell: None,
24312            });
24313        }
24314        for v in &other_members {
24315            members.push(SccMember {
24316                vertex: *v,
24317                cell: None,
24318            });
24319        }
24320        let n = members.len();
24321        // Indices addressable by the collector (cells + names); `other`
24322        // members can be neither edge sources nor targets.
24323        let recordable = cell_refs.len() + name_keys.len();
24324
24325        let circ_error = LiteralValue::Error(
24326            ExcelError::new(ExcelErrorKind::Circ)
24327                .with_message("Circular dependency detected".to_string()),
24328        );
24329
24330        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
24331        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
24332        // an iterating member's committed value is cycle STATE, not a
24333        // recomputable cache — and in canonical mode the overlay is its ONLY
24334        // home. If the overlay entry vanished since the last recalc, re-seed
24335        // it from the end-of-recalc snapshot (`iterative_state_values`) so
24336        // pass-1 reads (scalar AND range, via the overlay cascade) observe
24337        // the persisted value instead of silently restarting at Empty→0.
24338        // (Found by the iterate edge corpus: inserting/deleting an unrelated
24339        // row reset accumulators, violating spec §4/§7.15.)
24340        if !self.iterative_state_values.is_empty() {
24341            let restore: Vec<(VertexId, LiteralValue)> = members
24342                .iter()
24343                .filter_map(|m| {
24344                    let cell = m.cell?;
24345                    let persisted = self.iterative_state_values.get(&m.vertex)?;
24346                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24347                    let overlay = self
24348                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24349                        .unwrap_or(LiteralValue::Empty);
24350                    if matches!(overlay, LiteralValue::Empty) {
24351                        Some((m.vertex, persisted.clone()))
24352                    } else {
24353                        None
24354                    }
24355                })
24356                .collect();
24357            for (vertex, value) in restore {
24358                self.mirror_vertex_value_to_overlay(vertex, &value);
24359            }
24360        }
24361
24362        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
24363        // graph value map may be evicted in value-cache-disabled mode).
24364        let snapshot: Vec<LiteralValue> = members
24365            .iter()
24366            .map(|m| match m.cell {
24367                Some(cell) => {
24368                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24369                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24370                        .unwrap_or(LiteralValue::Empty)
24371                }
24372                None => self
24373                    .graph
24374                    .get_value(m.vertex)
24375                    .unwrap_or(LiteralValue::Empty),
24376            })
24377            .collect();
24378
24379        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
24380        // with full spill teardown (spec §7.9, #115) and excluded from
24381        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
24382        // and propagate). Non-evaluable defensive members are excluded too.
24383        let mut excluded = vec![false; n];
24384        let mut last_value = snapshot.clone();
24385        let mut stamped = 0usize;
24386        for (i, m) in members.iter().enumerate() {
24387            match self.graph.get_vertex_kind(m.vertex) {
24388                VertexKind::FormulaArray => {
24389                    // Deltas for the cleared spill-region cells (non-members)
24390                    // can only be recorded here; the anchor's own delta is
24391                    // covered by the end-of-task snapshot comparison (dedup).
24392                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
24393                    excluded[i] = true;
24394                    last_value[i] = circ_error.clone();
24395                    stamped += 1;
24396                }
24397                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
24398                _ => excluded[i] = true,
24399            }
24400        }
24401
24402        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
24403
24404        // Per-member live out-edges, refreshed whenever a member re-runs.
24405        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
24406        // Position of each member in the most recent pass (-1 = did not run).
24407        let mut pos: Vec<i64> = vec![-1; n];
24408        // Whether each member's committed value changed in the most recent pass.
24409        let mut changed = vec![false; n];
24410
24411        // Evaluate-and-commit one member; returns Ok(true) when the member was
24412        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
24413        macro_rules! run_member {
24414            ($i:expr) => {{
24415                let i: usize = $i;
24416                let m = &members[i];
24417                if i < recordable {
24418                    collector.set_current(i as u32);
24419                }
24420                let value = {
24421                    let ctx = RecordingContext::new(&*self, &collector);
24422                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
24423                        Ok(v) => v,
24424                        Err(e) => LiteralValue::Error(e),
24425                    }
24426                };
24427                let is_cell_formula = m.cell.is_some();
24428                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
24429                    // A member that *would* spill inside an SCC gets the
24430                    // conservative §7.9 verdict. It has never spilled before
24431                    // (a prior spill would make it FormulaArray, pre-stamped
24432                    // above), so there is no projection to tear down.
24433                    self.stamp_cycle_error(m.vertex, &circ_error, None);
24434                    excluded[i] = true;
24435                    stamped += 1;
24436                    changed[i] = last_value[i] != circ_error;
24437                    last_value[i] = circ_error.clone();
24438                } else {
24439                    self.graph.update_vertex_value(m.vertex, value.clone());
24440                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
24441                    // §7.14 invariant (G2): a formula member must never be
24442                    // shadowed by a user/delta overlay entry, or iteration
24443                    // reads would silently diverge from committed values.
24444                    #[cfg(debug_assertions)]
24445                    if let Some(cell) = m.cell {
24446                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24447                        debug_assert!(
24448                            self.read_delta_overlay_cell(
24449                                &sheet_name,
24450                                cell.coord.row() + 1,
24451                                cell.coord.col() + 1
24452                            )
24453                            .is_none(),
24454                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
24455                            cell.coord.row() + 1,
24456                            cell.coord.col() + 1
24457                        );
24458                    }
24459                    changed[i] = last_value[i] != value;
24460                    last_value[i] = value;
24461                }
24462            }};
24463        }
24464
24465        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
24466            if let Some(flag) = flag
24467                && flag.load(Ordering::Relaxed)
24468            {
24469                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24470                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
24471            }
24472            Ok(())
24473        };
24474
24475        // ── 3. Pass 1: all evaluable members in member order.
24476        check_cancel(cancel_flag)?;
24477        let mut passes = 1usize;
24478        {
24479            let mut p = 0i64;
24480            for i in 0..n {
24481                if excluded[i] {
24482                    continue;
24483                }
24484                run_member!(i);
24485                pos[i] = p;
24486                p += 1;
24487            }
24488        }
24489
24490        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
24491        //
24492        // Acyclic classifications settle stale readers exactly (identical
24493        // under both policies — phantom SCCs never iterate). A witnessed
24494        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
24495        // stops; `Iterate` keeps running full passes over all members in
24496        // member order until converged (spec §6) or capped at
24497        // `max_iterations` total passes. A live cycle that only appears
24498        // mid-settle takes the same arm, and a cycle that dissolves
24499        // mid-iteration falls back to exact acyclic settling.
24500        //
24501        // Defensive acyclic budget: the acyclic settle is monotone, so more
24502        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
24503        // remainder and set telemetry. Tracked via `settle_passes` so
24504        // iteration passes (legitimately many) don't consume the budget.
24505        let policy = self.config.cycle.policy;
24506        let cap = n + 2;
24507        let mut witnessed_cycles = 0usize;
24508        let mut capped = false;
24509        // ── Iterate-policy state ──
24510        let mut iterating = false;
24511        let mut converged = false;
24512        // Values committed by the last *full* pass; `None` until the first
24513        // iteration pass runs (pass 1 has no predecessor to compare against)
24514        // and reset when a settle pass runs (no cross-kind comparisons).
24515        let mut prev_pass: Option<Vec<LiteralValue>> = None;
24516        // Final-round convergence stats (overwritten per round so the values
24517        // reported are the ones observed at stop).
24518        let mut iter_max_delta = 0f64;
24519        let mut iter_nan_converged = 0usize;
24520        // Acyclic stale-reader re-eval passes (defensive budget; under pure
24521        // Error flow `1 + settle_passes == passes`, preserving Stage-2
24522        // behavior exactly).
24523        let mut settle_passes = 0usize;
24524        loop {
24525            // Drain this pass's recordings; members that ran replace their
24526            // out-edge set, members that didn't keep last-known edges.
24527            let drained = collector.take_edges();
24528            for i in 0..n {
24529                if pos[i] >= 0 {
24530                    out_edges[i].clear();
24531                }
24532            }
24533            for (from, to) in drained {
24534                debug_assert!(
24535                    pos[from as usize] >= 0,
24536                    "edge from a member that did not run"
24537                );
24538                out_edges[from as usize].push(to);
24539            }
24540            let mut edges: Vec<(u32, u32)> = Vec::new();
24541            for (i, outs) in out_edges.iter().enumerate() {
24542                if excluded[i] {
24543                    continue;
24544                }
24545                for &t in outs {
24546                    edges.push((i as u32, t));
24547                }
24548            }
24549            edges.sort_unstable();
24550            edges.dedup();
24551
24552            let analysis = analyze_live_graph(n, &edges);
24553
24554            if analysis.cycle_count > 0 {
24555                // Classification repeats every iteration pass under
24556                // `Iterate`; record the widest single witness instead of
24557                // accumulating so the count stays "distinct live cycles".
24558                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
24559                match policy {
24560                    CyclePolicy::Error => {
24561                        // POLICY (Error): stamp every member of a live cycle,
24562                        // then one settling pass over the remaining members in
24563                        // live-topological order so error propagation
24564                        // downstream is consistent (spec §3.4). Blast radius =
24565                        // live cycles only.
24566                        for i in 0..n {
24567                            if analysis.in_cycle[i] && !excluded[i] {
24568                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
24569                                excluded[i] = true;
24570                                last_value[i] = circ_error.clone();
24571                                stamped += 1;
24572                            }
24573                        }
24574                        check_cancel(cancel_flag)?;
24575                        let order: Vec<usize> = analysis
24576                            .topo
24577                            .iter()
24578                            .map(|&i| i as usize)
24579                            .filter(|&i| !excluded[i])
24580                            .collect();
24581                        if !order.is_empty() {
24582                            passes += 1;
24583                            for i in order {
24584                                run_member!(i);
24585                            }
24586                        }
24587                        break;
24588                    }
24589                    CyclePolicy::Iterate {
24590                        max_iterations,
24591                        max_change,
24592                    } => {
24593                        // POLICY (Iterate), spec §3.5/§6.
24594                        iterating = true;
24595
24596                        // Convergence test: the full pass that just completed
24597                        // vs the previous full pass, per the spec-§6 rules.
24598                        // `prev_pass` is `None` until an iteration pass has
24599                        // run — pass 1 has no predecessor, so no convergence
24600                        // test occurs before the second pass (spec §6).
24601                        if let Some(prev) = &prev_pass {
24602                            let mut round_max_delta = 0f64;
24603                            let mut round_nan = 0usize;
24604                            let mut all_converged = true;
24605                            for i in 0..n {
24606                                if excluded[i] {
24607                                    // Stamped mid-iteration (array result,
24608                                    // §7.9): the value is pinned and cannot
24609                                    // change again — trivially settled.
24610                                    continue;
24611                                }
24612                                let out = crate::engine::convergence::values_converged(
24613                                    &prev[i],
24614                                    &last_value[i],
24615                                    max_change,
24616                                    self.config.date_system,
24617                                );
24618                                if out.nan_converged {
24619                                    round_nan += 1;
24620                                }
24621                                if let Some(d) = out.abs_delta {
24622                                    round_max_delta = round_max_delta.max(d);
24623                                }
24624                                if !out.converged {
24625                                    all_converged = false;
24626                                }
24627                            }
24628                            // Overwrite (not max): telemetry reports the
24629                            // round observed at stop.
24630                            iter_max_delta = round_max_delta;
24631                            iter_nan_converged = round_nan;
24632                            if all_converged {
24633                                converged = true;
24634                                break;
24635                            }
24636                        }
24637
24638                        // ── Pass-counting reconciliation (spec §6/§7.6):
24639                        // `max_iterations` counts TOTAL passes, pass 1
24640                        // included, and pass 1 has already run by the time a
24641                        // live cycle is first witnessed here. The budget is
24642                        // therefore checked BEFORE evaluating anything more:
24643                        // with `max_iterations: 1` we stop right here — each
24644                        // member was evaluated exactly once this recalc (the
24645                        // Excel accumulator contract) and no convergence test
24646                        // ran (`prev_pass` is still `None`). Capping keeps
24647                        // the last committed values and is NOT an error
24648                        // (Excel parity); telemetry records it.
24649                        if passes >= max_iterations as usize {
24650                            capped = true;
24651                            break;
24652                        }
24653
24654                        check_cancel(cancel_flag)?;
24655                        // One more full pass over every evaluable member in
24656                        // member order (Gauss–Seidel: each commit is visible
24657                        // to later members within the pass). Live edges
24658                        // re-record — guards can flip near convergence
24659                        // (§7.3) — so classification repeats next time
24660                        // around, and a cycle that dissolves drops back to
24661                        // the exact acyclic settle below.
24662                        prev_pass = Some(last_value.clone());
24663                        for x in pos.iter_mut() {
24664                            *x = -1;
24665                        }
24666                        changed.fill(false);
24667                        passes += 1;
24668                        let mut p = 0i64;
24669                        for i in 0..n {
24670                            if excluded[i] {
24671                                continue;
24672                            }
24673                            run_member!(i);
24674                            pos[i] = p;
24675                            p += 1;
24676                        }
24677                        continue;
24678                    }
24679                }
24680            }
24681
24682            // Acyclic: find stale readers — members whose live read of `to`
24683            // happened before `to`'s value changed in the pass that just ran.
24684            let mut stale: Vec<usize> = Vec::new();
24685            for i in 0..n {
24686                if excluded[i] {
24687                    continue;
24688                }
24689                let is_stale = out_edges[i].iter().any(|&t| {
24690                    let t = t as usize;
24691                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
24692                });
24693                if is_stale {
24694                    stale.push(i);
24695                }
24696            }
24697            if stale.is_empty() {
24698                break; // values exact — phantom SCC (or dissolved live cycle)
24699            }
24700            if 1 + settle_passes >= cap {
24701                // Defensive only; hitting this is a bug (loud telemetry).
24702                capped = true;
24703                for (i, m) in members.iter().enumerate() {
24704                    if !excluded[i] {
24705                        self.stamp_cycle_error(m.vertex, &circ_error, None);
24706                        excluded[i] = true;
24707                        last_value[i] = circ_error.clone();
24708                        stamped += 1;
24709                    }
24710                }
24711                break;
24712            }
24713
24714            check_cancel(cancel_flag)?;
24715            // Re-evaluate stale readers in live-topo order, recording fresh
24716            // edges (branches may flip on re-eval — spec §7.3 — which is why
24717            // classification repeats).
24718            // A settle pass is a partial sweep: drop the full-pass baseline
24719            // so a live cycle (re)appearing afterwards never compares values
24720            // across mixed pass kinds.
24721            prev_pass = None;
24722            let topo_pos = analysis.topo_positions();
24723            stale.sort_unstable_by_key(|&i| topo_pos[i]);
24724            for x in pos.iter_mut() {
24725                *x = -1;
24726            }
24727            changed.fill(false);
24728            passes += 1;
24729            settle_passes += 1;
24730            for (p, i) in stale.into_iter().enumerate() {
24731                run_member!(i);
24732                pos[i] = p as i64;
24733            }
24734        }
24735
24736        // Iteration that ended because the live cycle dissolved and the
24737        // acyclic settle reached exactness counts as converged (values are
24738        // exact, strictly better than threshold-converged). The defensive
24739        // settle cap (`capped` + stamping) is not.
24740        if iterating && !converged && !capped {
24741            converged = true;
24742        }
24743
24744        // ── 5. End of task: one delta per member whose final value differs
24745        // from the pre-task snapshot (spec §3 side-effect rule, G11).
24746        collector.clear_current();
24747        if let Some(d) = delta
24748            && d.mode != DeltaMode::Off
24749        {
24750            for (i, m) in members.iter().enumerate() {
24751                if let Some(cell) = m.cell
24752                    && last_value[i] != snapshot[i]
24753                {
24754                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24755                }
24756            }
24757        }
24758
24759        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
24760        // Excel's circular cells: register them for the end-of-recalc
24761        // volatile-like redirty (see `pending_iterative_redirty`). Marking
24762        // any one member propagates around the (strongly connected) SCC and
24763        // to downstream dependents, but all members are registered so the
24764        // contract survives partial structural edits between recalcs.
24765        if iterating {
24766            self.pending_iterative_redirty
24767                .extend(members.iter().map(|m| m.vertex));
24768        }
24769
24770        {
24771            let t = &mut self.last_cycle_telemetry;
24772            t.static_sccs += 1;
24773            if witnessed_cycles == 0 && stamped == 0 && !capped {
24774                t.phantom_sccs += 1;
24775            }
24776            t.live_cycles_witnessed += witnessed_cycles;
24777            t.circ_cells_stamped += stamped;
24778            t.settle_passes_total += passes;
24779            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
24780            if iterating {
24781                t.iterated_sccs += 1;
24782                if converged {
24783                    t.converged_sccs += 1;
24784                }
24785                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
24786                t.nan_converged += iter_nan_converged;
24787            }
24788            if capped {
24789                t.capped_sccs += 1;
24790            }
24791            t.elapsed_ms += task_start.elapsed().as_millis();
24792        }
24793
24794        Ok(stamped)
24795    }
24796
24797    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
24798    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
24799    /// reads that actually occur are captured as live edges. Value semantics
24800    /// must match `evaluate_vertex_immutable` exactly (including the missing-
24801    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
24802    /// delegate to it after recording the definition region by hand (those
24803    /// reads bypass the context).
24804    fn evaluate_vertex_recorded(
24805        &self,
24806        vertex_id: VertexId,
24807        ctx: &RecordingContext<'_, R>,
24808        collector: &LiveEdgeCollector,
24809    ) -> Result<LiteralValue, ExcelError> {
24810        if !self.graph.vertex_exists(vertex_id) {
24811            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24812                .with_message(format!("Vertex not found: {vertex_id:?}")));
24813        }
24814
24815        let kind = self.graph.get_vertex_kind(vertex_id);
24816        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24817
24818        match kind {
24819            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24820                let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
24821                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
24822                };
24823                let sheet_name = self.graph.sheet_name(sheet_id);
24824                let cell_ref = self
24825                    .graph
24826                    .get_cell_ref(vertex_id)
24827                    .expect("cell ref for vertex");
24828                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24829                interpreter
24830                    .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24831                    .map(|cv| {
24832                        crate::engine::result_finalization::finalize_formula_result(
24833                            cv.into_literal(),
24834                        )
24835                    })
24836            }
24837            VertexKind::NamedScalar | VertexKind::NamedArray => {
24838                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24839                    ExcelError::new(ExcelErrorKind::Name)
24840                        .with_message("Named range metadata missing".to_string())
24841                })?;
24842
24843                match &named_range.definition {
24844                    NamedDefinition::Formula { ast, .. } => {
24845                        let context_sheet = match named_range.scope {
24846                            NameScope::Sheet(id) => id,
24847                            NameScope::Workbook => sheet_id,
24848                        };
24849                        let sheet_name = self.graph.sheet_name(context_sheet);
24850                        let cell_ref = self
24851                            .graph
24852                            .get_cell_ref(vertex_id)
24853                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24854                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24855                        if kind == VertexKind::NamedScalar {
24856                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24857                        } else {
24858                            match interpreter.evaluate_ast(ast) {
24859                                Ok(cv) => match cv.into_literal() {
24860                                    v @ LiteralValue::Array(_) => Ok(v),
24861                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
24862                                },
24863                                Err(err) => Ok(LiteralValue::Error(err)),
24864                            }
24865                        }
24866                    }
24867                    NamedDefinition::Cell(cell_ref) => {
24868                        // The definition is read via direct grid access in
24869                        // `evaluate_vertex_immutable`; record the live edge
24870                        // by hand before delegating.
24871                        collector.record_scalar(
24872                            cell_ref.sheet_id,
24873                            cell_ref.coord.row(),
24874                            cell_ref.coord.col(),
24875                        );
24876                        self.evaluate_vertex_immutable(vertex_id)
24877                    }
24878                    NamedDefinition::Range(range_ref) => {
24879                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
24880                            collector.record_rect(
24881                                range_ref.start.sheet_id,
24882                                range_ref.start.coord.row(),
24883                                range_ref.start.coord.col(),
24884                                range_ref.end.coord.row(),
24885                                range_ref.end.coord.col(),
24886                            );
24887                        }
24888                        self.evaluate_vertex_immutable(vertex_id)
24889                    }
24890                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
24891                }
24892            }
24893            _ => self.evaluate_vertex_immutable(vertex_id),
24894        }
24895    }
24896
24897    /// Helper: commit spill via shim and mirror resulting cells into Arrow overlay when enabled.
24898    fn commit_spill_and_mirror(
24899        &mut self,
24900        anchor_vertex: VertexId,
24901        targets: &[CellRef],
24902        rows: Vec<Vec<LiteralValue>>,
24903        delta: Option<&mut DeltaCollector>,
24904        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24905    ) -> Result<(), ExcelError> {
24906        let prev_spill_cells = self
24907            .graph
24908            .spill_cells_for_anchor(anchor_vertex)
24909            .map(|cells| cells.to_vec())
24910            .unwrap_or_default();
24911
24912        if let Some(delta) = delta
24913            && delta.mode != DeltaMode::Off
24914        {
24915            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
24916                targets.iter().copied().collect();
24917            let empty = LiteralValue::Empty;
24918
24919            // Clears (prev - targets)
24920            for cell in prev_spill_cells.iter() {
24921                if target_set.contains(cell) {
24922                    continue;
24923                }
24924                let sheet_name = self.graph.sheet_name(cell.sheet_id);
24925                let old = self
24926                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24927                    .unwrap_or(LiteralValue::Empty);
24928                if old != empty {
24929                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24930                }
24931            }
24932
24933            // Writes (targets)
24934            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
24935                let width = rows[0].len();
24936                for (idx, cell) in targets.iter().enumerate() {
24937                    let r_off = idx / width;
24938                    let c_off = idx % width;
24939                    let new = rows
24940                        .get(r_off)
24941                        .and_then(|r| r.get(c_off))
24942                        .cloned()
24943                        .unwrap_or(LiteralValue::Empty);
24944                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24945                    let old = self
24946                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24947                        .unwrap_or(LiteralValue::Empty);
24948                    if old != new {
24949                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24950                    }
24951                }
24952            } else {
24953                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
24954                for cell in targets.iter() {
24955                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24956                    let old = self
24957                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24958                        .unwrap_or(LiteralValue::Empty);
24959                    if !matches!(old, LiteralValue::Empty) {
24960                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24961                    }
24962                }
24963            }
24964        }
24965
24966        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
24967        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
24968        let arrow_sheets = &self.arrow_sheets;
24969        self.spill_mgr.commit_array_with_value_probe(
24970            &mut self.graph,
24971            anchor_vertex,
24972            targets,
24973            rows.clone(),
24974            overwritable_formulas,
24975            |g, cell| {
24976                let sheet_name = g.sheet_name(cell.sheet_id);
24977                let asheet = arrow_sheets.sheet(sheet_name)?;
24978                let r0 = cell.coord.row() as usize;
24979                let c0 = cell.coord.col() as usize;
24980                let v = asheet.get_cell_value(r0, c0);
24981                if matches!(v, LiteralValue::Empty) {
24982                    None
24983                } else {
24984                    Some(v)
24985                }
24986            },
24987        )?;
24988
24989        if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
24990            self.record_formula_plane_structural_change(scope);
24991        }
24992        if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
24993            self.record_formula_plane_structural_change(scope);
24994        }
24995
24996        if self.config.arrow_storage_enabled
24997            && self.config.delta_overlay_enabled
24998            && self.config.write_formula_overlay_enabled
24999        {
25000            if !prev_spill_cells.is_empty() {
25001                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
25002                    targets.iter().copied().collect();
25003                let empty = LiteralValue::Empty;
25004                for cell in prev_spill_cells.iter() {
25005                    if !target_set.contains(cell) {
25006                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25007                        self.mirror_value_to_computed_overlay(
25008                            &sheet_name,
25009                            cell.coord.row() + 1,
25010                            cell.coord.col() + 1,
25011                            &empty,
25012                        );
25013                    }
25014                }
25015            }
25016
25017            for (idx, cell) in targets.iter().enumerate() {
25018                if rows.is_empty() || rows[0].is_empty() {
25019                    break;
25020                }
25021                let width = rows[0].len();
25022                let r_off = idx / width;
25023                let c_off = idx % width;
25024                let v = rows[r_off][c_off].clone();
25025                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25026                self.mirror_value_to_computed_overlay(
25027                    &sheet_name,
25028                    cell.coord.row() + 1,
25029                    cell.coord.col() + 1,
25030                    &v,
25031                );
25032            }
25033        }
25034        Ok(())
25035    }
25036}
25037
25038// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
25039//
25040// Compute → Plan → Apply separation for evaluation side-effects.
25041
25042use crate::engine::effects::Effect;
25043use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
25044
25045impl<R> Engine<R>
25046where
25047    R: EvaluationContext,
25048{
25049    /// Plan effects for a single vertex after its value has been computed.
25050    ///
25051    /// This reads graph state but only performs lightweight mutations
25052    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
25053    /// during the planning phase.  Value-changing mutations are deferred to
25054    /// `apply_effect`.
25055    pub(crate) fn plan_vertex_effects(
25056        &mut self,
25057        vertex_id: VertexId,
25058        computed_value: LiteralValue,
25059        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25060    ) -> Result<Vec<Effect>, ExcelError> {
25061        let kind = self.graph.get_vertex_kind(vertex_id);
25062        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
25063
25064        // If this vertex's cell is currently covered by a spill from a different
25065        // anchor, ignore the computed result.  Formula vertices are exempt:
25066        // they must still evaluate so that overlapping spills produce #SPILL!.
25067        if !is_formula {
25068            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
25069                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
25070                && owner != vertex_id
25071            {
25072                return Ok(Vec::new());
25073            }
25074            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
25075            return Ok(vec![Effect::WriteCell {
25076                vertex_id,
25077                value: computed_value,
25078            }]);
25079        }
25080
25081        match computed_value {
25082            LiteralValue::Array(rows) => {
25083                self.plan_array_effects(vertex_id, rows, overwritable_formulas)
25084            }
25085            other => self.plan_scalar_effects(vertex_id, other),
25086        }
25087    }
25088
25089    /// Plan effects for a formula vertex that produced a scalar/error result.
25090    fn plan_scalar_effects(
25091        &self,
25092        vertex_id: VertexId,
25093        value: LiteralValue,
25094    ) -> Result<Vec<Effect>, ExcelError> {
25095        let has_spill = self
25096            .graph
25097            .spill_cells_for_anchor(vertex_id)
25098            .is_some_and(|c| !c.is_empty());
25099
25100        let mut effects = Vec::new();
25101        if has_spill {
25102            effects.push(Effect::SpillClear {
25103                anchor_vertex: vertex_id,
25104            });
25105        }
25106        effects.push(Effect::WriteCell { vertex_id, value });
25107        Ok(effects)
25108    }
25109
25110    /// Plan effects for a formula vertex that produced an array result.
25111    fn plan_array_effects(
25112        &mut self,
25113        vertex_id: VertexId,
25114        rows: Vec<Vec<LiteralValue>>,
25115        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25116    ) -> Result<Vec<Effect>, ExcelError> {
25117        // Lightweight mutation needed for correct spill-blocking checks.
25118        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
25119
25120        let anchor = self
25121            .graph
25122            .get_cell_ref(vertex_id)
25123            .expect("cell ref for vertex");
25124        let sheet_id = anchor.sheet_id;
25125        let h = rows.len() as u32;
25126        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
25127
25128        // Hard cap to avoid vertex explosion from huge dynamic arrays.
25129        let spill_cells = (h as u64).saturating_mul(w as u64);
25130        if spill_cells > self.config.spill.max_spill_cells as u64 {
25131            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
25132        }
25133
25134        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
25135        const PACKED_MAX_ROW: u32 = 1_048_575;
25136        const PACKED_MAX_COL: u32 = 16_383;
25137        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
25138        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
25139        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
25140            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
25141        }
25142
25143        let mut targets = Vec::new();
25144        for r in 0..h {
25145            for c in 0..w {
25146                targets.push(self.graph.make_cell_ref_internal(
25147                    sheet_id,
25148                    anchor.coord.row() + r,
25149                    anchor.coord.col() + c,
25150                ));
25151            }
25152        }
25153
25154        // Region lock via spill manager.
25155        match self.spill_mgr.reserve(
25156            vertex_id,
25157            anchor,
25158            SpillShape { rows: h, cols: w },
25159            SpillMeta {
25160                epoch: self.recalc_epoch,
25161                config: self.config.spill,
25162            },
25163        ) {
25164            Ok(()) => {
25165                // Validate spill region is available.
25166                if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
25167                    vertex_id,
25168                    &targets,
25169                    overwritable_formulas,
25170                ) {
25171                    return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
25172                }
25173
25174                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
25175                // cell values are not cached in the dependency graph. Consult Arrow storage to
25176                // detect occupied cells in the target region.
25177                if !self.graph.value_cache_enabled() {
25178                    let sheet_name = self.graph.sheet_name(sheet_id);
25179                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25180                        for cell in targets.iter() {
25181                            // Allow overwriting the anchor itself.
25182                            if *cell == anchor {
25183                                continue;
25184                            }
25185                            // Allow cells already owned by a spill (plan() validated spill ownership).
25186                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
25187                                continue;
25188                            }
25189                            // Skip formula blockers; plan() handled them (or allowed).
25190                            if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
25191                                && vid != vertex_id
25192                            {
25193                                match self.graph.get_vertex_kind(vid) {
25194                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
25195                                        continue;
25196                                    }
25197                                    _ => {}
25198                                }
25199                            }
25200
25201                            let v = asheet.get_cell_value(
25202                                cell.coord.row() as usize,
25203                                cell.coord.col() as usize,
25204                            );
25205                            if !matches!(v, LiteralValue::Empty) {
25206                                return self.plan_spill_error_effects(
25207                                    vertex_id,
25208                                    "BlockedByValue",
25209                                    h,
25210                                    w,
25211                                );
25212                            }
25213                        }
25214                    }
25215                }
25216
25217                let top_left = rows
25218                    .first()
25219                    .and_then(|r| r.first())
25220                    .cloned()
25221                    .unwrap_or(LiteralValue::Empty);
25222
25223                let mut effects = Vec::new();
25224                // Clear previous spill if any.
25225                let has_prev = self
25226                    .graph
25227                    .spill_cells_for_anchor(vertex_id)
25228                    .is_some_and(|c| !c.is_empty());
25229                if has_prev {
25230                    effects.push(Effect::SpillClear {
25231                        anchor_vertex: vertex_id,
25232                    });
25233                }
25234                effects.push(Effect::SpillCommit {
25235                    anchor_vertex: vertex_id,
25236                    anchor_cell: anchor,
25237                    target_cells: targets,
25238                    values: rows,
25239                });
25240                effects.push(Effect::WriteCell {
25241                    vertex_id,
25242                    value: top_left,
25243                });
25244                Ok(effects)
25245            }
25246            Err(e) => {
25247                let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
25248                self.plan_spill_error_effects(vertex_id, &msg, h, w)
25249            }
25250        }
25251    }
25252
25253    /// Build the effect list for a spill that failed validation.
25254    fn plan_spill_error_effects(
25255        &self,
25256        vertex_id: VertexId,
25257        message: &str,
25258        expected_rows: u32,
25259        expected_cols: u32,
25260    ) -> Result<Vec<Effect>, ExcelError> {
25261        let spill_err = ExcelError::new(ExcelErrorKind::Spill)
25262            .with_message(message)
25263            .with_extra(formualizer_common::ExcelErrorExtra::Spill {
25264                expected_rows,
25265                expected_cols,
25266            });
25267        let spill_val = LiteralValue::Error(spill_err);
25268
25269        let effects = vec![
25270            Effect::SpillClear {
25271                anchor_vertex: vertex_id,
25272            },
25273            Effect::WriteCell {
25274                vertex_id,
25275                value: spill_val,
25276            },
25277        ];
25278        Ok(effects)
25279    }
25280
25281    /// Apply a single effect, performing the actual graph mutations.
25282    pub(crate) fn apply_effect(
25283        &mut self,
25284        effect: &Effect,
25285        delta: Option<&mut DeltaCollector>,
25286        log: Option<&mut ChangeLog>,
25287    ) -> Result<(), ExcelError> {
25288        self.apply_effect_with_computed_writes(effect, delta, log, None)
25289    }
25290
25291    fn apply_effect_with_computed_writes(
25292        &mut self,
25293        effect: &Effect,
25294        delta: Option<&mut DeltaCollector>,
25295        log: Option<&mut ChangeLog>,
25296        computed_writes: Option<&mut ComputedWriteBuffer>,
25297    ) -> Result<(), ExcelError> {
25298        match effect {
25299            Effect::WriteCell { vertex_id, value } => {
25300                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
25301            }
25302            Effect::SpillClear { anchor_vertex } => {
25303                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
25304            }
25305            Effect::SpillCommit {
25306                anchor_vertex,
25307                anchor_cell: _,
25308                target_cells,
25309                values,
25310            } => {
25311                self.apply_spill_commit(
25312                    *anchor_vertex,
25313                    target_cells,
25314                    values.clone(),
25315                    delta,
25316                    log,
25317                    computed_writes,
25318                )?;
25319            }
25320        }
25321        Ok(())
25322    }
25323
25324    /// Apply a WriteCell effect.
25325    fn apply_write_cell(
25326        &mut self,
25327        vertex_id: VertexId,
25328        value: &LiteralValue,
25329        delta: Option<&mut DeltaCollector>,
25330        mut computed_writes: Option<&mut ComputedWriteBuffer>,
25331    ) -> Result<(), ExcelError> {
25332        if let Some(d) = delta
25333            && d.mode != DeltaMode::Off
25334        {
25335            if let Some(buffer) = computed_writes.as_deref_mut() {
25336                self.flush_computed_write_buffer(buffer)?;
25337            }
25338            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
25339                let sheet_name = self.graph.sheet_name(cell.sheet_id);
25340                let old = self
25341                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25342                    .unwrap_or(LiteralValue::Empty);
25343                if old != *value {
25344                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25345                }
25346            }
25347        }
25348        self.graph.update_vertex_value(vertex_id, value.clone());
25349        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
25350        Ok(())
25351    }
25352
25353    /// Apply a SpillClear effect.
25354    fn apply_spill_clear(
25355        &mut self,
25356        anchor_vertex: VertexId,
25357        delta: Option<&mut DeltaCollector>,
25358        log: Option<&mut ChangeLog>,
25359        computed_writes: Option<&mut ComputedWriteBuffer>,
25360    ) -> Result<(), ExcelError> {
25361        if let Some(buffer) = computed_writes {
25362            self.flush_computed_write_buffer(buffer)?;
25363        }
25364
25365        let spill_cells = self
25366            .graph
25367            .spill_cells_for_anchor(anchor_vertex)
25368            .map(|cells| cells.to_vec())
25369            .unwrap_or_default();
25370        if spill_cells.is_empty() {
25371            return Ok(());
25372        }
25373
25374        // Snapshot for ChangeLog before clearing.
25375        let snapshot = if log.is_some() {
25376            self.snapshot_spill_for_anchor(anchor_vertex)
25377        } else {
25378            None
25379        };
25380
25381        // Record delta for cleared cells.
25382        if let Some(d) = delta
25383            && d.mode != DeltaMode::Off
25384        {
25385            let empty = LiteralValue::Empty;
25386            for cell in spill_cells.iter() {
25387                let sheet_name = self.graph.sheet_name(cell.sheet_id);
25388                let old = self
25389                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25390                    .unwrap_or(LiteralValue::Empty);
25391                if old != empty {
25392                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25393                }
25394            }
25395        }
25396
25397        self.graph.clear_spill_region(anchor_vertex);
25398        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
25399            self.record_formula_plane_structural_change(scope);
25400        }
25401
25402        // Mirror Empty to Arrow overlay for cleared cells.
25403        if self.config.arrow_storage_enabled
25404            && self.config.delta_overlay_enabled
25405            && self.config.write_formula_overlay_enabled
25406        {
25407            let empty = LiteralValue::Empty;
25408            for cell in spill_cells.iter() {
25409                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25410                self.mirror_value_to_computed_overlay(
25411                    &sheet_name,
25412                    cell.coord.row() + 1,
25413                    cell.coord.col() + 1,
25414                    &empty,
25415                );
25416            }
25417        }
25418
25419        // ChangeLog.
25420        if let Some(log) = log
25421            && let Some(old) = snapshot
25422        {
25423            log.record(ChangeEvent::SpillCleared {
25424                anchor: anchor_vertex,
25425                old,
25426            });
25427        }
25428        Ok(())
25429    }
25430
25431    /// Apply a SpillCommit effect.
25432    fn apply_spill_commit(
25433        &mut self,
25434        anchor_vertex: VertexId,
25435        target_cells: &[CellRef],
25436        values: Vec<Vec<LiteralValue>>,
25437        delta: Option<&mut DeltaCollector>,
25438        log: Option<&mut ChangeLog>,
25439        computed_writes: Option<&mut ComputedWriteBuffer>,
25440    ) -> Result<(), ExcelError> {
25441        if let Some(buffer) = computed_writes {
25442            self.flush_computed_write_buffer(buffer)?;
25443        }
25444
25445        // Snapshot for ChangeLog before commit.
25446        let old_snapshot = if log.is_some() {
25447            self.snapshot_spill_for_anchor(anchor_vertex)
25448        } else {
25449            None
25450        };
25451
25452        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
25453        self.commit_spill_and_mirror(
25454            anchor_vertex,
25455            target_cells,
25456            values.clone(),
25457            delta,
25458            None, // overwritable_formulas already validated in plan phase
25459        )?;
25460
25461        // ChangeLog.
25462        if let Some(log) = log {
25463            log.record(ChangeEvent::SpillCommitted {
25464                anchor: anchor_vertex,
25465                old: old_snapshot,
25466                new: SpillSnapshot {
25467                    target_cells: target_cells.to_vec(),
25468                    values,
25469                },
25470            });
25471        }
25472        Ok(())
25473    }
25474
25475    /// Snapshot a spill region for ChangeLog recording.
25476    ///
25477    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
25478    /// without creating a `VertexEditor`.
25479    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
25480        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
25481        if cells.is_empty() {
25482            return None;
25483        }
25484
25485        let max = self.config.spill.max_spill_cells as usize;
25486        let mut cells = cells;
25487        if cells.len() > max {
25488            cells.truncate(max);
25489        }
25490
25491        let first = *cells.first().expect("non-empty spill cells");
25492        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
25493        let row0 = first.coord.row();
25494        let col0 = first.coord.col();
25495
25496        let mut max_row = row0;
25497        let mut max_col = col0;
25498        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
25499        for cell in &cells {
25500            max_row = max_row.max(cell.coord.row());
25501            max_col = max_col.max(cell.coord.col());
25502            let v = self
25503                .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25504                .unwrap_or(LiteralValue::Empty);
25505            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
25506        }
25507
25508        let rows = (max_row - row0 + 1) as usize;
25509        let cols = (max_col - col0 + 1) as usize;
25510        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
25511        for r in 0..rows {
25512            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
25513            for c in 0..cols {
25514                row.push(
25515                    by_coord
25516                        .get(&(row0 + r as u32, col0 + c as u32))
25517                        .cloned()
25518                        .unwrap_or(LiteralValue::Empty),
25519                );
25520            }
25521            values.push(row);
25522        }
25523
25524        Some(SpillSnapshot {
25525            target_cells: cells,
25526            values,
25527        })
25528    }
25529
25530    fn flush_before_range_dependent_vertex(
25531        &mut self,
25532        vertex_id: VertexId,
25533        computed_writes: &mut ComputedWriteBuffer,
25534    ) -> Result<(), ExcelError> {
25535        if self.graph.get_range_dependencies(vertex_id).is_some() {
25536            self.flush_computed_write_buffer(computed_writes)?;
25537        }
25538        Ok(())
25539    }
25540
25541    fn plan_vertex_effects_with_computed_flush(
25542        &mut self,
25543        vertex_id: VertexId,
25544        computed_value: LiteralValue,
25545        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25546        computed_writes: &mut ComputedWriteBuffer,
25547    ) -> Result<Vec<Effect>, ExcelError> {
25548        if matches!(&computed_value, LiteralValue::Array(_)) {
25549            self.flush_computed_write_buffer(computed_writes)?;
25550        }
25551        self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
25552    }
25553
25554    // ── Layer evaluation via effects pipeline ──────────────────────────────
25555
25556    fn evaluate_small_layer_direct_effects(
25557        &mut self,
25558        layer: &super::scheduler::Layer,
25559        mut delta: Option<&mut DeltaCollector>,
25560        mut log: Option<&mut ChangeLog>,
25561        cancel_flag: Option<&AtomicBool>,
25562        cancel_check_every: usize,
25563        cancel_message: &'static str,
25564    ) -> Result<usize, ExcelError> {
25565        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25566            if cancel_check_every > 0
25567                && i % cancel_check_every == 0
25568                && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
25569            {
25570                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25571                    .with_message(cancel_message.to_string()));
25572            }
25573            let value = match self.evaluate_vertex_immutable(vertex_id) {
25574                Ok(v) => v,
25575                Err(e) => LiteralValue::Error(e),
25576            };
25577            let effects = self.plan_vertex_effects(vertex_id, value, None)?;
25578            for effect in &effects {
25579                self.apply_effect_with_computed_writes(
25580                    effect,
25581                    delta.as_deref_mut(),
25582                    log.as_deref_mut(),
25583                    None,
25584                )?;
25585            }
25586        }
25587        Ok(layer.vertices.len())
25588    }
25589
25590    /// Evaluate a layer sequentially using the effects pipeline.
25591    fn evaluate_layer_sequential_effects(
25592        &mut self,
25593        layer: &super::scheduler::Layer,
25594    ) -> Result<usize, ExcelError> {
25595        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25596            return self.evaluate_small_layer_direct_effects(
25597                layer,
25598                None,
25599                None,
25600                None,
25601                0,
25602                "Evaluation cancelled within layer",
25603            );
25604        }
25605
25606        let mut computed_writes = ComputedWriteBuffer::default();
25607        for &vertex_id in &layer.vertices {
25608            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25609            let value = match self.evaluate_vertex_immutable(vertex_id) {
25610                Ok(v) => v,
25611                Err(e) => LiteralValue::Error(e),
25612            };
25613            let effects = match self.plan_vertex_effects_with_computed_flush(
25614                vertex_id,
25615                value,
25616                None,
25617                &mut computed_writes,
25618            ) {
25619                Ok(effects) => effects,
25620                Err(e) => {
25621                    self.flush_computed_write_buffer(&mut computed_writes)?;
25622                    return Err(e);
25623                }
25624            };
25625            for effect in &effects {
25626                if let Err(e) = self.apply_effect_with_computed_writes(
25627                    effect,
25628                    None,
25629                    None,
25630                    Some(&mut computed_writes),
25631                ) {
25632                    self.flush_computed_write_buffer(&mut computed_writes)?;
25633                    return Err(e);
25634                }
25635            }
25636        }
25637        self.flush_computed_write_buffer(&mut computed_writes)?;
25638        Ok(layer.vertices.len())
25639    }
25640
25641    /// Evaluate a layer sequentially with delta collection via effects pipeline.
25642    fn evaluate_layer_sequential_with_delta_effects(
25643        &mut self,
25644        layer: &super::scheduler::Layer,
25645        delta: &mut DeltaCollector,
25646    ) -> Result<usize, ExcelError> {
25647        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25648            return self.evaluate_small_layer_direct_effects(
25649                layer,
25650                Some(delta),
25651                None,
25652                None,
25653                0,
25654                "Evaluation cancelled within layer",
25655            );
25656        }
25657
25658        let mut computed_writes = ComputedWriteBuffer::default();
25659        for &vertex_id in &layer.vertices {
25660            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25661            let value = match self.evaluate_vertex_immutable(vertex_id) {
25662                Ok(v) => v,
25663                Err(e) => LiteralValue::Error(e),
25664            };
25665            let effects = match self.plan_vertex_effects_with_computed_flush(
25666                vertex_id,
25667                value,
25668                None,
25669                &mut computed_writes,
25670            ) {
25671                Ok(effects) => effects,
25672                Err(e) => {
25673                    self.flush_computed_write_buffer(&mut computed_writes)?;
25674                    return Err(e);
25675                }
25676            };
25677            for effect in &effects {
25678                if let Err(e) = self.apply_effect_with_computed_writes(
25679                    effect,
25680                    Some(delta),
25681                    None,
25682                    Some(&mut computed_writes),
25683                ) {
25684                    self.flush_computed_write_buffer(&mut computed_writes)?;
25685                    return Err(e);
25686                }
25687            }
25688        }
25689        self.flush_computed_write_buffer(&mut computed_writes)?;
25690        Ok(layer.vertices.len())
25691    }
25692
25693    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
25694    fn evaluate_layer_sequential_cancellable_effects(
25695        &mut self,
25696        layer: &super::scheduler::Layer,
25697        cancel_flag: &AtomicBool,
25698    ) -> Result<usize, ExcelError> {
25699        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25700            return self.evaluate_small_layer_direct_effects(
25701                layer,
25702                None,
25703                None,
25704                Some(cancel_flag),
25705                256,
25706                "Evaluation cancelled within layer",
25707            );
25708        }
25709
25710        let mut computed_writes = ComputedWriteBuffer::default();
25711        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25712            if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
25713                self.flush_computed_write_buffer(&mut computed_writes)?;
25714                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25715                    .with_message("Evaluation cancelled within layer".to_string()));
25716            }
25717            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25718            let value = match self.evaluate_vertex_immutable(vertex_id) {
25719                Ok(v) => v,
25720                Err(e) => LiteralValue::Error(e),
25721            };
25722            let effects = match self.plan_vertex_effects_with_computed_flush(
25723                vertex_id,
25724                value,
25725                None,
25726                &mut computed_writes,
25727            ) {
25728                Ok(effects) => effects,
25729                Err(e) => {
25730                    self.flush_computed_write_buffer(&mut computed_writes)?;
25731                    return Err(e);
25732                }
25733            };
25734            for effect in &effects {
25735                if let Err(e) = self.apply_effect_with_computed_writes(
25736                    effect,
25737                    None,
25738                    None,
25739                    Some(&mut computed_writes),
25740                ) {
25741                    self.flush_computed_write_buffer(&mut computed_writes)?;
25742                    return Err(e);
25743                }
25744            }
25745        }
25746        self.flush_computed_write_buffer(&mut computed_writes)?;
25747        Ok(layer.vertices.len())
25748    }
25749
25750    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
25751    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
25752        &mut self,
25753        layer: &super::scheduler::Layer,
25754        cancel_flag: &AtomicBool,
25755    ) -> Result<usize, ExcelError> {
25756        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25757            return self.evaluate_small_layer_direct_effects(
25758                layer,
25759                None,
25760                None,
25761                Some(cancel_flag),
25762                128,
25763                "Demand-driven evaluation cancelled within layer",
25764            );
25765        }
25766
25767        let mut computed_writes = ComputedWriteBuffer::default();
25768        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25769            if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
25770                self.flush_computed_write_buffer(&mut computed_writes)?;
25771                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25772                    .with_message("Demand-driven evaluation cancelled within layer".to_string()));
25773            }
25774            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25775            let value = match self.evaluate_vertex_immutable(vertex_id) {
25776                Ok(v) => v,
25777                Err(e) => LiteralValue::Error(e),
25778            };
25779            let effects = match self.plan_vertex_effects_with_computed_flush(
25780                vertex_id,
25781                value,
25782                None,
25783                &mut computed_writes,
25784            ) {
25785                Ok(effects) => effects,
25786                Err(e) => {
25787                    self.flush_computed_write_buffer(&mut computed_writes)?;
25788                    return Err(e);
25789                }
25790            };
25791            for effect in &effects {
25792                if let Err(e) = self.apply_effect_with_computed_writes(
25793                    effect,
25794                    None,
25795                    None,
25796                    Some(&mut computed_writes),
25797                ) {
25798                    self.flush_computed_write_buffer(&mut computed_writes)?;
25799                    return Err(e);
25800                }
25801            }
25802        }
25803        self.flush_computed_write_buffer(&mut computed_writes)?;
25804        Ok(layer.vertices.len())
25805    }
25806
25807    /// Evaluate a layer in parallel, applying via effects pipeline.
25808    fn evaluate_layer_parallel_effects(
25809        &mut self,
25810        layer: &super::scheduler::Layer,
25811    ) -> Result<usize, ExcelError> {
25812        use rayon::prelude::*;
25813
25814        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25815
25816        let mut phase1: Vec<VertexId> = Vec::new();
25817        let mut phase2: Vec<VertexId> = Vec::new();
25818        for &vid in &layer.vertices {
25819            if self.graph.get_range_dependencies(vid).is_some() {
25820                phase2.push(vid);
25821            } else {
25822                phase1.push(vid);
25823            }
25824        }
25825
25826        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25827        let mut applied = 0usize;
25828
25829        for group in [&phase1[..], &phase2[..]] {
25830            if group.is_empty() {
25831                continue;
25832            }
25833            let mut computed_writes = ComputedWriteBuffer::default();
25834
25835            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25836                thread_pool.install(|| {
25837                    group
25838                        .par_iter()
25839                        .map(
25840                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25841                                Ok(v) => Ok((vertex_id, v)),
25842                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25843                            },
25844                        )
25845                        .collect()
25846                });
25847
25848            match results {
25849                Ok(vertex_results) => {
25850                    // Arrays first, then scalars — establishes spill regions before
25851                    // scalar results that might land inside a spilled region.
25852                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25853                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25854                    for (vertex_id, result) in vertex_results {
25855                        if matches!(result, LiteralValue::Array(_)) {
25856                            arrays.push((vertex_id, result));
25857                        } else {
25858                            others.push((vertex_id, result));
25859                        }
25860                    }
25861                    for (vertex_id, result) in arrays {
25862                        let effects = match self.plan_vertex_effects_with_computed_flush(
25863                            vertex_id,
25864                            result,
25865                            Some(&inflight),
25866                            &mut computed_writes,
25867                        ) {
25868                            Ok(effects) => effects,
25869                            Err(e) => {
25870                                self.flush_computed_write_buffer(&mut computed_writes)?;
25871                                return Err(e);
25872                            }
25873                        };
25874                        for effect in &effects {
25875                            if let Err(e) = self.apply_effect_with_computed_writes(
25876                                effect,
25877                                None,
25878                                None,
25879                                Some(&mut computed_writes),
25880                            ) {
25881                                self.flush_computed_write_buffer(&mut computed_writes)?;
25882                                return Err(e);
25883                            }
25884                        }
25885                        applied = applied.saturating_add(1);
25886                    }
25887                    // Make all array spill/top-left writes visible before scalar effects in this group.
25888                    self.flush_computed_write_buffer(&mut computed_writes)?;
25889                    for (vertex_id, result) in others {
25890                        let effects = match self.plan_vertex_effects_with_computed_flush(
25891                            vertex_id,
25892                            result,
25893                            Some(&inflight),
25894                            &mut computed_writes,
25895                        ) {
25896                            Ok(effects) => effects,
25897                            Err(e) => {
25898                                self.flush_computed_write_buffer(&mut computed_writes)?;
25899                                return Err(e);
25900                            }
25901                        };
25902                        for effect in &effects {
25903                            if let Err(e) = self.apply_effect_with_computed_writes(
25904                                effect,
25905                                None,
25906                                None,
25907                                Some(&mut computed_writes),
25908                            ) {
25909                                self.flush_computed_write_buffer(&mut computed_writes)?;
25910                                return Err(e);
25911                            }
25912                        }
25913                        applied = applied.saturating_add(1);
25914                    }
25915                    // Flush at the group boundary; phase1 must be visible before phase2.
25916                    self.flush_computed_write_buffer(&mut computed_writes)?;
25917                }
25918                Err(e) => {
25919                    self.flush_computed_write_buffer(&mut computed_writes)?;
25920                    return Err(e);
25921                }
25922            }
25923        }
25924
25925        Ok(applied)
25926    }
25927
25928    /// Evaluate a layer in parallel with delta collection via effects pipeline.
25929    fn evaluate_layer_parallel_with_delta_effects(
25930        &mut self,
25931        layer: &super::scheduler::Layer,
25932        delta: &mut DeltaCollector,
25933    ) -> Result<usize, ExcelError> {
25934        use rayon::prelude::*;
25935
25936        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25937
25938        let mut phase1: Vec<VertexId> = Vec::new();
25939        let mut phase2: Vec<VertexId> = Vec::new();
25940        for &vid in &layer.vertices {
25941            if self.graph.get_range_dependencies(vid).is_some() {
25942                phase2.push(vid);
25943            } else {
25944                phase1.push(vid);
25945            }
25946        }
25947
25948        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25949        let mut applied = 0usize;
25950
25951        for group in [&phase1[..], &phase2[..]] {
25952            if group.is_empty() {
25953                continue;
25954            }
25955            let mut computed_writes = ComputedWriteBuffer::default();
25956            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25957                thread_pool.install(|| {
25958                    group
25959                        .par_iter()
25960                        .map(
25961                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25962                                Ok(v) => Ok((vertex_id, v)),
25963                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25964                            },
25965                        )
25966                        .collect()
25967                });
25968
25969            match results {
25970                Ok(vertex_results) => {
25971                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25972                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25973                    for (vertex_id, result) in vertex_results {
25974                        if matches!(result, LiteralValue::Array(_)) {
25975                            arrays.push((vertex_id, result));
25976                        } else {
25977                            others.push((vertex_id, result));
25978                        }
25979                    }
25980                    for (vertex_id, result) in arrays {
25981                        let effects = match self.plan_vertex_effects_with_computed_flush(
25982                            vertex_id,
25983                            result,
25984                            Some(&inflight),
25985                            &mut computed_writes,
25986                        ) {
25987                            Ok(effects) => effects,
25988                            Err(e) => {
25989                                self.flush_computed_write_buffer(&mut computed_writes)?;
25990                                return Err(e);
25991                            }
25992                        };
25993                        for effect in &effects {
25994                            if let Err(e) = self.apply_effect_with_computed_writes(
25995                                effect,
25996                                Some(delta),
25997                                None,
25998                                Some(&mut computed_writes),
25999                            ) {
26000                                self.flush_computed_write_buffer(&mut computed_writes)?;
26001                                return Err(e);
26002                            }
26003                        }
26004                        applied = applied.saturating_add(1);
26005                    }
26006                    self.flush_computed_write_buffer(&mut computed_writes)?;
26007                    for (vertex_id, result) in others {
26008                        let effects = match self.plan_vertex_effects_with_computed_flush(
26009                            vertex_id,
26010                            result,
26011                            Some(&inflight),
26012                            &mut computed_writes,
26013                        ) {
26014                            Ok(effects) => effects,
26015                            Err(e) => {
26016                                self.flush_computed_write_buffer(&mut computed_writes)?;
26017                                return Err(e);
26018                            }
26019                        };
26020                        for effect in &effects {
26021                            if let Err(e) = self.apply_effect_with_computed_writes(
26022                                effect,
26023                                Some(delta),
26024                                None,
26025                                Some(&mut computed_writes),
26026                            ) {
26027                                self.flush_computed_write_buffer(&mut computed_writes)?;
26028                                return Err(e);
26029                            }
26030                        }
26031                        applied = applied.saturating_add(1);
26032                    }
26033                    self.flush_computed_write_buffer(&mut computed_writes)?;
26034                }
26035                Err(e) => {
26036                    self.flush_computed_write_buffer(&mut computed_writes)?;
26037                    return Err(e);
26038                }
26039            }
26040        }
26041
26042        Ok(applied)
26043    }
26044
26045    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
26046    fn evaluate_layer_parallel_cancellable_effects(
26047        &mut self,
26048        layer: &super::scheduler::Layer,
26049        cancel_flag: &AtomicBool,
26050    ) -> Result<usize, ExcelError> {
26051        use rayon::prelude::*;
26052
26053        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26054
26055        if cancel_flag.load(Ordering::Relaxed) {
26056            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26057                .with_message("Parallel evaluation cancelled before starting".to_string()));
26058        }
26059
26060        let mut phase1: Vec<VertexId> = Vec::new();
26061        let mut phase2: Vec<VertexId> = Vec::new();
26062        for &vid in &layer.vertices {
26063            if self.graph.get_range_dependencies(vid).is_some() {
26064                phase2.push(vid);
26065            } else {
26066                phase1.push(vid);
26067            }
26068        }
26069
26070        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26071        let mut applied = 0usize;
26072
26073        for group in [&phase1[..], &phase2[..]] {
26074            if group.is_empty() {
26075                continue;
26076            }
26077            let mut computed_writes = ComputedWriteBuffer::default();
26078
26079            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26080                thread_pool.install(|| {
26081                    group
26082                        .par_iter()
26083                        .map(|&vertex_id| {
26084                            if cancel_flag.load(Ordering::Relaxed) {
26085                                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26086                                    .with_message(
26087                                        "Parallel evaluation cancelled during execution"
26088                                            .to_string(),
26089                                    ));
26090                            }
26091                            match self.evaluate_vertex_immutable(vertex_id) {
26092                                Ok(v) => Ok((vertex_id, v)),
26093                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26094                            }
26095                        })
26096                        .collect()
26097                });
26098
26099            match results {
26100                Ok(vertex_results) => {
26101                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26102                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26103                    for (vertex_id, result) in vertex_results {
26104                        if matches!(result, LiteralValue::Array(_)) {
26105                            arrays.push((vertex_id, result));
26106                        } else {
26107                            others.push((vertex_id, result));
26108                        }
26109                    }
26110                    for (vertex_id, result) in arrays {
26111                        let effects = match self.plan_vertex_effects_with_computed_flush(
26112                            vertex_id,
26113                            result,
26114                            Some(&inflight),
26115                            &mut computed_writes,
26116                        ) {
26117                            Ok(effects) => effects,
26118                            Err(e) => {
26119                                self.flush_computed_write_buffer(&mut computed_writes)?;
26120                                return Err(e);
26121                            }
26122                        };
26123                        for effect in &effects {
26124                            if let Err(e) = self.apply_effect_with_computed_writes(
26125                                effect,
26126                                None,
26127                                None,
26128                                Some(&mut computed_writes),
26129                            ) {
26130                                self.flush_computed_write_buffer(&mut computed_writes)?;
26131                                return Err(e);
26132                            }
26133                        }
26134                        applied = applied.saturating_add(1);
26135                    }
26136                    self.flush_computed_write_buffer(&mut computed_writes)?;
26137                    for (vertex_id, result) in others {
26138                        let effects = match self.plan_vertex_effects_with_computed_flush(
26139                            vertex_id,
26140                            result,
26141                            Some(&inflight),
26142                            &mut computed_writes,
26143                        ) {
26144                            Ok(effects) => effects,
26145                            Err(e) => {
26146                                self.flush_computed_write_buffer(&mut computed_writes)?;
26147                                return Err(e);
26148                            }
26149                        };
26150                        for effect in &effects {
26151                            if let Err(e) = self.apply_effect_with_computed_writes(
26152                                effect,
26153                                None,
26154                                None,
26155                                Some(&mut computed_writes),
26156                            ) {
26157                                self.flush_computed_write_buffer(&mut computed_writes)?;
26158                                return Err(e);
26159                            }
26160                        }
26161                        applied = applied.saturating_add(1);
26162                    }
26163                    self.flush_computed_write_buffer(&mut computed_writes)?;
26164                }
26165                Err(e) => {
26166                    self.flush_computed_write_buffer(&mut computed_writes)?;
26167                    return Err(e);
26168                }
26169            }
26170        }
26171
26172        Ok(applied)
26173    }
26174
26175    // ── Top-level evaluate_all_logged ───────────────────────────────────────
26176
26177    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
26178    ///
26179    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
26180    /// every effect application so that spill commits/clears are captured.
26181    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
26182        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
26183            engine.evaluate_all_logged_unobserved(log)
26184        })
26185    }
26186
26187    fn evaluate_all_logged_unobserved(
26188        &mut self,
26189        log: &mut ChangeLog,
26190    ) -> Result<EvalResult, ExcelError> {
26191        self.observe_function_semantic_epoch()?;
26192        self.begin_evaluation_request();
26193        let _source_cache = self.source_cache_session();
26194        self.validate_deterministic_mode()?;
26195        if self.config.defer_graph_building {
26196            self.build_graph_all()?;
26197        }
26198        if self.graph.formula_authority().active_span_count() > 0 {
26199            return self.evaluate_authoritative_formula_plane_all();
26200        }
26201        self.reset_virtual_dep_telemetry_if_disabled();
26202        let start = crate::instant::FzInstant::now();
26203        let mut computed_vertices = 0;
26204        let mut cycle_errors = 0;
26205
26206        let mut replan_iterations = 0;
26207        const MAX_REPLAN: usize = 5;
26208        let mut telemetry = self
26209            .config
26210            .enable_virtual_dep_telemetry
26211            .then(|| self.start_virtual_dep_telemetry());
26212
26213        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
26214
26215        let result = (|| -> Result<EvalResult, ExcelError> {
26216            loop {
26217                let to_evaluate = self.graph.get_evaluation_vertices();
26218                if to_evaluate.is_empty() {
26219                    if let Some(t) = telemetry.as_mut()
26220                        && t.bailout_reason.is_none()
26221                    {
26222                        t.bailout_reason = Some("no_work");
26223                    }
26224                    break;
26225                }
26226
26227                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
26228                if let Some(t) = telemetry.as_mut() {
26229                    Self::accumulate_schedule_meta(t, &meta);
26230                }
26231
26232                // Walk units in condensation order: stamp cycles at their
26233                // position, evaluate layers with ChangeLog recording.
26234                for &unit in &schedule.units {
26235                    match unit {
26236                        ScheduleUnit::Cycle(i) => {
26237                            // Journal integration (design doc §4 last row): the
26238                            // ChangeLog in this path only records SpillClear /
26239                            // SpillCommit events; WriteCell effects are never
26240                            // logged (see `apply_write_cell`). Runtime SCC tasks
26241                            // write values directly and never spill (§7.9 stamps
26242                            // would-be anchors), and their spill *teardown* is the
26243                            // same unlogged `stamp_cycle_error` the Static path
26244                            // already uses here — so direct commits coexist with
26245                            // the journal cleanly, with identical semantics to
26246                            // Static. Pinned by `scc_runtime_cycles` tests.
26247                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
26248                            {
26249                                cycle_errors += 1;
26250                            }
26251                        }
26252                        ScheduleUnit::Layer(i) => {
26253                            computed_vertices +=
26254                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
26255                        }
26256                    }
26257                }
26258
26259                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
26260                if let Some(t) = telemetry.as_mut() {
26261                    t.changed_vdeps_total += changed_vertices.len();
26262                }
26263                self.resource_checkpoint(0)?;
26264                self.graph.clear_dirty_flags(&to_evaluate);
26265                for v in &changed_vertices {
26266                    self.graph.set_dirty(*v, true);
26267                }
26268
26269                if changed_vertices.is_empty() {
26270                    if let Some(t) = telemetry.as_mut() {
26271                        t.bailout_reason = Some("converged");
26272                    }
26273                    break;
26274                }
26275                if replan_iterations >= MAX_REPLAN {
26276                    if let Some(mut t) = telemetry.take() {
26277                        t.bailout_reason = Some("max_replan");
26278                        t.replan_iterations = replan_iterations;
26279                        self.last_virtual_dep_telemetry = t;
26280                    }
26281                    return Err(
26282                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
26283                    );
26284                }
26285                replan_iterations += 1;
26286            }
26287
26288            if let Some(mut t) = telemetry {
26289                t.replan_iterations = replan_iterations;
26290                self.last_virtual_dep_telemetry = t;
26291            }
26292
26293            self.redirty_for_next_recalc();
26294            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
26295
26296            Ok(EvalResult {
26297                computed_vertices,
26298                cycle_errors,
26299                elapsed: start.elapsed(),
26300            })
26301        })();
26302        log.end_compound();
26303        result
26304    }
26305
26306    /// Evaluate a single layer with ChangeLog recording.
26307    fn evaluate_layer_logged(
26308        &mut self,
26309        layer: &super::scheduler::Layer,
26310        log: &mut ChangeLog,
26311    ) -> Result<usize, ExcelError> {
26312        self.resource_checkpoint(layer.vertices.len() as u64)?;
26313        let mut computed_writes = ComputedWriteBuffer::default();
26314        for &vertex_id in &layer.vertices {
26315            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
26316            let value = match self.evaluate_vertex_immutable(vertex_id) {
26317                Ok(v) => v,
26318                Err(e) => LiteralValue::Error(e),
26319            };
26320            let effects = match self.plan_vertex_effects_with_computed_flush(
26321                vertex_id,
26322                value,
26323                None,
26324                &mut computed_writes,
26325            ) {
26326                Ok(effects) => effects,
26327                Err(e) => {
26328                    self.flush_computed_write_buffer(&mut computed_writes)?;
26329                    return Err(e);
26330                }
26331            };
26332            for effect in &effects {
26333                if let Err(e) = self.apply_effect_with_computed_writes(
26334                    effect,
26335                    None,
26336                    Some(log),
26337                    Some(&mut computed_writes),
26338                ) {
26339                    self.flush_computed_write_buffer(&mut computed_writes)?;
26340                    return Err(e);
26341                }
26342            }
26343        }
26344        self.flush_computed_write_buffer(&mut computed_writes)?;
26345        Ok(layer.vertices.len())
26346    }
26347}