Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::prepared_legacy_graph::{
8    PreparedLegacyGraphError, PreparedLegacyGraphPlan,
9};
10use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
11use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
12use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
13use crate::engine::live_graph::analyze_live_graph;
14use crate::engine::lookup_index_cache::{
15    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
16    LookupIndexKey, estimate_bytes,
17};
18use crate::engine::named_range::{NameScope, NamedDefinition};
19use crate::engine::range_view::RangeView;
20use crate::engine::row_visibility::RowVisibilityState;
21use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
22use crate::engine::target_preparation::{
23    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
24};
25use crate::engine::used_extent::{
26    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
27};
28use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
29use crate::engine::{
30    CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
31    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
32    EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
33    FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
34    FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
35    FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
36    FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
37    VertexId, VertexKind, VisibilityMaskMode,
38};
39use crate::formula_plane::placement::prepare_anchor_once_fragment;
40use crate::formula_plane::placement::{
41    CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
42    place_candidate_family_with_analyses, prepare_anchor_once_family,
43    split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
44    validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
45};
46use crate::formula_plane::producer::{
47    DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
48    FormulaProducerWork, ProducerDirtyDomain, SpanReadSummary,
49};
50use crate::formula_plane::region_index::{DirtyDomain, Region};
51use crate::formula_plane::runtime::{
52    FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
53};
54use crate::formula_plane::scheduler::{
55    MixedSchedule, MixedScheduleFallbackReason, MixedTopology, MixedTopologyCompileResult,
56    MixedTopologyCompileStats, MixedTopologyConfig, build_demand_closure_cached,
57    build_demand_closure_in_memory_runs, build_demand_closure_paged,
58    build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
59    schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
60    schedule_dirty_work_repeated_passes,
61};
62#[cfg(not(target_arch = "wasm32"))]
63use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
64#[cfg(not(target_arch = "wasm32"))]
65use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
66#[cfg(test)]
67use crate::formula_plane::span_eval::SpanEvalReport;
68use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
69use crate::formula_plane::structural::relocate_ast_for_template_placement;
70use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
71use crate::function::FnCaps;
72use crate::interpreter::Interpreter;
73use crate::reference::{CellRef, Coord, RangeRef};
74use crate::traits::FunctionProvider;
75use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
76use formualizer_common::{
77    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
78};
79use formualizer_parse::parser::ReferenceType;
80use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
81use rayon::ThreadPoolBuilder;
82use rustc_hash::{FxHashMap, FxHashSet};
83use std::collections::{BTreeMap, BTreeSet, VecDeque};
84use std::sync::Arc;
85use std::sync::atomic::{AtomicBool, Ordering};
86use std::time::Instant;
87
88type StagedFormulaEntry = (u32, u32, String);
89type StagedSheetParts = (
90    Vec<StagedFormulaEntry>,
91    Option<crate::engine::DeferredFormulaPackage>,
92);
93
94/// Per-sheet staged-formula store (NOTE(#126) follow-up).
95///
96/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
97/// entries in INSERTION order, so the order-preserving `Vec` stays the
98/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
99/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
100/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
101/// 50k stages, release, before the index). `stage`/`get` are O(1);
102/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
103#[derive(Default)]
104pub(crate) struct StagedSheet {
105    entries: Vec<StagedFormulaEntry>,
106    index: FxHashMap<(u32, u32), usize>,
107    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
108}
109
110impl StagedSheet {
111    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
112        package
113            .invalidated
114            .extend(package.families.iter().map(|family| family.source_id));
115        package.invalidated.extend(
116            package
117                .partitioned_families
118                .iter()
119                .map(|family| family.source_id),
120        );
121    }
122
123    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
124        let Some(package) = self.deferred_package.as_mut() else {
125            return;
126        };
127        let family = package
128            .replay
129            .lock()
130            .ok()
131            .and_then(|mut replay| replay.formula_at(row, col).ok())
132            .flatten()
133            .and_then(|record| record.partition_owner.or(record.family));
134        if let Some(family) = family {
135            package.invalidated.insert(family);
136        } else {
137            // A poisoned lock, replay failure, malformed spool, or missing
138            // lookup result must never let a possibly edited family commit.
139            // Fail closed at package scope before suppressing the edited cell.
140            Self::invalidate_all_deferred_families(package);
141        }
142        package.suppressed.insert((row, col));
143    }
144
145    fn reconcile_attached_deferred_package(&mut self) {
146        let Some(package) = self.deferred_package.as_mut() else {
147            return;
148        };
149        if self.entries.is_empty() {
150            return;
151        }
152
153        let replayed = (|| {
154            let mut disposition = crate::engine::FormulaReplayDisposition::default();
155            for partition in &package.partitioned_families {
156                disposition.register_partition(partition, false)?;
157            }
158            package
159                .replay
160                .lock()
161                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
162                .replay_partitioned(&disposition, &package.partitioned_families)
163        })();
164
165        let Ok(records) = replayed else {
166            Self::invalidate_all_deferred_families(package);
167            package
168                .suppressed
169                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
170            return;
171        };
172        let mut owners = FxHashMap::default();
173        for record in &records {
174            owners
175                .entry((record.row, record.col))
176                .or_insert(record.partition_owner.or(record.family));
177        }
178        for (row, col, _) in &self.entries {
179            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
180                package.invalidated.insert(family);
181            } else {
182                Self::invalidate_all_deferred_families(package);
183            }
184            package.suppressed.insert((*row, *col));
185        }
186        package.reconciliation_replay = Some(records);
187    }
188
189    fn stage(&mut self, row: u32, col: u32, text: String) {
190        self.invalidate_deferred_at(row, col);
191        match self.index.entry((row, col)) {
192            std::collections::hash_map::Entry::Occupied(slot) => {
193                self.entries[*slot.get()].2 = text;
194            }
195            std::collections::hash_map::Entry::Vacant(slot) => {
196                slot.insert(self.entries.len());
197                self.entries.push((row, col, text));
198            }
199        }
200    }
201
202    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
203        let deferred_text = self.get(row, col);
204        self.invalidate_deferred_at(row, col);
205        let Some(idx) = self.index.remove(&(row, col)) else {
206            return deferred_text;
207        };
208        let (_, _, text) = self.entries.remove(idx);
209        // `Vec::remove` shifted everything after `idx` left by one.
210        for slot in self.index.values_mut() {
211            if *slot > idx {
212                *slot -= 1;
213            }
214        }
215        Some(text)
216    }
217
218    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
219        self.index
220            .get(&(row, col))
221            .map(|&i| self.entries[i].2.as_str())
222    }
223
224    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
225        let idx = self.index.remove(&(row, col))?;
226        let (_, _, text) = self.entries.remove(idx);
227        for slot in self.index.values_mut() {
228            if *slot > idx {
229                *slot -= 1;
230            }
231        }
232        Some(text)
233    }
234
235    fn get(&self, row: u32, col: u32) -> Option<String> {
236        if let Some(text) = self.get_ordinary(row, col) {
237            return Some(text.to_string());
238        }
239        let package = self.deferred_package.as_ref()?;
240        if package.suppressed.contains(&(row, col)) {
241            return None;
242        }
243        package
244            .replay
245            .lock()
246            .ok()?
247            .formula_at(row, col)
248            .ok()?
249            .map(|record| record.text)
250    }
251
252    fn len(&self) -> usize {
253        self.deferred_package
254            .as_ref()
255            .map_or(self.entries.len(), |package| {
256                usize::try_from(package.report.source_formula_records_spooled)
257                    .unwrap_or(usize::MAX)
258                    .saturating_sub(package.suppressed.len())
259                    .saturating_add(self.entries.len())
260            })
261    }
262
263    fn is_empty(&self) -> bool {
264        self.entries.is_empty() && self.deferred_package.is_none()
265    }
266
267    fn into_parts(self) -> StagedSheetParts {
268        (self.entries, self.deferred_package)
269    }
270}
271
272type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
273
274#[derive(Clone, Debug, PartialEq, Eq, Hash)]
275struct PreparationRegion {
276    sheet: String,
277    sheet_id: SheetId,
278    start_row: u32,
279    start_col: u32,
280    end_row: u32,
281    end_col: u32,
282}
283
284// Span roots dedupe by exact demanded-region identity. Overlapping, non-identical
285// regions remain ordered entries; downstream demand treats those entries as a union.
286struct OrderedTargetProducers {
287    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
288    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
289}
290
291impl OrderedTargetProducers {
292    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
293        let mut ordered = Vec::new();
294        ordered.try_reserve(capacity)?;
295        let mut seen = FxHashSet::default();
296        seen.try_reserve(capacity)?;
297        Ok(Self { ordered, seen })
298    }
299
300    fn from_ordered(
301        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
302    ) -> Result<Self, std::collections::TryReserveError> {
303        let mut seen = FxHashSet::default();
304        seen.try_reserve(ordered.len())?;
305        seen.extend(ordered.iter().copied());
306        Ok(Self { ordered, seen })
307    }
308
309    fn push(
310        &mut self,
311        producer: crate::engine::target_preparation::TargetProducer,
312    ) -> Result<bool, std::collections::TryReserveError> {
313        #[cfg(test)]
314        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
315        if self.seen.contains(&producer) {
316            return Ok(false);
317        }
318        self.seen.try_reserve(1)?;
319        self.ordered.try_reserve(1)?;
320        self.seen.insert(producer);
321        self.ordered.push(producer);
322        Ok(true)
323    }
324
325    fn len(&self) -> usize {
326        self.ordered.len()
327    }
328
329    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
330        self.ordered
331    }
332}
333
334#[cfg(test)]
335thread_local! {
336    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
337}
338
339fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
340    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
341        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
342        limit: u64::MAX,
343        observed: observed as u64,
344        request_id,
345    })
346    .into_excel_error()
347}
348
349#[derive(Debug)]
350struct PreparedOrdinaryStagedFormula {
351    sheet: String,
352    sheet_id: SheetId,
353    lease: StagedFormulaLease,
354    ast_id: Option<AstNodeId>,
355    plan: Option<DependencyPlanRow>,
356}
357
358struct PreparedTargetSourcePackage {
359    sheet: String,
360    sheet_id: SheetId,
361    lease: StagedPackageLease,
362    source_report: crate::engine::FormulaCompressedSourceReport,
363    replay_records: Vec<crate::engine::DeferredReplayFormula>,
364    disposition: crate::engine::FormulaReplayDisposition,
365    placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
366    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
367    direct_families: usize,
368    direct_cells: u64,
369    direct_fragments: u64,
370    direct_complete_families: u64,
371    direct_complete_cells: u64,
372    direct_partition_families: u64,
373    direct_partition_cells: u64,
374    anchor_parses: u64,
375    anchor_asts: u64,
376    anchor_analyses: u64,
377}
378
379impl PreparedTargetSourcePackage {
380    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
381        self.replay_records.iter().filter(|record| {
382            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
383                return true;
384            };
385            let coord = crate::engine::SourceCoord { row, col };
386            let disposition = match record.family {
387                Some(family) => self.disposition.shared_disposition(family, coord),
388                None => self.disposition.ordinary_disposition(coord).0,
389            };
390            !matches!(
391                disposition,
392                crate::engine::FormulaReplayCoordinateDisposition::Direct
393                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
394            )
395        })
396    }
397}
398
399fn producer_dirty_to_span_dirty(
400    dirty: ProducerDirtyDomain,
401    span_ref: FormulaSpanRef,
402) -> DirtyDomain {
403    match dirty {
404        ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
405        ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
406        ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
407    }
408}
409type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
410type StagedFormulaBatches = Vec<(String, StagedSheet)>;
411#[derive(Debug)]
412pub(crate) struct SourceFamilyPreparationError {
413    reason: String,
414    parse_attempted: bool,
415    ast_created: bool,
416    analysis_created: bool,
417}
418
419impl SourceFamilyPreparationError {
420    fn contract(reason: &'static str) -> Self {
421        Self {
422            reason: reason.to_string(),
423            parse_attempted: false,
424            ast_created: false,
425            analysis_created: false,
426        }
427    }
428
429    fn parse(reason: &'static str) -> Self {
430        Self {
431            reason: reason.to_string(),
432            parse_attempted: true,
433            ast_created: false,
434            analysis_created: false,
435        }
436    }
437
438    fn ast(reason: impl Into<String>) -> Self {
439        Self {
440            reason: reason.into(),
441            parse_attempted: true,
442            ast_created: true,
443            analysis_created: false,
444        }
445    }
446
447    fn analysis(reason: impl Into<String>) -> Self {
448        Self {
449            reason: reason.into(),
450            parse_attempted: true,
451            ast_created: true,
452            analysis_created: true,
453        }
454    }
455}
456
457struct PreparedPartitionShadow {
458    fragment_count: u64,
459    direct_cells: u64,
460    fallback_cells: u64,
461    function_semantics_used: bool,
462}
463
464pub(crate) fn classify_mixed_topology_incomplete(
465    observed: &MixedTopologyCompileStats,
466) -> crate::engine::EvaluationIncompleteReason {
467    if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
468        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
469    } else if observed.memory_overflow_count > 0 {
470        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
471    } else if observed.edge_overflow_count > 0 {
472        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
473    } else if observed.candidate_overflow_count > 0 {
474        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
475    } else {
476        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
477    }
478}
479
480#[derive(Clone, Debug, Default)]
481struct LegacyIslandPlan {
482    membership: Vec<VertexId>,
483    dirty_vertices: Vec<VertexId>,
484    sheet_ids: Vec<SheetId>,
485    island_id: u64,
486    retained_bytes: usize,
487    boundary_relationships: usize,
488    omitted_relationships: usize,
489    omitted_direct_relationships: usize,
490}
491
492impl LegacyIslandPlan {
493    fn is_empty(&self) -> bool {
494        self.membership.is_empty()
495    }
496}
497
498type FormulaPlaneMixedScheduleBuild = (
499    MixedSchedule,
500    BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
501    u64,
502    Vec<VertexId>,
503    Vec<usize>,
504    LegacyIslandPlan,
505);
506
507#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
508pub(crate) enum FormulaSpanDemotionFault {
509    #[default]
510    None,
511    AstPreparation,
512    LegacyGraphPreparation,
513    FinalLegacyGraphValidation,
514    FinalAuthorityValidation,
515    AllocationReservation,
516    BeforeFirstMutation,
517}
518
519#[derive(Debug)]
520pub(crate) enum FormulaSpanDemotionError {
521    StaleAuthority,
522    InvalidSpan,
523    CountOverflow,
524    Resource(ExcelError),
525    AstPreparation(ExcelError),
526    LegacyGraph(PreparedLegacyGraphError),
527    Injected(FormulaSpanDemotionFault),
528}
529
530impl std::fmt::Display for FormulaSpanDemotionError {
531    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
532        match self {
533            Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
534            Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
535            Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
536            Self::Resource(error) => write!(f, "{error}"),
537            Self::AstPreparation(error) => {
538                write!(f, "FormulaPlane demotion analysis failed: {error}")
539            }
540            Self::LegacyGraph(error) => {
541                write!(f, "FormulaPlane demotion graph plan failed: {error}")
542            }
543            Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
544        }
545    }
546}
547
548impl std::error::Error for FormulaSpanDemotionError {}
549
550pub(crate) struct PreparedFormulaSpanDemotion {
551    span_refs: Vec<FormulaSpanRef>,
552    expected_plane_epoch: u64,
553    expected_indexes_epoch: u64,
554    expected_indexed_plane_epoch: u64,
555    placement_count: usize,
556    legacy_graph: PreparedLegacyGraphPlan,
557    fault: FormulaSpanDemotionFault,
558}
559
560#[derive(Clone, Copy, Debug, PartialEq, Eq)]
561pub(crate) struct FormulaSpanDemotionReport {
562    pub(crate) spans_demoted: usize,
563    pub(crate) placements_materialized: usize,
564}
565
566type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
567type CompressedReplayBatch = (
568    FormulaIngestBatch,
569    crate::engine::FormulaCompressedSourceBatch,
570);
571type PreparedSourceBatch = (
572    FormulaIngestBatch,
573    crate::engine::FormulaCompressedSourceReport,
574    crate::engine::FormulaCompressedPreparation,
575);
576type PreparedStagedFormulaBatches = (
577    PreparedFormulaBatches,
578    Vec<CompressedReplayBatch>,
579    Vec<PreparedSourceBatch>,
580);
581
582/// Backend-neutral source-family ingress. Adapters may prepare candidates and
583/// submit exact replay, but FormulaPlane authority remains engine-owned.
584#[doc(hidden)]
585pub struct SourceFormulaIngress<'a, R> {
586    engine: &'a mut Engine<R>,
587}
588
589impl<R> SourceFormulaIngress<'_, R>
590where
591    R: EvaluationContext,
592{
593    pub fn prepare_families(
594        &mut self,
595        sheet_name: &str,
596        families: &[crate::engine::SourceFormulaFamily],
597    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
598        self.engine.observe_function_semantic_epoch()?;
599        Ok(self
600            .engine
601            .prepare_source_formula_families(sheet_name, families))
602    }
603
604    pub fn prepare_eager_proposals(
605        &mut self,
606        sheet_name: &str,
607        families: &[crate::engine::SourceFormulaFamily],
608        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
609        formula_record_count: u64,
610        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
611    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
612        self.engine.observe_function_semantic_epoch()?;
613        let replay = Arc::new(std::sync::Mutex::new(replay));
614        self.engine.prepare_source_formula_proposals(
615            sheet_name,
616            families,
617            partitions,
618            partitions,
619            formula_record_count,
620            replay,
621            &BTreeSet::new(),
622        )
623    }
624
625    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
626        self.engine.stage_deferred_formula_package(package);
627    }
628
629    pub fn ingest_replay_batches(
630        &mut self,
631        batches: Vec<(
632            FormulaIngestBatch,
633            crate::engine::FormulaCompressedSourceBatch,
634        )>,
635    ) -> Result<FormulaIngestReport, ExcelError> {
636        self.engine
637            .ingest_compressed_formula_source_batches(batches)
638    }
639
640    pub fn finish_prepared(
641        &mut self,
642        batches: Vec<(
643            FormulaIngestBatch,
644            crate::engine::FormulaCompressedSourceReport,
645            crate::engine::FormulaCompressedPreparation,
646        )>,
647    ) -> Result<FormulaIngestReport, ExcelError> {
648        self.engine.finish_compressed_formula_sources(batches)
649    }
650}
651
652// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
653// layers there is not enough work to amortize it, and the direct point-write path
654// is faster while preserving the same visibility semantics.
655const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
656
657#[derive(Debug, Clone, PartialEq)]
658pub(crate) enum ComputedWrite {
659    Cell {
660        seq: u64,
661        sheet_id: SheetId,
662        row0: u32,
663        col0: u32,
664        value: OverlayValue,
665    },
666    Rect {
667        seq: u64,
668        sheet_id: SheetId,
669        sr0: u32,
670        sc0: u32,
671        values: Vec<Vec<OverlayValue>>,
672    },
673}
674
675impl ComputedWrite {
676    #[inline]
677    pub(crate) fn seq(&self) -> u64 {
678        match self {
679            ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
680        }
681    }
682}
683
684#[derive(Debug, Default)]
685pub(crate) struct ComputedWriteBuffer {
686    writes: Vec<ComputedWrite>,
687    next_seq: u64,
688    estimated_bytes: usize,
689}
690
691impl ComputedWriteBuffer {
692    const ENTRY_BASE_BYTES: usize = 32;
693
694    #[inline]
695    pub(crate) fn is_empty(&self) -> bool {
696        self.writes.is_empty()
697    }
698
699    #[inline]
700    pub(crate) fn len(&self) -> usize {
701        self.writes.len()
702    }
703
704    #[inline]
705    pub(crate) fn estimated_bytes(&self) -> usize {
706        self.estimated_bytes
707    }
708
709    #[inline]
710    pub(crate) fn writes(&self) -> &[ComputedWrite] {
711        &self.writes
712    }
713
714    pub(crate) fn push_cell(
715        &mut self,
716        sheet_id: SheetId,
717        row0: u32,
718        col0: u32,
719        value: OverlayValue,
720    ) {
721        let seq = self.next_sequence();
722        self.estimated_bytes = self
723            .estimated_bytes
724            .saturating_add(Self::estimate_value_bytes(&value));
725        self.writes.push(ComputedWrite::Cell {
726            seq,
727            sheet_id,
728            row0,
729            col0,
730            value,
731        });
732    }
733
734    pub(crate) fn push_rect(
735        &mut self,
736        sheet_id: SheetId,
737        sr0: u32,
738        sc0: u32,
739        values: Vec<Vec<OverlayValue>>,
740    ) {
741        let seq = self.next_sequence();
742        let added = values
743            .iter()
744            .flat_map(|row| row.iter())
745            .map(Self::estimate_value_bytes)
746            .fold(0usize, usize::saturating_add);
747        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
748        self.writes.push(ComputedWrite::Rect {
749            seq,
750            sheet_id,
751            sr0,
752            sc0,
753            values,
754        });
755    }
756
757    pub(crate) fn clear(&mut self) {
758        self.writes.clear();
759        self.estimated_bytes = 0;
760    }
761
762    fn take_writes(&mut self) -> Vec<ComputedWrite> {
763        self.estimated_bytes = 0;
764        std::mem::take(&mut self.writes)
765    }
766
767    fn next_sequence(&mut self) -> u64 {
768        let seq = self.next_seq;
769        self.next_seq = self.next_seq.wrapping_add(1);
770        seq
771    }
772
773    #[inline]
774    fn estimate_value_bytes(value: &OverlayValue) -> usize {
775        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
776    }
777}
778
779#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
780struct ComputedWriteChunkKey {
781    sheet_id: SheetId,
782    col0: u32,
783    chunk_idx: usize,
784    chunk_start_row0: u32,
785}
786
787#[derive(Debug, Clone, PartialEq)]
788pub(crate) struct ComputedWriteChunkEntryPlan {
789    pub(crate) row_in_chunk: usize,
790    pub(crate) seq: u64,
791    pub(crate) value: OverlayValue,
792}
793
794#[derive(Debug, Clone, PartialEq, Eq)]
795pub(crate) enum ComputedWriteChunkPlanShape {
796    Point,
797    SparseOffsets {
798        entries: usize,
799        span_len: usize,
800    },
801    DenseRange {
802        start: usize,
803        len: usize,
804    },
805    RunRange {
806        start: usize,
807        len: usize,
808        runs: usize,
809    },
810}
811
812#[derive(Debug, Clone, PartialEq)]
813pub(crate) struct ComputedWriteChunkPlan {
814    pub(crate) sheet_id: SheetId,
815    pub(crate) col0: u32,
816    pub(crate) chunk_idx: usize,
817    pub(crate) chunk_start_row0: u32,
818    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
819    pub(crate) shape: ComputedWriteChunkPlanShape,
820}
821
822#[derive(Debug, Clone, Default, PartialEq)]
823pub(crate) struct ComputedWriteCoalescingPlan {
824    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
825    pub(crate) input_cells: usize,
826    pub(crate) coalesced_cells: usize,
827    pub(crate) overwritten_cells: usize,
828}
829
830impl ComputedWriteCoalescingPlan {
831    #[inline]
832    pub(crate) fn is_empty(&self) -> bool {
833        self.chunks.is_empty()
834    }
835}
836
837impl ComputedWriteChunkPlan {
838    fn from_group(
839        key: ComputedWriteChunkKey,
840        mut entries: Vec<ComputedWriteChunkEntryPlan>,
841    ) -> (Self, usize) {
842        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
843        let input_len = entries.len();
844        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
845        for entry in entries {
846            if let Some(prev) = coalesced.last_mut()
847                && prev.row_in_chunk == entry.row_in_chunk
848            {
849                *prev = entry;
850                continue;
851            }
852            coalesced.push(entry);
853        }
854        let overwritten = input_len.saturating_sub(coalesced.len());
855        let shape = Self::classify_shape(&coalesced);
856        (
857            Self {
858                sheet_id: key.sheet_id,
859                col0: key.col0,
860                chunk_idx: key.chunk_idx,
861                chunk_start_row0: key.chunk_start_row0,
862                entries: coalesced,
863                shape,
864            },
865            overwritten,
866        )
867    }
868
869    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
870        debug_assert!(!entries.is_empty());
871        if entries.len() == 1 {
872            return ComputedWriteChunkPlanShape::Point;
873        }
874
875        let start = entries[0].row_in_chunk;
876        let end = entries[entries.len() - 1].row_in_chunk;
877        let span_len = end.saturating_sub(start).saturating_add(1);
878        if span_len != entries.len() {
879            return ComputedWriteChunkPlanShape::SparseOffsets {
880                entries: entries.len(),
881                span_len,
882            };
883        }
884
885        let runs = Self::run_count(entries);
886        if runs < entries.len() {
887            ComputedWriteChunkPlanShape::RunRange {
888                start,
889                len: entries.len(),
890                runs,
891            }
892        } else {
893            ComputedWriteChunkPlanShape::DenseRange {
894                start,
895                len: entries.len(),
896            }
897        }
898    }
899
900    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
901        let mut runs = 0usize;
902        let mut prev: Option<&OverlayValue> = None;
903        for entry in entries {
904            if prev != Some(&entry.value) {
905                runs = runs.saturating_add(1);
906                prev = Some(&entry.value);
907            }
908        }
909        runs
910    }
911}
912
913pub struct Engine<R> {
914    pub(crate) graph: DependencyGraph,
915    resolver: R,
916    pub config: EvalConfig,
917    workbook_load_limits: crate::engine::WorkbookLoadLimits,
918    /// Clock for volatile date/time builtins, wrapped in a per-recalc
919    /// snapshot: sampled once at the start of every evaluation request
920    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
921    /// one recalc — including SCC iteration passes — agree (spec §7.11).
922    clock: crate::timezone::SnapshotClock,
923    thread_pool: Option<Arc<rayon::ThreadPool>>,
924    pub recalc_epoch: u64,
925    snapshot_id: std::sync::atomic::AtomicU64,
926    topology_epoch: u64,
927    /// False after a structural axis operation. While #171 remains open we
928    /// must not use relocated span read summaries to prove disconnection.
929    legacy_island_structural_summaries_trusted: bool,
930    cached_static_schedule: Option<CachedScheduleEntry>,
931    cached_mixed_topology: Option<CachedMixedTopology>,
932    mixed_topology_cache_builds: u64,
933    mixed_topology_cache_hits: u64,
934    mixed_topology_cache_overflows: u64,
935    mixed_topology_cache_skip_streak: u64,
936    spill_mgr: ShimSpillManager,
937    /// Arrow-backed storage for sheet values (Phase A)
938    arrow_sheets: SheetStore,
939    /// True if any edit after bulk load; disables Arrow reads for parity
940    has_edited: bool,
941    /// Overlay compaction counter (Phase C instrumentation)
942    overlay_compactions: u64,
943
944    // Overlay memory observability / budget (ticket 503)
945    computed_overlay_bytes_estimate: usize,
946    computed_overlay_mirroring_disabled: bool,
947    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
948    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
949    pub(crate) force_materialize_range_views: bool,
950    // Pass-scoped cache for Arrow used-row bounds per column
951    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
952    // Snapshot-scoped final used-axis bounds for open-ended references.
953    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
954    lookup_index_cache: LookupIndexCache,
955    source_cache: Arc<std::sync::RwLock<SourceCache>>,
956    /// Identity binding for opaque source-family preparations.
957    source_formula_token: Arc<()>,
958    /// Dedicated identity binding for reusable recalculation plans.
959    recalc_plan_token: Arc<()>,
960    /// Staged formulas by sheet when `defer_graph_building` is enabled.
961    staged_formulas: StagedFormulaMap,
962    /// Presence and generation authority for ordinary staged formula discovery.
963    staged_formula_index: StagedFormulaIndex,
964    /// Per-sheet row visibility sidecar state.
965    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
966    /// Cached row visibility masks keyed by sheet/span/mode/version.
967    row_visibility_mask_cache: std::sync::RwLock<
968        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
969    >,
970    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
971    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
972    /// Last centralized formula ingest report.
973    last_formula_ingest_report: Option<FormulaIngestReport>,
974    /// Aggregate centralized formula ingest report for this engine.
975    formula_ingest_report_total: FormulaIngestReport,
976    /// Count of FormulaPlane spans demoted to legacy because one or more of
977    /// their member cells participate in a statically-cyclic SCC. A span member
978    /// must never be span-evaluated (gotcha G8 of the cycle-architecture track,
979    /// refs #112): under `CycleDetection::Static` the cycle stamping would race
980    /// span writes, and under `Runtime` SCC members must be evaluated by the
981    /// legacy `evaluate_scc_unit` path. Cyclic spans are demoted at
982    /// schedule-build time (the earliest point cross-cell cycles through span
983    /// producers become visible) so the cycle members land on the legacy graph
984    /// path. Observational only.
985    formula_plane_cycle_member_span_demotions: u64,
986    /// Successfully completed non-cycle unsafe mixed requests. Incremented
987    /// only after every scheduled span has committed demotion and the single
988    /// legacy completion pass succeeds; failed attempts are not counted.
989    /// Observational only.
990    formula_plane_capacity_bailouts: u64,
991    /// Exact span candidates classified across structural mutations.
992    formula_plane_structural_span_candidates: u64,
993    /// Transient cancellation flag used during evaluation
994    active_cancel_flag: Option<crate::engine::CancelToken>,
995    /// Transient absolute deadline used by composed target and plan requests.
996    active_evaluation_deadline: Option<Instant>,
997
998    /// Engine-level action depth.
999    ///
1000    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1001    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1002    action_depth: u32,
1003
1004    // Phase 3b virtual-dependency convergence telemetry
1005    last_virtual_dep_telemetry: VirtualDepTelemetry,
1006    virtual_dep_fallback_activations: u64,
1007
1008    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1009    last_cycle_telemetry: CycleTelemetry,
1010
1011    // C0 evaluation-resource observability. IDs are never reset or reused.
1012    next_evaluation_resource_request_id: u64,
1013    evaluation_resource_request_depth: usize,
1014    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1015    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1016    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1017    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1018    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1019    evaluation_resource_config_diagnostic:
1020        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1021    active_resource_ledger: Option<ResourceLedger>,
1022
1023    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1024    /// with a witnessed live cycle) during the current evaluation request.
1025    ///
1026    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1027    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1028    /// members clean after a recalc and would otherwise skip them forever.
1029    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1030    /// the end of the same recalc that iterated them
1031    /// ([`Self::redirty_for_next_recalc`], called wherever
1032    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1033    /// if an edit breaks the cycle, the next recalc's SCC task either does
1034    /// not exist or settles as phantom, nothing re-registers, and the
1035    /// redirty chain stops by itself.
1036    pending_iterative_redirty: Vec<VertexId>,
1037
1038    /// Final committed values of iterating-SCC members as of the end of the
1039    /// most recent recalc (spec §4 persistence). In canonical (value-cache
1040    /// disabled) mode the computed overlay is the ONLY home of a formula's
1041    /// value, and structural edits clear computed overlays wholesale
1042    /// (`clear_computed_overlay_after_row/_col`) — destroying iteration
1043    /// state (accumulators reset to 0; found by the iterate edge corpus).
1044    /// This snapshot, refreshed by [`Self::redirty_for_next_recalc`], lets
1045    /// the next SCC task re-seed members whose overlay entry vanished.
1046    /// Empty unless something iterated — zero cost otherwise.
1047    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1048
1049    /// Global function-registry semantic epoch observed after the latest
1050    /// conservative FormulaPlane invalidation.
1051    function_semantic_epoch_seen: u64,
1052    /// Runtime-provider semantic revision observed after the latest conservative
1053    /// FormulaPlane invalidation.
1054    function_provider_revision_seen: Option<u64>,
1055
1056    #[cfg(test)]
1057    last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1058    #[cfg(test)]
1059    evaluation_request_begin_count_for_test: u64,
1060    #[cfg(any(test, feature = "test-support"))]
1061    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1062    #[cfg(test)]
1063    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1064    #[cfg(test)]
1065    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1066    #[cfg(test)]
1067    inject_target_semantic_stale_once_for_test: bool,
1068    #[cfg(test)]
1069    force_virtual_dep_changes_remaining_for_test: usize,
1070    #[cfg(test)]
1071    fail_evaluation_commit_preflight_once_for_test: bool,
1072    #[cfg(test)]
1073    force_source_family_fallback: bool,
1074    #[cfg(test)]
1075    rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1076    #[cfg(test)]
1077    fragmented_commit_fault_for_test:
1078        Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1079    #[cfg(test)]
1080    formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1081    #[cfg(test)]
1082    target_preparation_fault_for_test:
1083        Option<crate::engine::target_preparation::TargetPreparationFault>,
1084    #[cfg(test)]
1085    force_non_cycle_schedule_fallback_for_test: bool,
1086    #[cfg(test)]
1087    mixed_topology_index_builds_for_test: u64,
1088    #[cfg(test)]
1089    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1090    #[cfg(test)]
1091    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1092}
1093
1094/// Minimal edit surface used by `Engine::action`.
1095///
1096fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1097    fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1098        let node_type = match &ast.node_type {
1099            ASTNodeType::Literal(_) if !in_array => {
1100                let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1101                *next = next.saturating_add(1);
1102                ASTNodeType::Literal(value)
1103            }
1104            ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1105            ASTNodeType::Omitted => ASTNodeType::Omitted,
1106            ASTNodeType::Reference {
1107                original,
1108                reference,
1109            } => ASTNodeType::Reference {
1110                original: original.clone(),
1111                reference: reference.clone(),
1112            },
1113            ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1114                op: op.clone(),
1115                expr: Box::new(visit(expr, binding, next, in_array)),
1116            },
1117            ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1118                op: op.clone(),
1119                left: Box::new(visit(left, binding, next, in_array)),
1120                right: Box::new(visit(right, binding, next, in_array)),
1121            },
1122            ASTNodeType::Function { name, args } => ASTNodeType::Function {
1123                name: name.clone(),
1124                args: args
1125                    .iter()
1126                    .map(|arg| visit(arg, binding, next, in_array))
1127                    .collect(),
1128            },
1129            ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1130                callee: Box::new(visit(callee, binding, next, in_array)),
1131                args: args
1132                    .iter()
1133                    .map(|arg| visit(arg, binding, next, in_array))
1134                    .collect(),
1135            },
1136            ASTNodeType::Array(rows) => ASTNodeType::Array(
1137                rows.iter()
1138                    .map(|row| {
1139                        row.iter()
1140                            .map(|cell| visit(cell, binding, next, true))
1141                            .collect()
1142                    })
1143                    .collect(),
1144            ),
1145        };
1146        ASTNode::new(node_type, ast.source_token.clone())
1147    }
1148
1149    let mut next = 0;
1150    visit(ast, binding, &mut next, false)
1151}
1152
1153/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1154/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1155impl<R: EvaluationContext> Engine<R> {
1156    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1157        self.graph.ingest_pipeline(&self.resolver)
1158    }
1159}
1160
1161pub struct EngineAction<'a, R>
1162where
1163    R: EvaluationContext,
1164{
1165    engine: &'a mut Engine<R>,
1166    name: String,
1167    // Optional external ChangeLog pointer used by `Engine::action_with_logger`.
1168    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1169    log: Option<*mut crate::engine::ChangeLog>,
1170    // Optional Arrow undo journal used by `Engine::action_atomic`.
1171    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1172    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1173    // True when this EngineAction must enforce conservative atomic transaction policy.
1174    atomic_policy: bool,
1175}
1176
1177impl<'a, R> EngineAction<'a, R>
1178where
1179    R: EvaluationContext,
1180{
1181    #[inline]
1182    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1183        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1184        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1185        crate::reference::CellRef::new(sheet_id, coord)
1186    }
1187
1188    #[inline]
1189    pub fn name(&self) -> &str {
1190        &self.name
1191    }
1192
1193    #[inline]
1194    pub fn set_cell_value(
1195        &mut self,
1196        sheet: &str,
1197        row: u32,
1198        col: u32,
1199        value: LiteralValue,
1200    ) -> Result<(), crate::engine::EditorError> {
1201        if self.log.is_some() {
1202            let old_value = self.engine.read_cell_value(sheet, row, col);
1203            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1204            let addr = self.addr_for(sheet, row, col);
1205            let Some(log_ptr) = self.log else {
1206                return Err(crate::engine::EditorError::TransactionFailed {
1207                    reason: "action_with_logger: missing ChangeLog".to_string(),
1208                });
1209            };
1210
1211            // For atomic journal mode, record computed overlay effects for this cell.
1212            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1213            let old_comp = if self.arrow_undo.is_some() {
1214                self.engine.read_computed_overlay_cell(sheet, row, col)
1215            } else {
1216                None
1217            };
1218
1219            if self.engine.graph_admission_enabled() {
1220                let admission =
1221                    self.engine
1222                        .graph
1223                        .preview_value_mutation(addr.sheet_id, row, col)?;
1224                self.engine.preflight_graph_admission(admission)?;
1225            }
1226            self.engine.demote_span_containing_cell_for_write(
1227                addr.sheet_id,
1228                addr.coord.row(),
1229                addr.coord.col(),
1230            )?;
1231            if old_formula.is_none() {
1232                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1233            }
1234
1235            let delta_old_sem = if old_formula.is_some() {
1236                None
1237            } else {
1238                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1239            };
1240
1241            let start_len = unsafe { (&*log_ptr).len() };
1242
1243            // Safety: `log_ptr` comes from a unique `&mut ChangeLog` in `Engine::action_with_logger`.
1244            let log = unsafe { &mut *log_ptr };
1245            self.engine.edit_with_logger(log, |editor| {
1246                editor.set_cell_value_with_old_state(
1247                    addr,
1248                    value.clone(),
1249                    old_value.clone(),
1250                    old_formula.clone(),
1251                );
1252            })?;
1253            self.engine
1254                .record_formula_plane_structural_change(StructuralScope::Cell {
1255                    sheet: addr.sheet_id,
1256                    row: addr.coord.row(),
1257                    col: addr.coord.col(),
1258                });
1259
1260            if let Some(undo_ptr) = self.arrow_undo {
1261                // 1) Spill snapshot operations (computed overlay rect restore).
1262                let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1263                let undo = unsafe { &mut *undo_ptr };
1264                self.engine
1265                    .record_spill_ops_into_arrow_undo(undo, new_events);
1266
1267                // 2) Delta/computed overlay single-cell deltas.
1268                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1269                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1270                let row0 = row.saturating_sub(1);
1271                let col0 = col.saturating_sub(1);
1272                let delta_new_sem = Some(value.clone());
1273                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1274                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1275            }
1276            Ok(())
1277        } else {
1278            self.engine
1279                .set_cell_value(sheet, row, col, value)
1280                .map_err(crate::engine::EditorError::from)
1281        }
1282    }
1283
1284    #[inline]
1285    pub fn set_cell_formula(
1286        &mut self,
1287        sheet: &str,
1288        row: u32,
1289        col: u32,
1290        ast: ASTNode,
1291    ) -> Result<(), crate::engine::EditorError> {
1292        if self.log.is_some() {
1293            let old_value = self.engine.read_cell_value(sheet, row, col);
1294            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1295            let addr = self.addr_for(sheet, row, col);
1296            let Some(log_ptr) = self.log else {
1297                return Err(crate::engine::EditorError::TransactionFailed {
1298                    reason: "action_with_logger: missing ChangeLog".to_string(),
1299                });
1300            };
1301
1302            let admitted_formula = if self.engine.graph_admission_enabled() {
1303                let placement =
1304                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1305                let ingested = self.engine.ingest_pipeline().ingest_formula(
1306                    FormulaAstInput::Tree(ast.clone()),
1307                    placement,
1308                    None,
1309                )?;
1310                let admission = self.engine.graph.preview_formula_mutations(&[(
1311                    addr.sheet_id,
1312                    row,
1313                    col,
1314                    ingested.dep_plan.clone(),
1315                )])?;
1316                self.engine.preflight_graph_admission(admission)?;
1317                Some((ingested.ast_id, ingested.dep_plan))
1318            } else {
1319                None
1320            };
1321            self.engine.demote_span_containing_cell_for_write(
1322                addr.sheet_id,
1323                addr.coord.row(),
1324                addr.coord.col(),
1325            )?;
1326            if old_formula.is_none() {
1327                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1328            }
1329            let delta_old = if self.arrow_undo.is_some() {
1330                if old_formula.is_some() {
1331                    None
1332                } else {
1333                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1334                }
1335            } else {
1336                None
1337            };
1338            let start_len = unsafe { (&*log_ptr).len() };
1339
1340            // Safety: `log_ptr` comes from a unique `&mut ChangeLog` in `Engine::action_with_logger`.
1341            let log = unsafe { &mut *log_ptr };
1342            self.engine.edit_with_logger(log, |editor| {
1343                if let Some((ast_id, plan)) = admitted_formula {
1344                    editor.set_cell_formula_with_prepared_plan(
1345                        addr,
1346                        ast.clone(),
1347                        old_value,
1348                        old_formula,
1349                        ast_id,
1350                        plan,
1351                    );
1352                } else {
1353                    editor.set_cell_formula_with_old_state(
1354                        addr,
1355                        ast.clone(),
1356                        old_value,
1357                        old_formula,
1358                    );
1359                }
1360            })?;
1361            self.engine
1362                .record_formula_plane_structural_change(StructuralScope::Cell {
1363                    sheet: addr.sheet_id,
1364                    row: addr.coord.row(),
1365                    col: addr.coord.col(),
1366                });
1367
1368            if let Some(undo_ptr) = self.arrow_undo {
1369                let new_events = &unsafe { (&*log_ptr).events() }[start_len..];
1370                let undo = unsafe { &mut *undo_ptr };
1371                self.engine
1372                    .record_spill_ops_into_arrow_undo(undo, new_events);
1373                let delta_new: Option<LiteralValue> = None;
1374                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1375                let row0 = row.saturating_sub(1);
1376                let col0 = col.saturating_sub(1);
1377                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1378            }
1379            Ok(())
1380        } else {
1381            self.engine
1382                .set_cell_formula(sheet, row, col, ast)
1383                .map_err(crate::engine::EditorError::from)
1384        }
1385    }
1386
1387    #[inline]
1388    pub fn set_row_hidden(
1389        &mut self,
1390        sheet: &str,
1391        row_1based: u32,
1392        hidden: bool,
1393        source: RowVisibilitySource,
1394    ) -> Result<(), crate::engine::EditorError> {
1395        if self.log.is_some() {
1396            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1397            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1398            let old_hidden = self
1399                .engine
1400                .row_visibility
1401                .get(&sheet_id)
1402                .map(|state| state.is_row_hidden(row0, Some(source)))
1403                .unwrap_or(false);
1404            if old_hidden == hidden {
1405                return Ok(());
1406            }
1407
1408            let _ = self
1409                .engine
1410                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1411
1412            let Some(log_ptr) = self.log else {
1413                return Err(crate::engine::EditorError::TransactionFailed {
1414                    reason: "action_with_logger: missing ChangeLog".to_string(),
1415                });
1416            };
1417            unsafe { &mut *log_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1418                sheet_id,
1419                row0,
1420                source,
1421                old_hidden,
1422                new_hidden: hidden,
1423            });
1424
1425            Ok(())
1426        } else {
1427            self.engine
1428                .set_row_hidden(sheet, row_1based, hidden, source)
1429        }
1430    }
1431
1432    #[inline]
1433    pub fn set_rows_hidden(
1434        &mut self,
1435        sheet: &str,
1436        start_row_1based: u32,
1437        end_row_1based: u32,
1438        hidden: bool,
1439        source: RowVisibilitySource,
1440    ) -> Result<(), crate::engine::EditorError> {
1441        if self.log.is_some() {
1442            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1443            let (start_row0, end_row0) =
1444                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1445
1446            let Some(log_ptr) = self.log else {
1447                return Err(crate::engine::EditorError::TransactionFailed {
1448                    reason: "action_with_logger: missing ChangeLog".to_string(),
1449                });
1450            };
1451            let log = unsafe { &mut *log_ptr };
1452
1453            for row0 in start_row0..=end_row0 {
1454                let old_hidden = self
1455                    .engine
1456                    .row_visibility
1457                    .get(&sheet_id)
1458                    .map(|state| state.is_row_hidden(row0, Some(source)))
1459                    .unwrap_or(false);
1460                if old_hidden == hidden {
1461                    continue;
1462                }
1463
1464                let _ = self
1465                    .engine
1466                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1467
1468                log.record(crate::engine::ChangeEvent::SetRowVisibility {
1469                    sheet_id,
1470                    row0,
1471                    source,
1472                    old_hidden,
1473                    new_hidden: hidden,
1474                });
1475            }
1476
1477            Ok(())
1478        } else {
1479            self.engine
1480                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1481        }
1482    }
1483
1484    #[inline]
1485    pub fn insert_rows(
1486        &mut self,
1487        sheet: &str,
1488        before: u32,
1489        count: u32,
1490    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1491        if count == 0 {
1492            return Ok(crate::engine::ShiftSummary::default());
1493        }
1494        if self.log.is_some() {
1495            let Some(log_ptr) = self.log else {
1496                return Err(crate::engine::EditorError::TransactionFailed {
1497                    reason: "action_atomic: missing ChangeLog".to_string(),
1498                });
1499            };
1500
1501            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1502            let before0 = before.saturating_sub(1);
1503            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1504            // Authority geometry is not journaled. Materialize affected spans
1505            // before the logged graph shift so undo/redo remains exact instead
1506            // of leaving an unlogged shifted span behind.
1507            self.engine
1508                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1509
1510            // Graph structural insert (logged) - no snapshot bump.
1511            let summary = {
1512                let log = unsafe { &mut *log_ptr };
1513                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1514                    Ok(crate::engine::ShiftSummary::default());
1515                self.engine.edit_with_logger(log, |editor| {
1516                    out = editor.insert_rows(sheet_id, before0, count);
1517                })?;
1518                out?
1519            };
1520
1521            // Arrow insert (truth) + undo op.
1522            self.engine.ensure_arrow_sheet(sheet);
1523            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1524                asheet.insert_rows(before0 as usize, count as usize);
1525            }
1526            self.engine
1527                .shift_row_visibility_insert(sheet_id, before0, count);
1528            self.engine.mark_moved_formula_vertices_dirty(&summary);
1529            self.engine
1530                .clear_computed_overlay_after_row(sheet, before0 as usize);
1531            self.engine
1532                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1533            self.engine.mark_topology_edited();
1534            if let Some(undo_ptr) = self.arrow_undo {
1535                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1536            }
1537            Ok(summary)
1538        } else {
1539            self.engine.insert_rows(sheet, before, count)
1540        }
1541    }
1542
1543    #[inline]
1544    pub fn delete_rows(
1545        &mut self,
1546        sheet: &str,
1547        start: u32,
1548        count: u32,
1549    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1550        if count == 0 {
1551            return Ok(crate::engine::ShiftSummary::default());
1552        }
1553        if self.atomic_policy {
1554            return Err(crate::engine::EditorError::TransactionUnsupported {
1555                reason:
1556                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1557                        .to_string(),
1558            });
1559        }
1560        self.engine.delete_rows(sheet, start, count)
1561    }
1562
1563    #[inline]
1564    pub fn insert_columns(
1565        &mut self,
1566        sheet: &str,
1567        before: u32,
1568        count: u32,
1569    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1570        if count == 0 {
1571            return Ok(crate::engine::ShiftSummary::default());
1572        }
1573        if self.log.is_some() {
1574            let Some(log_ptr) = self.log else {
1575                return Err(crate::engine::EditorError::TransactionFailed {
1576                    reason: "action_atomic: missing ChangeLog".to_string(),
1577                });
1578            };
1579
1580            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1581            let before0 = before.saturating_sub(1);
1582            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1583            self.engine
1584                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1585
1586            let summary = {
1587                let log = unsafe { &mut *log_ptr };
1588                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1589                    Ok(crate::engine::ShiftSummary::default());
1590                self.engine.edit_with_logger(log, |editor| {
1591                    out = editor.insert_columns(sheet_id, before0, count);
1592                })?;
1593                out?
1594            };
1595
1596            self.engine.ensure_arrow_sheet(sheet);
1597            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1598                asheet.insert_columns(before0 as usize, count as usize);
1599            }
1600            self.engine.mark_moved_formula_vertices_dirty(&summary);
1601            self.engine
1602                .clear_computed_overlay_after_col(sheet, before0 as usize);
1603            self.engine
1604                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1605            self.engine.mark_topology_edited();
1606            if let Some(undo_ptr) = self.arrow_undo {
1607                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1608            }
1609            Ok(summary)
1610        } else {
1611            self.engine.insert_columns(sheet, before, count)
1612        }
1613    }
1614
1615    #[inline]
1616    pub fn delete_columns(
1617        &mut self,
1618        sheet: &str,
1619        start: u32,
1620        count: u32,
1621    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1622        if count == 0 {
1623            return Ok(crate::engine::ShiftSummary::default());
1624        }
1625        if self.atomic_policy {
1626            return Err(crate::engine::EditorError::TransactionUnsupported {
1627                reason:
1628                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1629                        .to_string(),
1630            });
1631        }
1632        self.engine.delete_columns(sheet, start, count)
1633    }
1634
1635    /// Start an action from within an action.
1636    ///
1637    /// Nested actions are currently disallowed (ticket 614), so this will return a
1638    /// `EditorError::TransactionFailed` while an outer action is active.
1639    #[inline]
1640    pub fn action<T>(
1641        &mut self,
1642        name: impl AsRef<str>,
1643        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1644    ) -> Result<T, crate::engine::EditorError> {
1645        self.engine.action(name, f)
1646    }
1647}
1648
1649struct ActionDepthGuard<'a, R> {
1650    engine: *mut Engine<R>,
1651    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1652}
1653
1654impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1655    fn drop(&mut self) {
1656        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1657        // than the surrounding `Engine::action` call.
1658        unsafe {
1659            let e = &mut *self.engine;
1660            e.action_depth = e.action_depth.saturating_sub(1);
1661        }
1662    }
1663}
1664
1665#[derive(Default)]
1666struct SourceCache {
1667    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1668    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1669}
1670
1671#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1672struct VisibilityMaskCacheKey {
1673    sheet_id: SheetId,
1674    start_row0: u32,
1675    end_row0: u32,
1676    mode: VisibilityMaskMode,
1677    version: u64,
1678}
1679
1680#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1681enum StructuralScope {
1682    Cell { sheet: SheetId, row: u32, col: u32 },
1683    Region(Region),
1684    Sheet(SheetId),
1685    RemovedSheet(SheetId),
1686    OpaqueGlobal,
1687    AllSheets,
1688}
1689
1690struct SourceCacheSession {
1691    cache: Arc<std::sync::RwLock<SourceCache>>,
1692}
1693
1694impl Drop for SourceCacheSession {
1695    fn drop(&mut self) {
1696        if let Ok(mut g) = self.cache.write() {
1697            *g = SourceCache::default();
1698        }
1699    }
1700}
1701
1702#[derive(Debug)]
1703#[non_exhaustive]
1704pub struct EvalResult {
1705    pub computed_vertices: usize,
1706    pub cycle_errors: usize,
1707    pub elapsed: std::time::Duration,
1708}
1709
1710#[derive(Clone, Debug, PartialEq, Eq)]
1711#[non_exhaustive]
1712pub struct TableMetadata {
1713    pub name: String,
1714    pub sheet: String,
1715    pub start_row: u32,
1716    pub start_col: u32,
1717    pub end_row: u32,
1718    pub end_col: u32,
1719    pub header_row: bool,
1720    pub headers: Vec<String>,
1721    pub totals_row: bool,
1722}
1723
1724/// Read-only engine counters used by benchmark/instrumentation tooling.
1725///
1726/// These counters are deliberately observational: collecting them must not mutate engine state or
1727/// alter formula evaluation semantics.
1728#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1729#[non_exhaustive]
1730pub struct EngineBaselineStats {
1731    pub graph_vertex_count: usize,
1732    pub graph_formula_vertex_count: usize,
1733    pub graph_edge_count: usize,
1734    pub dirty_vertex_count: usize,
1735    pub evaluation_vertex_count: usize,
1736    pub formula_ast_root_count: usize,
1737    pub formula_ast_node_count: usize,
1738    pub staged_formula_count: usize,
1739    pub formula_plane_active_span_count: usize,
1740    pub formula_plane_producer_result_entries: usize,
1741    pub formula_plane_consumer_read_entries: usize,
1742    pub formula_plane_mixed_topology_cache_builds: u64,
1743    pub formula_plane_mixed_topology_cache_hits: u64,
1744    pub formula_plane_mixed_topology_cache_overflows: u64,
1745    pub formula_plane_dirty_pending_events: usize,
1746    pub formula_plane_dirty_region_events_recorded: u64,
1747    pub formula_plane_dirty_span_region_events_recorded: u64,
1748    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1749    pub formula_plane_dirty_global_invalidations: u64,
1750    pub formula_plane_structural_span_candidates: u64,
1751    /// Number of spans demoted to legacy because a member participated in a
1752    /// statically-cyclic SCC (gotcha G8, refs #112).
1753    pub formula_plane_cycle_member_span_demotions: u64,
1754}
1755
1756#[derive(Debug, Clone, Default)]
1757#[non_exhaustive]
1758pub struct VirtualDepTelemetry {
1759    pub candidate_vertices_total: usize,
1760    pub vdeps_vertices_total: usize,
1761    pub vdeps_edges_total: usize,
1762    pub builder_elapsed_ms_total: u128,
1763    pub schedule_virtual_passes: usize,
1764    pub schedule_static_passes: usize,
1765    pub schedule_cache_hits: usize,
1766    pub schedule_cache_misses: usize,
1767    pub reused_schedule_vertices_total: usize,
1768    pub replan_iterations: usize,
1769    pub changed_vdeps_total: usize,
1770    pub bailout_reason: Option<&'static str>,
1771    pub fallback_mode_activations: u64,
1772}
1773
1774/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
1775/// (spec `formualizer-cycle-semantics-spec.md` §10).
1776///
1777/// Collection is unconditional: SCC tasks are rare relative to ordinary
1778/// vertex evaluation and the counters are a handful of integer adds per
1779/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
1780/// which pays per-schedule costs). Counters reset at the start of every
1781/// evaluation request.
1782#[derive(Debug, Clone, Default, PartialEq)]
1783#[non_exhaustive]
1784pub struct CycleTelemetry {
1785    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
1786    pub static_sccs: usize,
1787    /// SCC tasks whose live subgraph was acyclic — values produced.
1788    pub phantom_sccs: usize,
1789    /// Distinct live cycles witnessed across all SCC tasks.
1790    pub live_cycles_witnessed: usize,
1791    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
1792    pub circ_cells_stamped: usize,
1793    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
1794    /// (pass 1 included).
1795    pub settle_passes_total: usize,
1796    /// Largest pass count any single SCC task needed.
1797    pub max_passes_single_scc: usize,
1798    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
1799    /// with a witnessed live cycle). RFC #113, Stage 3.
1800    pub iterated_sccs: usize,
1801    /// Iterating SCC tasks that stopped because every member passed the
1802    /// spec-§6 convergence test.
1803    pub converged_sccs: usize,
1804    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
1805    /// this is the Excel `max_iterations` cap (NOT an error — last values
1806    /// are kept; includes the no-convergence-test `max_iterations: 1`
1807    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
1808    /// settle cap (|SCC| + 2), which only a bug can hit.
1809    pub capped_sccs: usize,
1810    /// Largest `|Δ|` observed in any member's final-pass convergence
1811    /// comparison across iterating SCC tasks (numeric-class members only).
1812    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
1813    pub max_abs_delta_at_stop: f64,
1814    /// Identical-bit NaN vs NaN member comparisons that were treated as
1815    /// converged (spec §6 NaN rule).
1816    pub nan_converged: usize,
1817    /// Total wall-clock time spent inside Runtime SCC tasks.
1818    pub elapsed_ms: u128,
1819}
1820
1821#[derive(Debug, Clone, Copy)]
1822struct ScheduleBuildMeta {
1823    candidate_vertices: usize,
1824    vdeps_vertices: usize,
1825    vdeps_edges: usize,
1826    builder_elapsed_ms: u128,
1827    used_virtual_schedule: bool,
1828    schedule_cache_hit: bool,
1829    schedule_cache_eligible: bool,
1830}
1831
1832#[derive(Debug, Clone)]
1833struct CachedScheduleEntry {
1834    topology_epoch: u64,
1835    candidate_vertices: Vec<VertexId>,
1836    schedule: crate::engine::scheduler::Schedule,
1837}
1838
1839#[derive(Clone, Debug, PartialEq, Eq)]
1840struct MixedTopologyCacheKey {
1841    engine_topology_epoch: u64,
1842    graph_topology_revision: u64,
1843    authority_indexes_epoch: u64,
1844    /// Explicit revisions for retained island membership and boundary views.
1845    /// They currently share the authoritative graph/index clocks but remain
1846    /// distinct key fields so future independent indexes cannot omit them.
1847    legacy_island_revision: u64,
1848    boundary_index_revision: u64,
1849    function_semantic_epoch: u64,
1850    function_provider_revision: Option<u64>,
1851    max_candidates: usize,
1852    max_edges: usize,
1853    max_memory_bytes: usize,
1854}
1855
1856#[derive(Debug)]
1857struct CachedMixedTopology {
1858    key: MixedTopologyCacheKey,
1859    topology: MixedTopology,
1860    producer_results: FormulaProducerResultIndex,
1861    consumer_reads: FormulaConsumerReadIndex,
1862    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1863    plane_epoch: u64,
1864    island: LegacyIslandPlan,
1865}
1866
1867#[derive(Debug)]
1868struct SkippedMixedTopology {
1869    reason: MixedScheduleFallbackReason,
1870    observed: MixedTopologyCompileStats,
1871    producer_results: FormulaProducerResultIndex,
1872    consumer_reads: FormulaConsumerReadIndex,
1873    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
1874    plane_epoch: u64,
1875    island: LegacyIslandPlan,
1876}
1877
1878#[derive(Debug)]
1879enum FormulaPlaneTopologyCompileResult {
1880    Cached(CachedMixedTopology),
1881    CacheSkipped(SkippedMixedTopology),
1882}
1883
1884#[derive(Clone, Copy, Debug)]
1885struct ExactDemandScratchEstimates {
1886    paged: u64,
1887    runs: u64,
1888    native: u64,
1889    repeated: u64,
1890}
1891
1892fn exact_demand_scratch_estimates(
1893    read_count: usize,
1894    closure_base: u64,
1895) -> ExactDemandScratchEstimates {
1896    let read_count = read_count as u64;
1897    let page_count = read_count.div_ceil(128);
1898    let run_count = read_count.div_ceil(256);
1899    let pointer_bytes = std::mem::size_of::<usize>() as u64;
1900    let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
1901        + std::mem::size_of::<Vec<usize>>()
1902        + 12 * std::mem::size_of::<usize>()) as u64;
1903    let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
1904        page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
1905    );
1906    let runs_memory = read_count
1907        .saturating_mul(pointer_bytes)
1908        .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
1909    ExactDemandScratchEstimates {
1910        paged: closure_base.saturating_add(paged_memory),
1911        runs: closure_base.saturating_add(runs_memory),
1912        native: closure_base
1913            .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
1914        repeated: closure_base.saturating_add(512),
1915    }
1916}
1917
1918fn native_exact_demand_allowed(
1919    policy: Option<crate::engine::DiskScratchPolicy>,
1920    target_supports_native: bool,
1921) -> bool {
1922    target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
1923}
1924
1925fn select_exact_demand_strategy(
1926    estimates: ExactDemandScratchEstimates,
1927    native_allowed: bool,
1928    mut can_reserve: impl FnMut(u64) -> bool,
1929) -> (FormulaPlaneTopologyStrategy, u64) {
1930    if can_reserve(estimates.paged) {
1931        (
1932            FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1933            estimates.paged,
1934        )
1935    } else if can_reserve(estimates.runs) {
1936        (
1937            FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1938            estimates.runs,
1939        )
1940    } else if native_allowed && can_reserve(estimates.native) {
1941        (
1942            FormulaPlaneTopologyStrategy::ExactNativeScratch,
1943            estimates.native,
1944        )
1945    } else {
1946        (
1947            FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1948            estimates.repeated,
1949        )
1950    }
1951}
1952
1953#[cfg(test)]
1954mod exact_demand_strategy_tests {
1955    use super::*;
1956
1957    #[test]
1958    fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
1959        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
1960        assert!(estimates.paged > estimates.runs);
1961        assert!(estimates.runs > estimates.native);
1962        assert!(estimates.native > estimates.repeated);
1963        for (limit, native_allowed, expected) in [
1964            (
1965                estimates.paged,
1966                true,
1967                FormulaPlaneTopologyStrategy::ExactPagedIndexed,
1968            ),
1969            (
1970                estimates.runs,
1971                true,
1972                FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
1973            ),
1974            (
1975                estimates.native,
1976                true,
1977                FormulaPlaneTopologyStrategy::ExactNativeScratch,
1978            ),
1979            (
1980                estimates.repeated,
1981                true,
1982                FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
1983            ),
1984        ] {
1985            assert_eq!(
1986                select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
1987                expected
1988            );
1989        }
1990    }
1991
1992    #[cfg(not(target_arch = "wasm32"))]
1993    #[test]
1994    fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
1995        let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
1996        let path = scratch.path.clone();
1997        let auxiliary_path = scratch.auxiliary_path.clone();
1998        assert!(path.exists());
1999        assert!(auxiliary_path.exists());
2000        drop(scratch);
2001        assert!(!path.exists());
2002        assert!(!auxiliary_path.exists());
2003    }
2004
2005    #[test]
2006    fn native_policy_and_wasm_target_support_gate_native_rung() {
2007        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2008        assert!(!native_exact_demand_allowed(
2009            Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2010            true,
2011        ));
2012        assert!(!native_exact_demand_allowed(
2013            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2014            false,
2015        ));
2016        let native_allowed = native_exact_demand_allowed(
2017            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2018            !cfg!(target_arch = "wasm32"),
2019        );
2020        let expected = if cfg!(target_arch = "wasm32") {
2021            FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2022        } else {
2023            FormulaPlaneTopologyStrategy::ExactNativeScratch
2024        };
2025        assert_eq!(
2026            select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2027                bytes <= estimates.native
2028            })
2029            .0,
2030            expected
2031        );
2032    }
2033}
2034
2035#[cfg(not(target_arch = "wasm32"))]
2036struct NativeTopologyScratch {
2037    path: std::path::PathBuf,
2038    file: Option<std::fs::File>,
2039    auxiliary_path: std::path::PathBuf,
2040    auxiliary_file: Option<std::fs::File>,
2041}
2042
2043#[cfg(not(target_arch = "wasm32"))]
2044impl NativeTopologyScratch {
2045    fn create(request_id: u64) -> std::io::Result<Self> {
2046        use std::fs::OpenOptions;
2047        for attempt in 0..32_u64 {
2048            let path = std::env::temp_dir().join(format!(
2049                "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2050                std::process::id()
2051            ));
2052            match OpenOptions::new()
2053                .read(true)
2054                .write(true)
2055                .create_new(true)
2056                .open(&path)
2057            {
2058                Ok(file) => {
2059                    let auxiliary_path = path.with_extension("aux.tmp");
2060                    match OpenOptions::new()
2061                        .read(true)
2062                        .write(true)
2063                        .create_new(true)
2064                        .open(&auxiliary_path)
2065                    {
2066                        Ok(auxiliary_file) => {
2067                            return Ok(Self {
2068                                path,
2069                                file: Some(file),
2070                                auxiliary_path,
2071                                auxiliary_file: Some(auxiliary_file),
2072                            });
2073                        }
2074                        Err(error) => {
2075                            drop(file);
2076                            let _ = std::fs::remove_file(&path);
2077                            if error.kind() == std::io::ErrorKind::AlreadyExists {
2078                                continue;
2079                            }
2080                            return Err(error);
2081                        }
2082                    }
2083                }
2084                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2085                Err(error) => return Err(error),
2086            }
2087        }
2088        Err(std::io::Error::new(
2089            std::io::ErrorKind::AlreadyExists,
2090            "could not allocate unique topology scratch file",
2091        ))
2092    }
2093}
2094
2095#[cfg(not(target_arch = "wasm32"))]
2096impl Drop for NativeTopologyScratch {
2097    fn drop(&mut self) {
2098        drop(self.file.take());
2099        drop(self.auxiliary_file.take());
2100        let _ = std::fs::remove_file(&self.path);
2101        let _ = std::fs::remove_file(&self.auxiliary_path);
2102    }
2103}
2104
2105fn estimated_span_bindings_bytes(
2106    bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2107) -> Option<usize> {
2108    const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2109    bindings.len().checked_mul(
2110        std::mem::size_of::<FormulaSpanId>()
2111            .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2112            .checked_add(TREE_ENTRY_OVERHEAD)?,
2113    )
2114}
2115
2116type ScheduleBuildOutput = (
2117    crate::engine::scheduler::Schedule,
2118    FxHashMap<VertexId, Vec<VertexId>>,
2119    ScheduleBuildMeta,
2120);
2121
2122/// Opaque, revision-bound recalculation recipe.
2123#[derive(Debug)]
2124pub struct RecalcPlan {
2125    key: RecalcPlanKey,
2126    kind: RecalcPlanKind,
2127}
2128
2129#[derive(Debug)]
2130struct RecalcPlanKey {
2131    engine_token: Arc<()>,
2132    revisions: PlanningRevisionSnapshot,
2133}
2134
2135#[derive(Clone, Debug, PartialEq, Eq)]
2136struct PlanningRevisionSnapshot {
2137    engine_topology_epoch: u64,
2138    graph_topology_revision: u64,
2139    authority: u64,
2140    authority_indexes: u64,
2141    authority_indexed_plane: u64,
2142    staged: u64,
2143    symbols: u64,
2144    semantic: u64,
2145    provider: Option<u64>,
2146    formula_plane_mode: FormulaPlaneMode,
2147    deterministic_mode: crate::engine::DeterministicMode,
2148    budgets: crate::engine::EvaluationBudgets,
2149    span_refs: Vec<FormulaSpanRef>,
2150}
2151
2152#[derive(Debug)]
2153enum RecalcPlanKind {
2154    CompatibilityFull {
2155        schedule: crate::engine::Schedule,
2156        has_dynamic_refs: bool,
2157    },
2158    Target {
2159        targets: Vec<crate::engine::EvaluationTarget>,
2160        scope: crate::engine::PrepareScope,
2161        topology: RecalcTopology,
2162        dynamic_policy: DynamicPlanPolicy,
2163    },
2164}
2165
2166#[derive(Debug)]
2167enum RecalcTopology {
2168    RunLocalRecipe,
2169    Workbook,
2170}
2171
2172#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2173enum DynamicPlanPolicy {
2174    BoundedTargetReplan,
2175}
2176
2177impl RecalcPlan {
2178    /// Returns the retained compatibility schedule depth. Target plans retain a
2179    /// run-local recipe rather than a schedule, so their layer count is zero.
2180    pub fn layer_count(&self) -> usize {
2181        match &self.kind {
2182            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2183            RecalcPlanKind::Target { .. } => 0,
2184        }
2185    }
2186
2187    pub fn has_dynamic_refs(&self) -> bool {
2188        match &self.kind {
2189            RecalcPlanKind::CompatibilityFull {
2190                has_dynamic_refs, ..
2191            } => *has_dynamic_refs,
2192            RecalcPlanKind::Target { .. } => false,
2193        }
2194    }
2195
2196    #[cfg(test)]
2197    pub(crate) fn force_stale_reasons_for_test(
2198        &mut self,
2199        reasons: &[formualizer_common::PlanStaleReason],
2200    ) {
2201        use formualizer_common::PlanStaleReason;
2202        for reason in reasons {
2203            match reason {
2204                PlanStaleReason::Engine => {
2205                    self.key.engine_token = Arc::new(());
2206                }
2207                PlanStaleReason::Provider => {
2208                    self.key.revisions.provider = Some(
2209                        self.key
2210                            .revisions
2211                            .provider
2212                            .unwrap_or_default()
2213                            .wrapping_add(1),
2214                    );
2215                }
2216                PlanStaleReason::Semantic => {
2217                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2218                }
2219                PlanStaleReason::Budget => {
2220                    let current = self.key.revisions.budgets.work.max_work_units;
2221                    self.key.revisions.budgets.work.max_work_units =
2222                        Some(current.unwrap_or_default().wrapping_add(1));
2223                }
2224                PlanStaleReason::Staged => {
2225                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2226                }
2227                PlanStaleReason::Symbols => {
2228                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2229                }
2230                PlanStaleReason::Authority => {
2231                    self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2232                }
2233                PlanStaleReason::SpanGeneration => {
2234                    self.key.revisions.span_refs.push(FormulaSpanRef {
2235                        id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2236                        generation: u32::MAX,
2237                        version: u32::MAX,
2238                    });
2239                }
2240                PlanStaleReason::Graph => {
2241                    self.key.revisions.graph_topology_revision =
2242                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2243                }
2244                _ => {}
2245            }
2246        }
2247    }
2248}
2249
2250#[cfg(test)]
2251pub(crate) mod criteria_mask_test_hooks {
2252    use std::cell::Cell;
2253
2254    thread_local! {
2255        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2256        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2257    }
2258
2259    pub fn reset_text_segment_counters() {
2260        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2261        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2262    }
2263
2264    pub fn text_segment_counters() -> (usize, usize) {
2265        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2266        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2267        (a, b)
2268    }
2269
2270    pub(crate) fn inc_total() {
2271        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2272    }
2273    pub(crate) fn inc_all_null() {
2274        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2275    }
2276}
2277
2278#[cfg(test)]
2279pub(crate) mod visibility_mask_test_hooks {
2280    use std::cell::Cell;
2281
2282    thread_local! {
2283        static HITS: Cell<usize> = const { Cell::new(0) };
2284        static MISSES: Cell<usize> = const { Cell::new(0) };
2285        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2286    }
2287
2288    pub fn reset() {
2289        HITS.with(|c| c.set(0));
2290        MISSES.with(|c| c.set(0));
2291        EVICTIONS.with(|c| c.set(0));
2292    }
2293
2294    pub fn counters() -> (usize, usize, usize) {
2295        let hits = HITS.with(|c| c.get());
2296        let misses = MISSES.with(|c| c.get());
2297        let evictions = EVICTIONS.with(|c| c.get());
2298        (hits, misses, evictions)
2299    }
2300
2301    pub(crate) fn inc_hit() {
2302        HITS.with(|c| c.set(c.get() + 1));
2303    }
2304
2305    pub(crate) fn inc_miss() {
2306        MISSES.with(|c| c.set(c.get() + 1));
2307    }
2308
2309    pub(crate) fn inc_eviction() {
2310        EVICTIONS.with(|c| c.set(c.get() + 1));
2311    }
2312}
2313
2314fn compute_criteria_mask(
2315    view: &RangeView<'_>,
2316    col_in_view: usize,
2317    pred: &crate::args::CriteriaPredicate,
2318) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2319    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2320    use arrow::compute::kernels::comparison::{ilike, nilike};
2321    use arrow_array::{
2322        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2323    };
2324
2325    // Helper: apply a numeric predicate to a single Float64Array chunk
2326    fn apply_numeric_pred(
2327        chunk: &Float64Array,
2328        pred: &crate::args::CriteriaPredicate,
2329    ) -> Option<BooleanArray> {
2330        match pred {
2331            crate::args::CriteriaPredicate::Gt(n) => {
2332                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2333            }
2334            crate::args::CriteriaPredicate::Ge(n) => {
2335                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2336            }
2337            crate::args::CriteriaPredicate::Lt(n) => {
2338                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2339            }
2340            crate::args::CriteriaPredicate::Le(n) => {
2341                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2342            }
2343            crate::args::CriteriaPredicate::Eq(v) => match v {
2344                formualizer_common::LiteralValue::Number(x) => {
2345                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2346                }
2347                formualizer_common::LiteralValue::Int(i) => {
2348                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2349                }
2350                _ => None,
2351            },
2352            crate::args::CriteriaPredicate::Ne(v) => match v {
2353                formualizer_common::LiteralValue::Number(x) => {
2354                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2355                }
2356                formualizer_common::LiteralValue::Int(i) => {
2357                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2358                }
2359                _ => None,
2360            },
2361            _ => None,
2362        }
2363    }
2364
2365    // Check if this is a numeric predicate that can be applied per-chunk
2366    let is_numeric_pred = matches!(
2367        pred,
2368        crate::args::CriteriaPredicate::Gt(_)
2369            | crate::args::CriteriaPredicate::Ge(_)
2370            | crate::args::CriteriaPredicate::Lt(_)
2371            | crate::args::CriteriaPredicate::Le(_)
2372            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2373            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2374            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2375            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2376    );
2377
2378    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2379    // This avoids materializing the full numeric column (64-bit per element) and instead
2380    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2381    if is_numeric_pred {
2382        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2383        for res in view.numbers_slices() {
2384            let (_rs, _rl, cols_seg) = res.ok()?;
2385            if col_in_view < cols_seg.len() {
2386                let chunk = cols_seg[col_in_view].as_ref();
2387                let mask = apply_numeric_pred(chunk, pred)?;
2388                bool_parts.push(mask);
2389            }
2390        }
2391
2392        if bool_parts.is_empty() {
2393            return None;
2394        } else if bool_parts.len() == 1 {
2395            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2396        } else {
2397            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2398            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2399                .iter()
2400                .map(|a| a as &dyn arrow_array::Array)
2401                .collect();
2402            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2403            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2404            return Some(std::sync::Arc::new(ba));
2405        }
2406    }
2407
2408    // TEXT PATH: build masks per row-chunk using lowered text slices.
2409    // This avoids concatenating full-string columns just to compute a boolean mask.
2410    let (text_kind, text_pat, empty_special) = match pred {
2411        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2412            (0u8, t.to_lowercase(), t.is_empty())
2413        }
2414        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2415            (1u8, t.to_lowercase(), false)
2416        }
2417        crate::args::CriteriaPredicate::TextLike {
2418            pattern,
2419            case_insensitive,
2420        } => {
2421            let p = if *case_insensitive {
2422                pattern.to_lowercase()
2423            } else {
2424                pattern.clone()
2425            };
2426            (2u8, p.replace('*', "%").replace('?', "_"), false)
2427        }
2428        _ => return None,
2429    };
2430
2431    let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2432    let pat = StringArray::new_scalar(text_pat);
2433    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2434
2435    for res in view.iter_row_chunks() {
2436        let cs = res.ok()?;
2437        if cs.row_len == 0 {
2438            continue;
2439        }
2440        #[cfg(test)]
2441        criteria_mask_test_hooks::inc_total();
2442
2443        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2444        if col_in_view >= slices.len() {
2445            return None;
2446        }
2447
2448        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2449        let seg = match seg_opt {
2450            Some(s) => s,
2451            None => {
2452                #[cfg(test)]
2453                criteria_mask_test_hooks::inc_all_null();
2454                if (text_kind == 0 && empty_special) || ne_matches_blank {
2455                    // Eq("") treats nulls (Empty) as equal.
2456                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2457                    bb.append_n(cs.row_len, true);
2458                    bool_parts.push(bb.finish());
2459                } else {
2460                    // For non-empty patterns, ilike/nilike return null on null inputs.
2461                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2462                }
2463                continue;
2464            }
2465        };
2466
2467        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2468        let mut m = match text_kind {
2469            0 => ilike(seg_sa, &pat).ok()?,
2470            1 => nilike(seg_sa, &pat).ok()?,
2471            2 => ilike(seg_sa, &pat).ok()?,
2472            _ => return None,
2473        };
2474
2475        // Only fold blank/Empty (null) cells into the mask when the segment
2476        // actually contains any. The null-fill loop + or_kleene are pure
2477        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2478        // over a column with no blanks stays fully vectorized on the ilike/
2479        // nilike result.
2480        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2481            // Treat nulls as equal to empty string
2482            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2483            for i in 0..seg_sa.len() {
2484                bb.append_value(seg_sa.is_null(i));
2485            }
2486            let nulls = bb.finish();
2487            m = boolean::or_kleene(&m, &nulls).ok()?;
2488        }
2489
2490        bool_parts.push(m);
2491    }
2492
2493    if bool_parts.is_empty() {
2494        None
2495    } else if bool_parts.len() == 1 {
2496        Some(std::sync::Arc::new(bool_parts.remove(0)))
2497    } else {
2498        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2499            .iter()
2500            .map(|a| a as &dyn arrow_array::Array)
2501            .collect();
2502        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2503        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2504        Some(std::sync::Arc::new(ba))
2505    }
2506}
2507
2508#[derive(Debug, Clone)]
2509pub struct LayerInfo {
2510    pub vertex_count: usize,
2511    pub parallel_eligible: bool,
2512    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2513}
2514
2515#[derive(Debug, Clone)]
2516pub struct EvalPlan {
2517    pub total_vertices_to_evaluate: usize,
2518    pub layers: Vec<LayerInfo>,
2519    pub cycles_detected: usize,
2520    pub dirty_count: usize,
2521    pub volatile_count: usize,
2522    pub parallel_enabled: bool,
2523    pub estimated_parallel_layers: usize,
2524    pub target_cells: Vec<String>,
2525}
2526
2527impl<R> Engine<R>
2528where
2529    R: EvaluationContext,
2530{
2531    /// # Panics
2532    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2533    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2534    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2535    /// rejects these at build; this re-validates configs assembled via
2536    /// struct literals.
2537    pub fn new(resolver: R, config: EvalConfig) -> Self {
2538        if let Err(msg) = config.cycle.validate() {
2539            panic!("invalid CycleConfig: {msg}");
2540        }
2541        crate::builtins::load_builtins();
2542        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2543            &config.evaluation_budgets,
2544            config.max_vertices,
2545            config.max_memory_mb,
2546            config.max_eval_time,
2547        );
2548
2549        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2550            #[cfg(feature = "system-clock")]
2551            {
2552                Arc::new(crate::timezone::SystemClock::new(
2553                    crate::timezone::TimeZoneSpec::default(),
2554                ))
2555            }
2556            #[cfg(not(feature = "system-clock"))]
2557            {
2558                Arc::new(crate::timezone::FixedClock::new(
2559                    chrono::DateTime::UNIX_EPOCH,
2560                    crate::timezone::TimeZoneSpec::Utc,
2561                ))
2562            }
2563        });
2564
2565        // Initialize thread pool based on config
2566        let thread_pool = if config.enable_parallel {
2567            let mut builder = ThreadPoolBuilder::new();
2568            if let Some(max_threads) = config.max_threads {
2569                builder = builder.num_threads(max_threads);
2570            }
2571
2572            match builder.build() {
2573                Ok(pool) => Some(Arc::new(pool)),
2574                Err(_) => {
2575                    // Fall back to sequential evaluation if thread pool creation fails
2576                    None
2577                }
2578            }
2579        } else {
2580            None
2581        };
2582
2583        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2584        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2585        let function_provider_revision_seen = resolver.planning_semantic_revision();
2586        let mut engine = Self {
2587            graph: DependencyGraph::new_with_config(config.clone()),
2588            resolver,
2589            config,
2590            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2591            clock: crate::timezone::SnapshotClock::new(clock),
2592            thread_pool,
2593            recalc_epoch: 0,
2594            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2595            topology_epoch: 0,
2596            legacy_island_structural_summaries_trusted: true,
2597            cached_static_schedule: None,
2598            cached_mixed_topology: None,
2599            mixed_topology_cache_builds: 0,
2600            mixed_topology_cache_hits: 0,
2601            mixed_topology_cache_overflows: 0,
2602            mixed_topology_cache_skip_streak: 0,
2603            spill_mgr: ShimSpillManager::default(),
2604            arrow_sheets: SheetStore::default(),
2605            has_edited: false,
2606            overlay_compactions: 0,
2607            computed_overlay_bytes_estimate: 0,
2608            computed_overlay_mirroring_disabled: false,
2609            force_materialize_range_views: false,
2610            row_bounds_cache: std::sync::RwLock::new(None),
2611            used_axis_bounds_cache: std::sync::RwLock::new(None),
2612            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2613            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2614            source_formula_token: Arc::new(()),
2615            recalc_plan_token: Arc::new(()),
2616            staged_formulas: std::collections::HashMap::new(),
2617            staged_formula_index: StagedFormulaIndex::default(),
2618            row_visibility: FxHashMap::default(),
2619            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2620            formula_parse_diagnostics: Vec::new(),
2621            last_formula_ingest_report: None,
2622            formula_ingest_report_total: FormulaIngestReport::default(),
2623            formula_plane_cycle_member_span_demotions: 0,
2624            formula_plane_capacity_bailouts: 0,
2625            formula_plane_structural_span_candidates: 0,
2626            active_cancel_flag: None,
2627            active_evaluation_deadline: None,
2628            action_depth: 0,
2629            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2630            virtual_dep_fallback_activations: 0,
2631            last_cycle_telemetry: CycleTelemetry::default(),
2632            next_evaluation_resource_request_id: 1,
2633            evaluation_resource_request_depth: 0,
2634            active_evaluation_resource_request: None,
2635            last_evaluation_resource_request: None,
2636            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2637            evaluation_resource_request_started_at: None,
2638            evaluation_resource_budgets: resolved_resources.budgets,
2639            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2640            active_resource_ledger: None,
2641            pending_iterative_redirty: Vec::new(),
2642            iterative_state_values: FxHashMap::default(),
2643            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2644            function_provider_revision_seen,
2645            #[cfg(test)]
2646            last_formula_plane_span_eval_report: None,
2647            #[cfg(test)]
2648            evaluation_request_begin_count_for_test: 0,
2649            #[cfg(any(test, feature = "test-support"))]
2650            before_prepared_span_commit_hook: None,
2651            #[cfg(test)]
2652            before_target_preparation_commit_hook: None,
2653            #[cfg(test)]
2654            before_target_planning_snapshot_hook: None,
2655            #[cfg(test)]
2656            inject_target_semantic_stale_once_for_test: false,
2657            #[cfg(test)]
2658            force_virtual_dep_changes_remaining_for_test: 0,
2659            #[cfg(test)]
2660            fail_evaluation_commit_preflight_once_for_test: false,
2661            #[cfg(test)]
2662            force_source_family_fallback: false,
2663            #[cfg(test)]
2664            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2665            #[cfg(test)]
2666            fragmented_commit_fault_for_test: None,
2667            #[cfg(test)]
2668            formula_span_demotion_fault_for_test: None,
2669            #[cfg(test)]
2670            target_preparation_fault_for_test: None,
2671            #[cfg(test)]
2672            force_non_cycle_schedule_fallback_for_test: false,
2673            #[cfg(test)]
2674            mixed_topology_index_builds_for_test: 0,
2675            #[cfg(test)]
2676            before_legacy_fallback_final_provider_sample_hook: None,
2677            #[cfg(test)]
2678            after_eager_proposal_commit_hook: None,
2679        };
2680        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
2681        engine.config.arrow_storage_enabled = true;
2682        engine.config.delta_overlay_enabled = true;
2683        engine.config.write_formula_overlay_enabled = true;
2684        let default_sheet = engine.graph.default_sheet_name().to_string();
2685        engine.ensure_arrow_sheet(&default_sheet);
2686        engine
2687    }
2688
2689    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
2690    ///
2691    /// # Panics
2692    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
2693    pub fn with_thread_pool(
2694        resolver: R,
2695        config: EvalConfig,
2696        thread_pool: Arc<rayon::ThreadPool>,
2697    ) -> Self {
2698        if let Err(msg) = config.cycle.validate() {
2699            panic!("invalid CycleConfig: {msg}");
2700        }
2701        crate::builtins::load_builtins();
2702        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2703            &config.evaluation_budgets,
2704            config.max_vertices,
2705            config.max_memory_mb,
2706            config.max_eval_time,
2707        );
2708        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2709            #[cfg(feature = "system-clock")]
2710            {
2711                Arc::new(crate::timezone::SystemClock::new(
2712                    crate::timezone::TimeZoneSpec::default(),
2713                ))
2714            }
2715            #[cfg(not(feature = "system-clock"))]
2716            {
2717                Arc::new(crate::timezone::FixedClock::new(
2718                    chrono::DateTime::UNIX_EPOCH,
2719                    crate::timezone::TimeZoneSpec::Utc,
2720                ))
2721            }
2722        });
2723        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2724        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2725        let function_provider_revision_seen = resolver.planning_semantic_revision();
2726        let mut engine = Self {
2727            graph: DependencyGraph::new_with_config(config.clone()),
2728            resolver,
2729            config,
2730            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2731            clock: crate::timezone::SnapshotClock::new(clock),
2732            thread_pool: Some(thread_pool),
2733            recalc_epoch: 0,
2734            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2735            topology_epoch: 0,
2736            legacy_island_structural_summaries_trusted: true,
2737            cached_static_schedule: None,
2738            cached_mixed_topology: None,
2739            mixed_topology_cache_builds: 0,
2740            mixed_topology_cache_hits: 0,
2741            mixed_topology_cache_overflows: 0,
2742            mixed_topology_cache_skip_streak: 0,
2743            spill_mgr: ShimSpillManager::default(),
2744            arrow_sheets: SheetStore::default(),
2745            has_edited: false,
2746            overlay_compactions: 0,
2747            computed_overlay_bytes_estimate: 0,
2748            computed_overlay_mirroring_disabled: false,
2749            force_materialize_range_views: false,
2750            row_bounds_cache: std::sync::RwLock::new(None),
2751            used_axis_bounds_cache: std::sync::RwLock::new(None),
2752            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2753            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2754            source_formula_token: Arc::new(()),
2755            recalc_plan_token: Arc::new(()),
2756            staged_formulas: std::collections::HashMap::new(),
2757            staged_formula_index: StagedFormulaIndex::default(),
2758            row_visibility: FxHashMap::default(),
2759            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2760            formula_parse_diagnostics: Vec::new(),
2761            last_formula_ingest_report: None,
2762            formula_ingest_report_total: FormulaIngestReport::default(),
2763            formula_plane_cycle_member_span_demotions: 0,
2764            formula_plane_capacity_bailouts: 0,
2765            formula_plane_structural_span_candidates: 0,
2766            active_cancel_flag: None,
2767            active_evaluation_deadline: None,
2768            action_depth: 0,
2769            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2770            virtual_dep_fallback_activations: 0,
2771            last_cycle_telemetry: CycleTelemetry::default(),
2772            next_evaluation_resource_request_id: 1,
2773            evaluation_resource_request_depth: 0,
2774            active_evaluation_resource_request: None,
2775            last_evaluation_resource_request: None,
2776            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2777            evaluation_resource_request_started_at: None,
2778            evaluation_resource_budgets: resolved_resources.budgets,
2779            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2780            active_resource_ledger: None,
2781            pending_iterative_redirty: Vec::new(),
2782            iterative_state_values: FxHashMap::default(),
2783            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2784            function_provider_revision_seen,
2785            #[cfg(test)]
2786            last_formula_plane_span_eval_report: None,
2787            #[cfg(test)]
2788            evaluation_request_begin_count_for_test: 0,
2789            #[cfg(any(test, feature = "test-support"))]
2790            before_prepared_span_commit_hook: None,
2791            #[cfg(test)]
2792            before_target_preparation_commit_hook: None,
2793            #[cfg(test)]
2794            before_target_planning_snapshot_hook: None,
2795            #[cfg(test)]
2796            inject_target_semantic_stale_once_for_test: false,
2797            #[cfg(test)]
2798            force_virtual_dep_changes_remaining_for_test: 0,
2799            #[cfg(test)]
2800            fail_evaluation_commit_preflight_once_for_test: false,
2801            #[cfg(test)]
2802            force_source_family_fallback: false,
2803            #[cfg(test)]
2804            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2805            #[cfg(test)]
2806            fragmented_commit_fault_for_test: None,
2807            #[cfg(test)]
2808            formula_span_demotion_fault_for_test: None,
2809            #[cfg(test)]
2810            target_preparation_fault_for_test: None,
2811            #[cfg(test)]
2812            force_non_cycle_schedule_fallback_for_test: false,
2813            #[cfg(test)]
2814            mixed_topology_index_builds_for_test: 0,
2815            #[cfg(test)]
2816            before_legacy_fallback_final_provider_sample_hook: None,
2817            #[cfg(test)]
2818            after_eager_proposal_commit_hook: None,
2819        };
2820        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
2821        engine.config.arrow_storage_enabled = true;
2822        engine.config.delta_overlay_enabled = true;
2823        engine.config.write_formula_overlay_enabled = true;
2824        let default_sheet = engine.graph.default_sheet_name().to_string();
2825        engine.ensure_arrow_sheet(&default_sheet);
2826        engine
2827    }
2828
2829    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
2830        &self.workbook_load_limits
2831    }
2832
2833    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
2834        self.workbook_load_limits = limits;
2835    }
2836
2837    fn clear_source_cache(&self) {
2838        if let Ok(mut g) = self.source_cache.write() {
2839            *g = SourceCache::default();
2840        }
2841    }
2842
2843    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
2844        &self.last_virtual_dep_telemetry
2845    }
2846
2847    /// Telemetry from Runtime SCC evaluation during the most recent
2848    /// evaluation request (always default-zero under `CycleDetection::Static`
2849    /// or when `enable_virtual_dep_telemetry` is off).
2850    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
2851        &self.last_cycle_telemetry
2852    }
2853
2854    /// Resource observations for the most recently completed public evaluation request.
2855    pub fn last_evaluation_resource_request_stats(
2856        &self,
2857    ) -> Option<&EvaluationResourceRequestStats> {
2858        self.last_evaluation_resource_request.as_ref()
2859    }
2860
2861    /// Cumulative resource observations since engine creation or the last telemetry reset.
2862    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
2863        self.evaluation_resource_baseline
2864    }
2865
2866    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
2867        &self.evaluation_resource_budgets
2868    }
2869
2870    /// At most one diagnostic is emitted for deprecated resource fields.
2871    pub fn evaluation_resource_config_diagnostic(
2872        &self,
2873    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
2874        self.evaluation_resource_config_diagnostic.as_ref()
2875    }
2876
2877    /// Reset accumulated and last-request observations without reusing request IDs.
2878    pub fn reset_evaluation_resource_telemetry(&mut self) {
2879        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
2880        self.last_evaluation_resource_request = None;
2881    }
2882
2883    fn duration_ns(duration: std::time::Duration) -> u64 {
2884        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
2885    }
2886
2887    fn observe_evaluation_resource_request<T>(
2888        &mut self,
2889        kind: EvaluationRequestKind,
2890        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
2891    ) -> Result<T, ExcelError> {
2892        let outermost = self.evaluation_resource_request_depth == 0;
2893        if outermost {
2894            let request_id = self.next_evaluation_resource_request_id;
2895            self.next_evaluation_resource_request_id = request_id
2896                .checked_add(1)
2897                .expect("evaluation resource request ID exhausted");
2898            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
2899                request_id,
2900                kind,
2901                self.config.formula_plane_mode,
2902                self.staged_formula_count(),
2903            ));
2904            self.evaluation_resource_baseline.record_started(request_id);
2905            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
2906            self.active_resource_ledger = Some(ResourceLedger::new(
2907                Some(request_id),
2908                self.evaluation_resource_budgets.clone(),
2909            ));
2910        }
2911        self.evaluation_resource_request_depth =
2912            self.evaluation_resource_request_depth.saturating_add(1);
2913        let result = if outermost {
2914            self.resource_checkpoint(0).and_then(|()| evaluate(self))
2915        } else {
2916            evaluate(self)
2917        };
2918        self.evaluation_resource_request_depth =
2919            self.evaluation_resource_request_depth.saturating_sub(1);
2920
2921        if outermost {
2922            let total_ns = self
2923                .evaluation_resource_request_started_at
2924                .take()
2925                .map(|start| Self::duration_ns(start.elapsed()))
2926                .unwrap_or(0);
2927            let mut stats = self
2928                .active_evaluation_resource_request
2929                .take()
2930                .expect("outer evaluation resource request has active stats");
2931            let mut ledger = self
2932                .active_resource_ledger
2933                .take()
2934                .expect("outer evaluation resource request has active ledger");
2935            ledger.release_all_scratch();
2936            stats.ledger.update(ledger.snapshot());
2937            stats.outcome = match &result {
2938                Ok(_) => EvaluationRequestOutcome::Success,
2939                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
2940                    EvaluationRequestOutcome::Cancelled
2941                }
2942                Err(_) => EvaluationRequestOutcome::Error,
2943            };
2944            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
2945                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
2946                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
2947                } else {
2948                    FormulaDirtyLeaseOutcome::RetainedOnError
2949                };
2950            }
2951            stats.phases.total_ns = total_ns;
2952            let attributed = stats
2953                .phases
2954                .staged_prepare_ns
2955                .saturating_add(stats.phases.topology_ns)
2956                .saturating_add(stats.phases.materialization_ns);
2957            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
2958            self.evaluation_resource_baseline.record_finished(&stats);
2959            self.last_evaluation_resource_request = Some(stats);
2960        }
2961        result
2962    }
2963
2964    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
2965        self.evaluation_resource_budgets = budgets.clone();
2966        self.config.evaluation_budgets = budgets.clone();
2967        self.graph.set_evaluation_budgets(budgets);
2968    }
2969
2970    #[cfg(test)]
2971    pub(crate) fn set_evaluation_budgets_for_test(
2972        &mut self,
2973        budgets: crate::engine::EvaluationBudgets,
2974    ) {
2975        self.set_evaluation_resource_budgets(budgets);
2976    }
2977
2978    fn resource_loop_checkpoint(
2979        ledger: &mut Option<ResourceLedger>,
2980        work_units: u64,
2981    ) -> Result<(), ExcelError> {
2982        ledger
2983            .as_mut()
2984            .map_or(Ok(()), |ledger| {
2985                ledger
2986                    .charge_work(work_units)
2987                    .and_then(|()| ledger.checkpoint_deadline())
2988            })
2989            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
2990    }
2991
2992    fn preflight_evaluation_commit_window(
2993        &mut self,
2994        bounded_writes: usize,
2995    ) -> Result<crate::instant::FzInstant, ExcelError> {
2996        #[cfg(test)]
2997        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
2998            return Err(crate::engine::ResourceLedgerError::Exhausted(
2999                formualizer_common::ResourceExhaustionDetail {
3000                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3001                    limit: 0,
3002                    observed: 1,
3003                    request_id: self
3004                        .active_evaluation_resource_request
3005                        .as_ref()
3006                        .map(|stats| stats.request_id),
3007                },
3008            )
3009            .into_excel_error());
3010        }
3011        let estimate = std::time::Duration::from_nanos(
3012            u64::try_from(bounded_writes)
3013                .unwrap_or(u64::MAX)
3014                .saturating_mul(100),
3015        );
3016        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3017            ledger
3018                .preflight_commit_window(estimate)
3019                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3020        }
3021        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3022            stats.evaluation_commit_preflight_count =
3023                stats.evaluation_commit_preflight_count.saturating_add(1);
3024            stats.evaluation_commit_estimated_ns = stats
3025                .evaluation_commit_estimated_ns
3026                .saturating_add(Self::duration_ns(estimate));
3027        }
3028        Ok(crate::instant::FzInstant::now())
3029    }
3030
3031    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3032        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3033            stats.evaluation_commit_actual_ns = stats
3034                .evaluation_commit_actual_ns
3035                .saturating_add(Self::duration_ns(started.elapsed()));
3036        }
3037    }
3038
3039    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3040        if self
3041            .active_cancel_flag
3042            .as_ref()
3043            .is_some_and(|cancel| cancel.is_cancelled())
3044        {
3045            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3046        }
3047        if self
3048            .active_evaluation_deadline
3049            .is_some_and(|deadline| Instant::now() >= deadline)
3050        {
3051            return Err(crate::engine::ResourceLedgerError::Exhausted(
3052                formualizer_common::ResourceExhaustionDetail {
3053                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3054                    limit: 0,
3055                    observed: 1,
3056                    request_id: self
3057                        .active_evaluation_resource_request
3058                        .as_ref()
3059                        .map(|request| request.request_id),
3060                },
3061            )
3062            .into_excel_error()
3063            .with_message(message));
3064        }
3065        Ok(())
3066    }
3067
3068    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3069        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3070            return Ok(());
3071        };
3072        ledger
3073            .charge_work(work_units)
3074            .and_then(|()| ledger.checkpoint_deadline())
3075            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3076    }
3077
3078    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3079        if work_units == 0 {
3080            return self.resource_checkpoint(0);
3081        }
3082        while work_units > 0 {
3083            let chunk = work_units.min(256);
3084            self.resource_checkpoint(chunk)?;
3085            work_units -= chunk;
3086        }
3087        Ok(())
3088    }
3089
3090    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3091        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3092            // Exact request topology and activation of the scratch cap are C1b. C1a records
3093            // scoped ownership but must not introduce a new skip or terminal path.
3094            ledger.observe_scratch(bytes);
3095        }
3096        Ok(())
3097    }
3098
3099    fn release_request_scratch(&mut self, bytes: u64) {
3100        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3101            let released = ledger.release_scratch(bytes);
3102            debug_assert!(
3103                released.is_ok(),
3104                "request scratch release exceeded the outstanding reservation"
3105            );
3106        }
3107    }
3108
3109    fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3110        self.active_resource_ledger
3111            .as_ref()
3112            .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3113    }
3114
3115    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3116        self.active_resource_ledger
3117            .as_mut()
3118            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3119            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3120    }
3121
3122    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3123        self.active_resource_ledger
3124            .as_mut()
3125            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3126            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3127    }
3128
3129    fn with_request_scratch<T>(
3130        &mut self,
3131        bytes: u64,
3132        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3133    ) -> Result<T, ExcelError> {
3134        self.reserve_request_scratch(bytes)?;
3135        let result = work(self);
3136        self.release_request_scratch(bytes);
3137        result
3138    }
3139
3140    fn observe_staged_preparation(
3141        &mut self,
3142        selected: usize,
3143        retained: usize,
3144        elapsed: std::time::Duration,
3145    ) {
3146        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3147            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3148            stats.staged_retained = retained as u64;
3149            stats.phases.staged_prepare_ns = stats
3150                .phases
3151                .staged_prepare_ns
3152                .saturating_add(Self::duration_ns(elapsed));
3153        }
3154    }
3155
3156    fn graph_admission_enabled(&self) -> bool {
3157        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3158    }
3159
3160    fn preflight_graph_admission(
3161        &mut self,
3162        usage: crate::engine::resource_ledger::GraphAdmission,
3163    ) -> Result<(), ExcelError> {
3164        let request_id = self
3165            .active_evaluation_resource_request
3166            .as_ref()
3167            .map(|stats| stats.request_id);
3168        crate::engine::resource_ledger::preflight_graph_admission(
3169            &self.evaluation_resource_budgets,
3170            usage,
3171            request_id,
3172        )
3173        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3174    }
3175
3176    fn prepared_legacy_admission(
3177        &mut self,
3178        plan: &PreparedLegacyGraphPlan,
3179        materialization_cells: u64,
3180    ) -> Result<(), ExcelError> {
3181        if !self.graph_admission_enabled() {
3182            return Ok(());
3183        }
3184        let stats = self.graph.baseline_stats();
3185        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3186            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3187        })?;
3188        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3189            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3190        })?;
3191        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3192            final_vertices: stats
3193                .graph_vertex_count
3194                .checked_add(plan.new_vertex_count())
3195                .ok_or_else(|| {
3196                    ExcelError::new(ExcelErrorKind::NImpl)
3197                        .with_message("graph vertex count overflow")
3198                })?,
3199            final_edges: stats
3200                .graph_edge_count
3201                .checked_sub(removed_edges)
3202                .and_then(|count| count.checked_add(added_edges))
3203                .ok_or_else(|| {
3204                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3205                })?,
3206            materialization_cells,
3207            added_vertices: plan.new_vertex_count(),
3208            added_edges,
3209        })
3210    }
3211
3212    fn observe_target_admission_failure(
3213        &mut self,
3214        reason: formualizer_common::ResourceExhaustionReason,
3215    ) {
3216        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3217            stats.target_admission_failure = Some(reason);
3218        }
3219    }
3220
3221    fn observe_target_preparation_report(
3222        &mut self,
3223        report: &crate::engine::PreparedTargetGraphReport,
3224    ) {
3225        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3226            let index = match reason {
3227                crate::engine::OpaqueReason::DynamicReference => 0,
3228                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3229                crate::engine::OpaqueReason::UnknownFunction => 2,
3230                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3231                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3232                crate::engine::OpaqueReason::UnresolvedName => 5,
3233                crate::engine::OpaqueReason::UnresolvedTable => 6,
3234                crate::engine::OpaqueReason::FormulaName => 7,
3235                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3236                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3237                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3238            };
3239            1u64 << index
3240        };
3241        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3242            stats.staged_selected = report.selected_staged_cells as u64;
3243            stats.staged_retained = report.retained_staged_cells as u64;
3244            stats.target_requested = report.requested_targets as u64;
3245            stats.target_normalized_regions = report.normalized_regions as u64;
3246            stats.target_scope_level = match &report.widened_scope {
3247                crate::engine::PrepareScope::Exact => 0,
3248                crate::engine::PrepareScope::Sheets(_) => 1,
3249                crate::engine::PrepareScope::Workbook => 2,
3250            };
3251            stats.target_widening_reason_bits = report
3252                .widening_reasons
3253                .iter()
3254                .copied()
3255                .fold(0, |bits, reason| bits | reason_bit(reason));
3256            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3257            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3258            stats.target_commit_estimated_work = report.estimated_commit_work;
3259            stats.target_commit_actual_work = report.actual_commit_work;
3260            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3261            stats.phases.staged_prepare_ns = stats
3262                .phases
3263                .staged_prepare_ns
3264                .saturating_add(Self::duration_ns(report.commit_window));
3265        }
3266    }
3267
3268    fn observe_topology(
3269        &mut self,
3270        outcome: FormulaPlaneTopologyCacheOutcome,
3271        strategy: FormulaPlaneTopologyStrategy,
3272        observed: &MixedTopologyCompileStats,
3273        elapsed: std::time::Duration,
3274    ) {
3275        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3276            if outcome.severity() > stats.topology.cache_outcome.severity() {
3277                stats.topology.cache_outcome = outcome;
3278            }
3279            if strategy.severity() > stats.topology.strategy.severity() {
3280                stats.topology.strategy = strategy;
3281            }
3282            match outcome {
3283                FormulaPlaneTopologyCacheOutcome::Hit => {
3284                    stats.topology.cache_hit_events =
3285                        stats.topology.cache_hit_events.saturating_add(1);
3286                }
3287                FormulaPlaneTopologyCacheOutcome::Built => {
3288                    stats.topology.cache_build_events =
3289                        stats.topology.cache_build_events.saturating_add(1);
3290                }
3291                FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3292                | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3293                    stats.topology.cache_build_events =
3294                        stats.topology.cache_build_events.saturating_add(1);
3295                    stats.topology.cache_skip_events =
3296                        stats.topology.cache_skip_events.saturating_add(1);
3297                }
3298                FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3299            }
3300            if matches!(
3301                outcome,
3302                FormulaPlaneTopologyCacheOutcome::Built
3303                    | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3304                    | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3305            ) {
3306                stats.topology.producers_observed = stats
3307                    .topology
3308                    .producers_observed
3309                    .saturating_add(observed.producers as u64);
3310                stats.topology.candidates_observed = stats
3311                    .topology
3312                    .candidates_observed
3313                    .saturating_add(observed.candidates as u64);
3314                stats.topology.edges_observed = stats
3315                    .topology
3316                    .edges_observed
3317                    .saturating_add(observed.relationships as u64);
3318                stats.topology.candidate_cap_hits = stats
3319                    .topology
3320                    .candidate_cap_hits
3321                    .saturating_add(observed.candidate_overflow_count as u64);
3322                stats.topology.edge_cap_hits = stats
3323                    .topology
3324                    .edge_cap_hits
3325                    .saturating_add(observed.edge_overflow_count as u64);
3326                stats.topology.byte_cap_hits = stats
3327                    .topology
3328                    .byte_cap_hits
3329                    .saturating_add(observed.memory_overflow_count as u64);
3330                stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3331                    Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3332                } else if stats.topology.edge_cap_hits > 0 {
3333                    Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3334                } else if stats.topology.candidate_cap_hits > 0 {
3335                    Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3336                } else {
3337                    None
3338                };
3339                stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3340                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3341                } else if stats.topology.edge_cap_hits > 0 {
3342                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3343                } else if stats.topology.candidate_cap_hits > 0 {
3344                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3345                } else {
3346                    None
3347                };
3348            }
3349            stats.topology.retained_bytes_observed = stats
3350                .topology
3351                .retained_bytes_observed
3352                .max(observed.estimated_memory_bytes as u64);
3353            stats.phases.topology_ns = stats
3354                .phases
3355                .topology_ns
3356                .saturating_add(Self::duration_ns(elapsed));
3357        }
3358    }
3359
3360    fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3361        if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3362            && strategy.severity() > stats.topology.strategy.severity()
3363        {
3364            stats.topology.strategy = strategy;
3365        }
3366    }
3367
3368    fn observe_formula_plane_route(
3369        &mut self,
3370        plan: &LegacyIslandPlan,
3371        phase: FormulaPlaneRoutePhase,
3372        reason: FormulaPlaneRouteTransitionReason,
3373        demotion_generation: usize,
3374        replan_generation: usize,
3375    ) {
3376        if plan.is_empty() {
3377            return;
3378        }
3379        let mut sheet_ids = [0_u16;
3380            crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3381        let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3382        sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3383        let authority = self.graph.formula_authority();
3384        let event = FormulaPlaneRouteEvent {
3385            island_id: plan.island_id,
3386            phase,
3387            route: FormulaPlaneRoute::ContractedLegacyIsland,
3388            sheet_ids,
3389            sheet_count: sheet_count as u8,
3390            authority_epoch: authority.plane.epoch().0,
3391            index_epoch: authority.indexes_epoch(),
3392            transition_reason: reason,
3393            demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3394            replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3395        };
3396        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3397            let index = usize::from(stats.topology.route_event_count);
3398            if index < stats.topology.route_events.len() {
3399                stats.topology.route_events[index] = event;
3400                stats.topology.route_event_count =
3401                    stats.topology.route_event_count.saturating_add(1);
3402            } else {
3403                stats.topology.route_events_dropped =
3404                    stats.topology.route_events_dropped.saturating_add(1);
3405            }
3406            stats.topology.route_event_bytes_observed = stats
3407                .topology
3408                .route_event_bytes_observed
3409                .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3410            stats.topology.island_membership_vertices = plan.membership.len() as u64;
3411            stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3412            stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3413            stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3414        }
3415    }
3416
3417    fn observe_materialization(
3418        &mut self,
3419        placements: usize,
3420        cycle: bool,
3421        elapsed: std::time::Duration,
3422    ) {
3423        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3424            if cycle {
3425                stats.cycle_materialized_cells = stats
3426                    .cycle_materialized_cells
3427                    .saturating_add(placements as u64);
3428            } else {
3429                stats.fallback_materialized_cells = stats
3430                    .fallback_materialized_cells
3431                    .saturating_add(placements as u64);
3432                stats.topology.strategy =
3433                    FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3434            }
3435            stats.phases.materialization_ns = stats
3436                .phases
3437                .materialization_ns
3438                .saturating_add(Self::duration_ns(elapsed));
3439        }
3440    }
3441
3442    fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3443        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3444            stats.dirty_lease = if empty {
3445                FormulaDirtyLeaseOutcome::Empty
3446            } else {
3447                FormulaDirtyLeaseOutcome::Acquired
3448            };
3449        }
3450    }
3451
3452    fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3453        let empty = lease.is_empty();
3454        let _ = self.graph.ack_formula_dirty(lease);
3455        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3456            stats.dirty_lease = if empty {
3457                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3458            } else {
3459                FormulaDirtyLeaseOutcome::Acknowledged
3460            };
3461        }
3462    }
3463
3464    fn checked_ack_formula_dirty_observed(
3465        &mut self,
3466        lease: FormulaDirtyLease,
3467    ) -> Result<(), ExcelError> {
3468        self.resource_checkpoint(0)?;
3469        self.ack_formula_dirty_observed(lease);
3470        Ok(())
3471    }
3472
3473    fn checked_ack_formula_dirty_sublease_observed(
3474        &mut self,
3475        lease: FormulaDirtyLease,
3476        owned_events: &[usize],
3477    ) -> Result<(), ExcelError> {
3478        self.resource_checkpoint(owned_events.len() as u64)?;
3479        let empty = lease.is_empty();
3480        let partial = owned_events.len() < lease.len();
3481        let sublease = lease.sublease(owned_events.iter().copied());
3482        if !self.graph.ack_formula_dirty_sublease(sublease) {
3483            return Err(ExcelError::new(ExcelErrorKind::NImpl)
3484                .with_message("FormulaPlane dirty sublease became stale"));
3485        }
3486        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3487            stats.dirty_lease = if empty {
3488                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3489            } else if partial {
3490                FormulaDirtyLeaseOutcome::AcknowledgedPartial
3491            } else {
3492                FormulaDirtyLeaseOutcome::Acknowledged
3493            };
3494        }
3495        Ok(())
3496    }
3497
3498    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3499    /// take the per-recalc volatile clock sample. Called at the start of
3500    /// every evaluation request that walks schedule units.
3501    fn begin_evaluation_request(&mut self) {
3502        #[cfg(test)]
3503        {
3504            self.evaluation_request_begin_count_for_test = self
3505                .evaluation_request_begin_count_for_test
3506                .saturating_add(1);
3507        }
3508        self.last_cycle_telemetry = CycleTelemetry::default();
3509        // Defensive: consumed at the end of the previous request; a request
3510        // that errored out mid-walk must not leak its members into this one.
3511        self.pending_iterative_redirty.clear();
3512        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3513        // read within this request (including SCC iteration passes) observes
3514        // this sample.
3515        self.clock.refresh();
3516    }
3517
3518    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3519    /// SCCs that iterated this recalc (`CyclePolicy::Iterate`), so circular
3520    /// cells re-evaluate on every recalc exactly like Excel's iterative
3521    /// calculation (spec §4 persistence / §7.6 accumulator / §7.11 volatile
3522    /// redirty). Replaces the bare `graph.redirty_volatiles()` call at every
3523    /// evaluation-flow exit; must run AFTER the flow's `clear_dirty_flags`.
3524    fn redirty_for_next_recalc(&mut self) {
3525        self.graph.redirty_volatiles();
3526        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3527        // Refresh the §4-persistence snapshot: these final values survive
3528        // structural edits that clear the computed overlay (the only value
3529        // home in canonical mode) so the next SCC task can re-seed from them
3530        // (see `iterative_state_values`). Replaced wholesale each recalc —
3531        // when nothing iterates the map empties and stays free.
3532        self.iterative_state_values.clear();
3533        for &vertex in &pending {
3534            if !self.graph.vertex_exists(vertex) {
3535                continue;
3536            }
3537            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3538                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3539                if let Some(value) =
3540                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
3541                    && !matches!(value, LiteralValue::Empty)
3542                {
3543                    self.iterative_state_values.insert(vertex, value);
3544                }
3545            }
3546        }
3547        if !pending.is_empty() {
3548            self.graph.redirty_iterative_members(&pending);
3549        }
3550    }
3551
3552    pub fn virtual_dep_fallback_activations(&self) -> u64 {
3553        self.virtual_dep_fallback_activations
3554    }
3555
3556    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
3557        self.lookup_index_cache.report()
3558    }
3559
3560    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
3561        let start_row = view.start_row();
3562        let end_row = view.end_row();
3563        let start_col = view.start_col();
3564        let end_col = view.end_col();
3565        for row in start_row..=end_row {
3566            let Ok(row_u32) = u32::try_from(row) else {
3567                return true;
3568            };
3569            for col in start_col..=end_col {
3570                let Ok(col_u32) = u32::try_from(col) else {
3571                    return true;
3572                };
3573                let cell_ref = self
3574                    .graph
3575                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
3576                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
3577                    && self.graph.is_volatile(*vertex_id)
3578                {
3579                    return true;
3580                }
3581            }
3582        }
3583        false
3584    }
3585
3586    fn build_lookup_index_impl(
3587        &self,
3588        view: &RangeView<'_>,
3589        axis: LookupAxis,
3590    ) -> Option<Arc<LookupIndex>> {
3591        let (rows, cols) = view.dims();
3592        if rows == 0 || cols == 0 {
3593            self.lookup_index_cache.note_skipped_tiny();
3594            return None;
3595        }
3596        let len = match axis {
3597            LookupAxis::ColumnInView(col) => {
3598                if col >= cols {
3599                    self.lookup_index_cache.note_skipped_tiny();
3600                    return None;
3601                }
3602                rows
3603            }
3604            LookupAxis::RowInView(row) => {
3605                if row >= rows {
3606                    self.lookup_index_cache.note_skipped_tiny();
3607                    return None;
3608                }
3609                cols
3610            }
3611        };
3612        if len < 64 {
3613            self.lookup_index_cache.note_skipped_tiny();
3614            return None;
3615        }
3616
3617        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
3618        let key = LookupIndexKey {
3619            sheet_id,
3620            start_row: u32::try_from(view.start_row()).ok()?,
3621            start_col: u32::try_from(view.start_col()).ok()?,
3622            end_row: u32::try_from(view.end_row()).ok()?,
3623            end_col: u32::try_from(view.end_col()).ok()?,
3624            axis,
3625            snapshot_id: self.data_snapshot_id(),
3626        };
3627        if let Some(index) = self.lookup_index_cache.get(&key) {
3628            return Some(index);
3629        }
3630        if self
3631            .lookup_index_cache
3632            .would_exceed_cap(estimate_bytes(len, 0))
3633        {
3634            self.lookup_index_cache.note_skipped_cap();
3635            return None;
3636        }
3637        if !self.lookup_index_cache.should_build(key) {
3638            return None;
3639        }
3640        if self.lookup_index_cache.is_known_volatile(&key) {
3641            self.lookup_index_cache.note_skipped_volatile();
3642            return None;
3643        }
3644        if self.lookup_view_contains_volatile(view, sheet_id) {
3645            self.lookup_index_cache.note_volatile_key(key);
3646            self.lookup_index_cache.note_skipped_volatile();
3647            return None;
3648        }
3649        match LookupIndex::build(view, axis, self.config.date_system).ok()? {
3650            BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
3651            BuildOutcome::ErrorInLookupAxis => {
3652                self.lookup_index_cache.note_skipped_error();
3653                None
3654            }
3655            BuildOutcome::Degenerate => {
3656                self.lookup_index_cache.note_skipped_tiny();
3657                None
3658            }
3659        }
3660    }
3661
3662    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
3663        if !self.config.enable_virtual_dep_telemetry {
3664            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
3665                fallback_mode_activations: self.virtual_dep_fallback_activations,
3666                ..VirtualDepTelemetry::default()
3667            };
3668        }
3669    }
3670
3671    fn source_cache_session(&self) -> SourceCacheSession {
3672        self.clear_source_cache();
3673        SourceCacheSession {
3674            cache: self.source_cache.clone(),
3675        }
3676    }
3677
3678    fn resolve_source_scalar_cached(
3679        &self,
3680        name: &str,
3681        version: Option<u64>,
3682    ) -> Result<LiteralValue, ExcelError> {
3683        let key = (name.to_string(), version);
3684        if let Ok(mut g) = self.source_cache.write() {
3685            if let Some(v) = g.scalars.get(&key) {
3686                return Ok(v.clone());
3687            }
3688
3689            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
3690                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3691                    ExcelError::new(ExcelErrorKind::Ref)
3692                        .with_message(format!("Unresolved source scalar: {name}"))
3693                } else {
3694                    err
3695                }
3696            })?;
3697            g.scalars.insert(key, v.clone());
3698            Ok(v)
3699        } else {
3700            self.resolver.resolve_source_scalar(name).map_err(|err| {
3701                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3702                    ExcelError::new(ExcelErrorKind::Ref)
3703                        .with_message(format!("Unresolved source scalar: {name}"))
3704                } else {
3705                    err
3706                }
3707            })
3708        }
3709    }
3710
3711    fn resolve_source_table_cached(
3712        &self,
3713        name: &str,
3714        version: Option<u64>,
3715    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
3716        let key = (name.to_string(), version);
3717        if let Ok(mut g) = self.source_cache.write() {
3718            if let Some(t) = g.tables.get(&key) {
3719                return Ok(t.clone());
3720            }
3721
3722            let t = self.resolver.resolve_source_table(name).map_err(|err| {
3723                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3724                    ExcelError::new(ExcelErrorKind::Ref)
3725                        .with_message(format!("Unresolved source table: {name}"))
3726                } else {
3727                    err
3728                }
3729            })?;
3730            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
3731            g.tables.insert(key, t.clone());
3732            Ok(t)
3733        } else {
3734            self.resolver
3735                .resolve_source_table(name)
3736                .map_err(|err| {
3737                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
3738                        ExcelError::new(ExcelErrorKind::Ref)
3739                            .with_message(format!("Unresolved source table: {name}"))
3740                    } else {
3741                        err
3742                    }
3743                })
3744                .map(Arc::from)
3745        }
3746    }
3747
3748    fn source_table_to_range_view(
3749        &self,
3750        table: &dyn crate::traits::Table,
3751        spec: &Option<formualizer_parse::parser::TableSpecifier>,
3752    ) -> Result<RangeView<'static>, ExcelError> {
3753        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
3754
3755        let owned = match spec {
3756            Some(TableSpecifier::Column(c)) => {
3757                let c = c.trim();
3758                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
3759                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3760                        "Complex structured references not yet supported".to_string(),
3761                    ));
3762                }
3763                table.get_column(c)?.materialise().into_owned()
3764            }
3765            Some(TableSpecifier::ColumnRange(start, end)) => {
3766                let cols = table.columns();
3767                let start = start.trim();
3768                let end = end.trim();
3769                let start_key = start.to_lowercase();
3770                let end_key = end.to_lowercase();
3771                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
3772                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
3773                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
3774                    if si > ei {
3775                        std::mem::swap(&mut si, &mut ei);
3776                    }
3777                    let h = table.data_height();
3778                    let w = ei - si + 1;
3779                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
3780                    for (offset, ci) in (si..=ei).enumerate() {
3781                        let cname = &cols[ci];
3782                        let col_range = table.get_column(cname)?;
3783                        let (rh, _) = col_range.dimensions();
3784                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
3785                            row[offset] = col_range.get(r, 0)?;
3786                        }
3787                    }
3788                    rows
3789                } else {
3790                    return Err(ExcelError::new(ExcelErrorKind::Ref)
3791                        .with_message("Column range refers to unknown column(s)".to_string()));
3792                }
3793            }
3794            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
3795            | Some(TableSpecifier::Headers) => table
3796                .headers_row()
3797                .map(|r| r.materialise().into_owned())
3798                .unwrap_or_default(),
3799            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
3800            | Some(TableSpecifier::Totals) => table
3801                .totals_row()
3802                .map(|r| r.materialise().into_owned())
3803                .unwrap_or_default(),
3804            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
3805                table
3806                    .data_body()
3807                    .map(|r| r.materialise().into_owned())
3808                    .unwrap_or_default()
3809            }
3810            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
3811                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
3812                if let Some(h) = table.headers_row() {
3813                    out.extend(h.iter_rows());
3814                }
3815                if let Some(body) = table.data_body() {
3816                    out.extend(body.iter_rows());
3817                }
3818                if let Some(tr) = table.totals_row() {
3819                    out.extend(tr.iter_rows());
3820                }
3821                out
3822            }
3823            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
3824                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
3825                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
3826                ));
3827            }
3828            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
3829                return Err(ExcelError::new(ExcelErrorKind::NImpl)
3830                    .with_message("Complex structured references not yet supported".to_string()));
3831            }
3832            None => {
3833                return Err(ExcelError::new(ExcelErrorKind::NImpl)
3834                    .with_message("Table reference without specifier is unsupported".to_string()));
3835            }
3836        };
3837
3838        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
3839    }
3840
3841    pub fn default_sheet_id(&self) -> SheetId {
3842        self.graph.default_sheet_id()
3843    }
3844
3845    pub fn default_sheet_name(&self) -> &str {
3846        self.graph.default_sheet_name()
3847    }
3848
3849    /// Update the workbook seed for deterministic RNGs in functions.
3850    pub fn set_workbook_seed(&mut self, seed: u64) {
3851        self.config.workbook_seed = seed;
3852    }
3853
3854    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
3855    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
3856        self.config.volatile_level = level;
3857    }
3858
3859    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
3860    pub fn set_deterministic_mode(
3861        &mut self,
3862        mode: crate::engine::DeterministicMode,
3863    ) -> Result<(), ExcelError> {
3864        let clock = mode.build_clock()?;
3865        self.config.deterministic_mode = mode;
3866        self.clock = crate::timezone::SnapshotClock::new(clock);
3867        Ok(())
3868    }
3869
3870    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
3871    /// volatile date/time builtins (`NOW()`, `TODAY()`).
3872    ///
3873    /// The provider is the clock *source*; per spec §7.11 the engine samples
3874    /// it once at the start of every evaluation request and all reads within
3875    /// that recalc (including SCC iteration passes) observe the frozen
3876    /// sample.
3877    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
3878        self.clock = crate::timezone::SnapshotClock::new(clock);
3879    }
3880
3881    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
3882        self.config.deterministic_mode.validate()
3883    }
3884
3885    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
3886        self.graph.sheet_id(name)
3887    }
3888
3889    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
3890        self.add_sheet(name)
3891            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
3892    }
3893
3894    pub fn sheet_name(&self, id: SheetId) -> &str {
3895        self.graph.sheet_name(id)
3896    }
3897
3898    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
3899        let id = self.graph.add_sheet(name)?;
3900        self.ensure_arrow_sheet(name);
3901        // Adding a sheet does not change any existing span result or dependency.
3902        self.mark_topology_edited();
3903        Ok(id)
3904    }
3905
3906    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
3907        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
3908            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
3909        })?;
3910        if new_name.is_empty() || new_name.len() > 255 {
3911            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
3912        }
3913        if self.graph.sheet_id(new_name).is_some() {
3914            return Err(ExcelError::new(ExcelErrorKind::Value)
3915                .with_message(format!("Sheet '{new_name}' already exists")));
3916        }
3917        // Materialize only spans on the source sheet so graph duplication sees
3918        // the formulas being copied. Spans on unrelated sheets remain active.
3919        self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
3920            .map_err(Self::editor_error_to_excel)?;
3921        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
3922
3923        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
3924            let mut copied_sheet = source_sheet;
3925            copied_sheet.name = Arc::<str>::from(new_name);
3926            self.arrow_sheets.sheets.push(copied_sheet);
3927        } else {
3928            self.ensure_arrow_sheet(new_name);
3929        }
3930
3931        let duplicated_formulas = self
3932            .graph
3933            .formula_vertices()
3934            .into_iter()
3935            .filter(|vertex| {
3936                self.graph
3937                    .get_cell_ref(*vertex)
3938                    .is_some_and(|cell| cell.sheet_id == new_id)
3939            })
3940            .collect::<Vec<_>>();
3941        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
3942        self.mark_topology_edited();
3943        Ok(new_id)
3944    }
3945
3946    fn ensure_arrow_sheet(&mut self, name: &str) {
3947        if self.arrow_sheets.sheet(name).is_some() {
3948            return;
3949        }
3950        self.arrow_sheets
3951            .sheets
3952            .push(crate::arrow_store::ArrowSheet {
3953                name: std::sync::Arc::<str>::from(name),
3954                date_system: self.config.date_system,
3955                columns: Vec::new(),
3956                nrows: 0,
3957                chunk_starts: Vec::new(),
3958                chunk_rows: 32 * 1024,
3959            });
3960    }
3961
3962    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
3963        let name = self.graph.sheet_name(sheet_id).to_string();
3964        // Removing a sheet only affects spans on that sheet and spans reading
3965        // from that sheet. Preserve spans on unrelated sheets so sheet
3966        // lifecycle operations do not collapse the whole FormulaPlane.
3967        self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
3968            .map_err(Self::editor_error_to_excel)?;
3969        self.graph.remove_sheet(sheet_id)?;
3970        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
3971        // Sheet removal can change cross-sheet refs, names, and default-sheet
3972        // resolution. Until those domains have a complete exact dependency
3973        // proof, retain the documented graph-owned global invalidation.
3974        self.clear_all_computed_overlays();
3975        self.mark_all_formula_vertices_dirty();
3976        self.clear_staged_formulas_for_sheet(&name);
3977        if self.row_visibility.remove(&sheet_id).is_some() {
3978            self.invalidate_row_visibility_mask_cache();
3979        }
3980        self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
3981        self.mark_topology_edited();
3982        Ok(())
3983    }
3984
3985    /// Helper to synchronize the Arrow-backed storage layer.
3986    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
3987        if let Some(asheet) = self
3988            .arrow_sheets
3989            .sheets
3990            .iter_mut()
3991            .find(|s| s.name.as_ref() == target_name)
3992        {
3993            asheet.name = std::sync::Arc::<str>::from(new_name);
3994            return true;
3995        }
3996        false
3997    }
3998
3999    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4000        let old_name = self.graph.sheet_name(sheet_id).to_string();
4001
4002        // Speculative Storage Update
4003        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4004        self.rename_sheet_in_arrow_store(&old_name, new_name);
4005
4006        // Graph Update (Metadata + Rescue Logic)
4007        match self.graph.rename_sheet(sheet_id, new_name) {
4008            Ok(_) => {
4009                self.rename_staged_formula_sheet(&old_name, new_name);
4010                // Success! Invalidate cache for the moved sheet
4011                let sheet_vertices: Vec<VertexId> =
4012                    self.graph.vertices_in_sheet(sheet_id).collect();
4013                for v_id in sheet_vertices {
4014                    self.graph.mark_vertex_dirty(v_id);
4015                }
4016                // Sheet rename preserves SheetId and therefore formula dependencies.
4017                self.mark_topology_edited();
4018                Ok(())
4019            }
4020            Err(e) => {
4021                // ROLLBACK: Revert storage if graph rejected the name
4022                self.rename_sheet_in_arrow_store(new_name, &old_name);
4023                Err(e)
4024            }
4025        }
4026    }
4027
4028    pub fn named_ranges_iter(
4029        &self,
4030    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4031        self.graph.named_ranges_iter()
4032    }
4033
4034    pub fn sheet_named_ranges_iter(
4035        &self,
4036    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4037        self.graph.sheet_named_ranges_iter()
4038    }
4039
4040    pub fn resolve_name_entry(
4041        &self,
4042        name: &str,
4043        current_sheet: SheetId,
4044    ) -> Option<&crate::engine::named_range::NamedRange> {
4045        self.graph.resolve_name_entry(name, current_sheet)
4046    }
4047
4048    /// The [`NameScope`] an optional scope-sheet argument denotes.
4049    ///
4050    /// `None` means **workbook scope**, not "the default sheet": a caller that
4051    /// supplies no sheet context is asking about workbook-scoped names only.
4052    /// An unknown sheet name is a malformed query and errors rather than
4053    /// silently degrading to another sheet's scope (issue #110).
4054    ///
4055    /// This is the one owned derivation from `Option<&str>` to a name scope;
4056    /// every scope-taking entry point routes through it.
4057    pub(crate) fn name_query_scope(
4058        &self,
4059        scope_sheet: Option<&str>,
4060    ) -> Result<NameScope, ExcelError> {
4061        match scope_sheet {
4062            None => Ok(NameScope::Workbook),
4063            Some(sheet) => self
4064                .graph
4065                .sheet_id(sheet)
4066                .map(NameScope::Sheet)
4067                .ok_or_else(|| {
4068                    ExcelError::new(ExcelErrorKind::Ref)
4069                        .with_message(format!("name scope sheet not found: {sheet}"))
4070                }),
4071        }
4072    }
4073
4074    /// Whether `name` resolves in the scope denoted by `scope_sheet`.
4075    ///
4076    /// `scope_sheet == None` asks about workbook scope only; a name scoped to a
4077    /// single sheet (including the default sheet) does not answer it. An unknown
4078    /// sheet name resolves nothing.
4079    /// Resolve a [`SharedSheetLocator`](crate::reference::SharedSheetLocator)
4080    /// against an explicit context sheet.
4081    ///
4082    /// Thin forwarder to
4083    /// [`SheetRegistry::resolve_locator`](crate::engine::sheet_registry::SheetRegistry::resolve_locator),
4084    /// the single owned derivation. `Current` resolves to `context_sheet`, never
4085    /// to the workbook's default sheet.
4086    fn resolve_sheet_locator(
4087        &self,
4088        locator: &crate::reference::SharedSheetLocator<'_>,
4089        context_sheet: SheetId,
4090    ) -> Result<SheetId, ExcelError> {
4091        self.graph
4092            .sheet_reg()
4093            .resolve_locator(locator, context_sheet)
4094    }
4095
4096    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4097        let Ok(scope) = self.name_query_scope(scope_sheet) else {
4098            return false;
4099        };
4100        self.graph
4101            .resolve_name_entry_in_scope(name, scope)
4102            .is_some()
4103    }
4104
4105    /// The current value of `name` in the scope denoted by `scope_sheet`.
4106    ///
4107    /// Scoping follows [`Self::has_name`]: `None` is workbook scope only.
4108    pub fn resolved_name_value(
4109        &self,
4110        name: &str,
4111        scope_sheet: Option<&str>,
4112    ) -> Option<LiteralValue> {
4113        let scope = self.name_query_scope(scope_sheet).ok()?;
4114        let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4115        self.graph.get_value(entry.vertex)
4116    }
4117
4118    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4119        let entry = self.graph.resolve_table_entry(name)?;
4120        Some(TableMetadata {
4121            name: entry.name.clone(),
4122            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4123            start_row: entry.range.start.coord.row() + 1,
4124            start_col: entry.range.start.coord.col() + 1,
4125            end_row: entry.range.end.coord.row() + 1,
4126            end_col: entry.range.end.coord.col() + 1,
4127            header_row: entry.header_row,
4128            headers: entry.headers.clone(),
4129            totals_row: entry.totals_row,
4130        })
4131    }
4132
4133    /// Metadata for every defined table, ordered by name.
4134    pub fn tables(&self) -> Vec<TableMetadata> {
4135        self.graph
4136            .table_names()
4137            .into_iter()
4138            .filter_map(|name| self.table_metadata(&name))
4139            .collect()
4140    }
4141
4142    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4143        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4144
4145        for (name, named) in self.graph.named_ranges_iter() {
4146            out.push(crate::engine::named_range::NamedRangeSnapshot {
4147                name: name.clone(),
4148                scope: NameScope::Workbook,
4149                definition: named.definition.clone(),
4150            });
4151        }
4152
4153        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4154            out.push(crate::engine::named_range::NamedRangeSnapshot {
4155                name: name.clone(),
4156                scope: NameScope::Sheet(*sheet_id),
4157                definition: named.definition.clone(),
4158            });
4159        }
4160
4161        out.sort_by(|a, b| {
4162            let a_scope = match a.scope {
4163                NameScope::Workbook => (0u8, 0u32),
4164                NameScope::Sheet(id) => (1u8, u32::from(id)),
4165            };
4166            let b_scope = match b.scope {
4167                NameScope::Workbook => (0u8, 0u32),
4168                NameScope::Sheet(id) => (1u8, u32::from(id)),
4169            };
4170            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4171        });
4172
4173        out
4174    }
4175
4176    pub fn named_ranges_snapshot_for_sheet(
4177        &self,
4178        sheet_id: SheetId,
4179    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4180        self.named_ranges_snapshot()
4181            .into_iter()
4182            .filter(|entry| match entry.scope {
4183                NameScope::Workbook => true,
4184                NameScope::Sheet(id) => id == sheet_id,
4185            })
4186            .collect()
4187    }
4188
4189    pub fn define_name(
4190        &mut self,
4191        name: &str,
4192        definition: NamedDefinition,
4193        scope: NameScope,
4194    ) -> Result<(), ExcelError> {
4195        // A new define can flip resolution for spans that previously resolved
4196        // the same name through another scope (e.g. a sheet-scoped name
4197        // shadowing a workbook-scoped one). Demote those spans BEFORE the
4198        // registry changes so their cells re-ingest and re-resolve through
4199        // the normal legacy path.
4200        self.graph.validate_define_name(name, scope)?;
4201        let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4202        if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4203            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4204                "formula-name shadowing with active FormulaPlane dependents is not supported",
4205            ));
4206        }
4207        let prepared = self
4208            .prepare_name_dependent_span_demotion([name])
4209            .map_err(Self::editor_error_to_excel)?;
4210        let demoted = self
4211            .commit_name_dependent_span_demotion(prepared)
4212            .map_err(Self::editor_error_to_excel)?;
4213        self.graph.define_name(name, definition, scope)?;
4214        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4215        if !demoted {
4216            self.mark_topology_edited();
4217        }
4218        Ok(())
4219    }
4220
4221    pub fn update_name(
4222        &mut self,
4223        name: &str,
4224        definition: NamedDefinition,
4225        scope: NameScope,
4226    ) -> Result<(), ExcelError> {
4227        // Demote name-dependent spans BEFORE the registry update: the demoted
4228        // cells re-materialize as legacy vertices attached to the name vertex
4229        // (via their resolved-name dep plans), so the registry update's
4230        // dependent dirtying reaches them exactly like long-lived legacy
4231        // formulas.
4232        self.graph.validate_existing_name(name, scope)?;
4233        let prepared = self
4234            .prepare_name_dependent_span_demotion([name])
4235            .map_err(Self::editor_error_to_excel)?;
4236        let demoted = self
4237            .commit_name_dependent_span_demotion(prepared)
4238            .map_err(Self::editor_error_to_excel)?;
4239        self.graph.update_name(name, definition, scope)?;
4240        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4241        if !demoted {
4242            self.mark_topology_edited();
4243        }
4244        Ok(())
4245    }
4246
4247    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4248        // Demote first (see update_name): the demoted legacy vertices become
4249        // dependents of the name vertex, so delete_name dirties them and they
4250        // re-evaluate to #NAME? exactly as legacy formulas do.
4251        self.graph.validate_existing_name(name, scope)?;
4252        let prepared = self
4253            .prepare_name_dependent_span_demotion([name])
4254            .map_err(Self::editor_error_to_excel)?;
4255        let demoted = self
4256            .commit_name_dependent_span_demotion(prepared)
4257            .map_err(Self::editor_error_to_excel)?;
4258        self.graph.delete_name(name, scope)?;
4259        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4260        if !demoted {
4261            self.mark_topology_edited();
4262        }
4263        Ok(())
4264    }
4265
4266    fn exact_name_dependent_span_refs<'a>(
4267        &self,
4268        names: impl IntoIterator<Item = &'a str>,
4269    ) -> Vec<FormulaSpanRef> {
4270        let authority = self.graph.formula_authority();
4271        let mut refs = names
4272            .into_iter()
4273            .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4274            .collect::<Vec<_>>();
4275        refs.sort_unstable_by_key(|span_ref| {
4276            (span_ref.id.0, span_ref.generation, span_ref.version)
4277        });
4278        refs.dedup();
4279        refs
4280    }
4281
4282    fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4283        events
4284            .iter()
4285            .filter_map(|event| match event {
4286                ChangeEvent::DefineName { name, .. }
4287                | ChangeEvent::UpdateName { name, .. }
4288                | ChangeEvent::DeleteName { name, .. }
4289                | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4290                _ => None,
4291            })
4292            .collect()
4293    }
4294
4295    fn prepare_name_dependent_span_demotion<'a>(
4296        &mut self,
4297        names: impl IntoIterator<Item = &'a str>,
4298    ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4299    {
4300        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4301            return Ok(None);
4302        }
4303        let refs = self.exact_name_dependent_span_refs(names);
4304        if refs.is_empty() {
4305            return Ok(None);
4306        }
4307
4308        // Exact demotion creates dirty legacy vertices. Preserve the old
4309        // span's clean/dirty state so the following name mutation remains the
4310        // sole authority that dirties formulas whose resolution changed.
4311        let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4312        let clean_cells = {
4313            let authority = self.graph.formula_authority();
4314            refs.iter()
4315                .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4316                .flat_map(|span| span.domain.iter())
4317                .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4318                .map(|coord| {
4319                    CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4320                })
4321                .collect::<Vec<_>>()
4322        };
4323        self.prepare_formula_span_demotion(&refs)
4324            .map(|prepared| Some((prepared, clean_cells)))
4325            .map_err(|error| match error {
4326                FormulaSpanDemotionError::Resource(error) => error.into(),
4327                error => crate::engine::EditorError::TransactionFailed {
4328                    reason: format!(
4329                        "FormulaPlane name-dependent demotion preparation failed: {error}"
4330                    ),
4331                },
4332            })
4333    }
4334
4335    fn commit_name_dependent_span_demotion(
4336        &mut self,
4337        prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4338    ) -> Result<bool, crate::engine::EditorError> {
4339        let Some((prepared, clean_cells)) = prepared else {
4340            return Ok(false);
4341        };
4342        self.commit_prepared_formula_span_demotion(prepared)
4343            .map_err(|error| crate::engine::EditorError::TransactionFailed {
4344                reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4345            })?;
4346        for cell in clean_cells {
4347            if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4348                self.graph.set_dirty(vertex_id, false);
4349            }
4350        }
4351        Ok(true)
4352    }
4353
4354    pub fn define_table(
4355        &mut self,
4356        name: &str,
4357        range: crate::reference::RangeRef,
4358        header_row: bool,
4359        headers: Vec<String>,
4360        totals_row: bool,
4361    ) -> Result<(), ExcelError> {
4362        self.graph
4363            .define_table(name, range, header_row, headers, totals_row)?;
4364        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4365        self.mark_topology_edited();
4366        Ok(())
4367    }
4368
4369    pub fn define_source_scalar(
4370        &mut self,
4371        name: &str,
4372        version: Option<u64>,
4373    ) -> Result<(), ExcelError> {
4374        self.graph.define_source_scalar(name, version)?;
4375        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4376        self.mark_topology_edited();
4377        Ok(())
4378    }
4379
4380    pub fn define_source_table(
4381        &mut self,
4382        name: &str,
4383        version: Option<u64>,
4384    ) -> Result<(), ExcelError> {
4385        self.graph.define_source_table(name, version)?;
4386        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4387        self.mark_topology_edited();
4388        Ok(())
4389    }
4390
4391    pub fn set_source_scalar_version(
4392        &mut self,
4393        name: &str,
4394        version: Option<u64>,
4395    ) -> Result<(), ExcelError> {
4396        self.graph.set_source_scalar_version(name, version)?;
4397        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4398            self.graph
4399                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4400        }
4401        Ok(())
4402    }
4403
4404    pub fn set_source_table_version(
4405        &mut self,
4406        name: &str,
4407        version: Option<u64>,
4408    ) -> Result<(), ExcelError> {
4409        self.graph.set_source_table_version(name, version)?;
4410        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4411            self.graph
4412                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4413        }
4414        Ok(())
4415    }
4416
4417    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4418        self.graph.invalidate_source(name)?;
4419        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4420            self.graph
4421                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4422        }
4423        Ok(())
4424    }
4425
4426    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4427        self.graph.get_value(vertex)
4428    }
4429
4430    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4431        self.graph.get_cell_value(sheet, row, col)
4432    }
4433
4434    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4435        self.graph.get_vertex_for_cell(cell)
4436    }
4437
4438    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4439        self.graph.get_evaluation_vertices()
4440    }
4441
4442    /// Return read-only baseline counters for FormulaPlane/dispatch benchmarking.
4443    pub fn baseline_stats(&self) -> EngineBaselineStats {
4444        let graph = self.graph.baseline_stats();
4445        let formula_authority = self.graph.formula_authority();
4446        let formula_dirty = self.graph.formula_dirty_stats();
4447        EngineBaselineStats {
4448            graph_vertex_count: graph.graph_vertex_count,
4449            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4450            graph_edge_count: graph.graph_edge_count,
4451            dirty_vertex_count: graph.dirty_vertex_count,
4452            evaluation_vertex_count: graph.evaluation_vertex_count,
4453            formula_ast_root_count: graph.formula_ast_root_count,
4454            formula_ast_node_count: graph.formula_ast_node_count,
4455            staged_formula_count: self.staged_formula_count(),
4456            formula_plane_active_span_count: formula_authority.active_span_count(),
4457            formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4458            formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4459            formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4460            formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4461            formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4462            formula_plane_dirty_pending_events: formula_dirty.pending_events,
4463            formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4464            formula_plane_dirty_span_region_events_recorded: formula_dirty
4465                .span_region_events_recorded,
4466            formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4467            formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4468            formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4469            formula_plane_cycle_member_span_demotions: self
4470                .formula_plane_cycle_member_span_demotions,
4471        }
4472    }
4473
4474    /// Mutation revision captured by read-only engine reports.
4475    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4476        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4477    }
4478
4479    #[cfg(test)]
4480    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4481        self.used_axis_bounds_cache
4482            .read()
4483            .ok()
4484            .and_then(|guard| {
4485                guard.as_ref().map(|cache| {
4486                    (
4487                        cache.row_hits.load(Ordering::Relaxed),
4488                        cache.row_misses.load(Ordering::Relaxed),
4489                        cache.col_hits.load(Ordering::Relaxed),
4490                        cache.col_misses.load(Ordering::Relaxed),
4491                    )
4492                })
4493            })
4494            .unwrap_or((0, 0, 0, 0))
4495    }
4496
4497    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4498        self.graph.set_first_load_assume_new(enabled);
4499    }
4500
4501    pub fn first_load_assume_new(&self) -> bool {
4502        self.graph.first_load_assume_new()
4503    }
4504
4505    pub fn reset_ensure_touched(&mut self) {
4506        self.graph.reset_ensure_touched();
4507    }
4508
4509    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4510        self.graph.finalize_sheet_index(sheet);
4511    }
4512
4513    /// Execute a named Engine action.
4514    ///
4515    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4516    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4517    /// provide rollback/atomicity.
4518    ///
4519    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4520    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
4521    pub fn action<T>(
4522        &mut self,
4523        name: impl AsRef<str>,
4524        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4525    ) -> Result<T, crate::engine::EditorError> {
4526        if self.action_depth != 0 {
4527            return Err(crate::engine::EditorError::TransactionFailed {
4528                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
4529                    .to_string(),
4530            });
4531        }
4532
4533        self.action_depth = 1;
4534        let engine_ptr: *mut Engine<R> = self;
4535        let _guard = ActionDepthGuard {
4536            engine: engine_ptr,
4537            _marker: std::marker::PhantomData,
4538        };
4539
4540        let mut tx = EngineAction {
4541            engine: self,
4542            name: name.as_ref().to_string(),
4543            log: None,
4544            arrow_undo: None,
4545            atomic_policy: false,
4546        };
4547        f(&mut tx)
4548    }
4549
4550    /// Execute a named Engine action with atomic commit/rollback semantics.
4551    ///
4552    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
4553    pub fn action_atomic<T>(
4554        &mut self,
4555        name: impl Into<String>,
4556        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4557    ) -> Result<T, crate::engine::EditorError> {
4558        let (v, _j) = self.action_atomic_journal(name, f)?;
4559        Ok(v)
4560    }
4561
4562    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
4563    pub fn action_atomic_journal<T>(
4564        &mut self,
4565        name: impl Into<String>,
4566        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4567    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4568        if self.action_depth != 0 {
4569            return Err(crate::engine::EditorError::TransactionFailed {
4570                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4571                    .to_string(),
4572            });
4573        }
4574
4575        self.action_depth = 1;
4576        let engine_ptr: *mut Engine<R> = self;
4577        let _guard = ActionDepthGuard {
4578            engine: engine_ptr,
4579            _marker: std::marker::PhantomData,
4580        };
4581
4582        let name_str = name.into();
4583        let mut log = crate::engine::ChangeLog::new();
4584        let start_len = log.len();
4585        self.action_atomic_impl(&mut log, start_len, name_str, f)
4586    }
4587
4588    fn action_atomic_impl<T>(
4589        &mut self,
4590        log: &mut crate::engine::ChangeLog,
4591        start_len: usize,
4592        name: String,
4593        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4594    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
4595        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
4596        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
4597
4598        let log_ptr: *mut crate::engine::ChangeLog = log;
4599        let mut tx = EngineAction {
4600            engine: self,
4601            name: name.clone(),
4602            log: Some(log_ptr),
4603            arrow_undo: Some(arrow_ptr),
4604            atomic_policy: true,
4605        };
4606
4607        let res = f(&mut tx);
4608
4609        // Capture graph structural delta for this action.
4610        let graph_events: Vec<crate::engine::ChangeEvent> =
4611            unsafe { (&*log_ptr).events() }[start_len..].to_vec();
4612        let graph_batch = crate::engine::GraphUndoBatch {
4613            events: graph_events,
4614        };
4615        let affected_cells = arrow_undo.ops.len();
4616        let journal = crate::engine::ActionJournal {
4617            name,
4618            graph: graph_batch,
4619            arrow: arrow_undo,
4620            affected_cells,
4621        };
4622
4623        match res {
4624            Ok(v) => {
4625                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4626                    for event in &journal.graph.events {
4627                        self.record_formula_plane_change_for_event(event);
4628                    }
4629                    self.mark_data_edited();
4630                }
4631                Ok((v, journal))
4632            }
4633            Err(e) => {
4634                if let Err(rb) = self.rollback_from_action_journal(&journal) {
4635                    return Err(crate::engine::EditorError::TransactionFailed {
4636                        reason: format!(
4637                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
4638                        ),
4639                    });
4640                }
4641                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
4642                    for event in &journal.graph.events {
4643                        self.record_formula_plane_change_for_event(event);
4644                    }
4645                }
4646                Err(e)
4647            }
4648        }
4649    }
4650
4651    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
4652    ///
4653    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
4654    /// - Dependency graph structural edits (via inverse ChangeEvents)
4655    /// - Arrow-truth overlay writes mirrored from ChangeEvents
4656    /// - ChangeLog entries (truncated back to the pre-action length)
4657    pub fn action_with_logger<T>(
4658        &mut self,
4659        log: &mut crate::engine::ChangeLog,
4660        name: impl AsRef<str>,
4661        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
4662    ) -> Result<T, crate::engine::EditorError> {
4663        if self.action_depth != 0 {
4664            return Err(crate::engine::EditorError::TransactionFailed {
4665                reason: "Nested Engine::action calls are not supported (deterministic rule)"
4666                    .to_string(),
4667            });
4668        }
4669
4670        self.action_depth = 1;
4671        let engine_ptr: *mut Engine<R> = self;
4672        let _guard = ActionDepthGuard {
4673            engine: engine_ptr,
4674            _marker: std::marker::PhantomData,
4675        };
4676
4677        let start_len = log.len();
4678        let name_str = name.as_ref().to_string();
4679        log.begin_compound(name_str.clone());
4680
4681        // Use the provided ChangeLog as an observability sink.
4682        // Correctness is provided by the internal `ActionJournal` returned from the atomic impl.
4683        let res = self.action_atomic_impl(log, start_len, name_str, f);
4684
4685        match res {
4686            Ok((v, _journal)) => {
4687                log.end_compound();
4688                Ok(v)
4689            }
4690            Err(e) => {
4691                // Close compound and truncate log as cleanup only.
4692                log.end_compound();
4693                log.truncate(start_len);
4694                Err(e)
4695            }
4696        }
4697    }
4698
4699    fn rollback_from_action_journal(
4700        &mut self,
4701        journal: &crate::engine::ActionJournal,
4702    ) -> Result<(), crate::engine::EditorError> {
4703        // 1) Roll back the dependency graph structure.
4704        journal.graph.undo(&mut self.graph)?;
4705        // 2) Roll back engine row-visibility sidecar events.
4706        self.apply_inverse_row_visibility_events(&journal.graph.events);
4707        // 3) Roll back Arrow-truth overlays.
4708        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
4709        Ok(())
4710    }
4711
4712    fn rollback_from_change_events(
4713        &mut self,
4714        events: &[crate::engine::ChangeEvent],
4715    ) -> Result<(), crate::engine::EditorError> {
4716        use crate::engine::ChangeEvent;
4717
4718        // 1) Roll back the dependency graph.
4719        {
4720            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
4721            let mut compound_stack: Vec<usize> = Vec::new();
4722            for ev in events.iter().rev() {
4723                match ev {
4724                    ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
4725                    ChangeEvent::CompoundStart { depth, .. } => {
4726                        if compound_stack.last() == Some(depth) {
4727                            compound_stack.pop();
4728                        }
4729                    }
4730                    ChangeEvent::SetRowVisibility { .. } => {
4731                        // Engine-side metadata handled after dropping graph editor borrow.
4732                    }
4733                    _ => {
4734                        editor.apply_inverse(ev.clone())?;
4735                    }
4736                }
4737            }
4738        }
4739
4740        // 2) Roll back engine row-visibility metadata.
4741        for ev in events.iter().rev() {
4742            self.apply_inverse_row_visibility_event(ev);
4743        }
4744
4745        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
4746        for ev in events.iter().rev() {
4747            self.mirror_inverse_change_to_arrow(ev);
4748        }
4749
4750        Ok(())
4751    }
4752
4753    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
4754        let sheet_id = self.graph.sheet_id(sheet)?;
4755        let coord = Coord::from_excel(row, col, true, true);
4756        let cell = CellRef::new(sheet_id, coord);
4757        let vid = self.graph.get_vertex_for_cell(&cell)?;
4758        let ast_id = self.graph.get_formula_id(vid)?;
4759        self.graph
4760            .data_store()
4761            .retrieve_ast(ast_id, self.graph.sheet_reg())
4762    }
4763
4764    pub fn define_name_with_logger(
4765        &mut self,
4766        log: &mut crate::engine::ChangeLog,
4767        name: &str,
4768        definition: NamedDefinition,
4769        scope: NameScope,
4770    ) -> Result<(), crate::engine::EditorError> {
4771        self.graph
4772            .validate_define_name(name, scope)
4773            .map_err(crate::engine::EditorError::Excel)?;
4774        let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
4775        if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
4776            return Err(crate::engine::EditorError::TransactionUnsupported {
4777                reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
4778                    .to_string(),
4779            });
4780        }
4781        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4782        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4783        {
4784            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4785            editor.define_name(name, definition, scope)?;
4786        }
4787        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4788        if !demoted {
4789            self.mark_topology_edited();
4790        }
4791        Ok(())
4792    }
4793
4794    pub fn update_name_with_logger(
4795        &mut self,
4796        log: &mut crate::engine::ChangeLog,
4797        name: &str,
4798        definition: NamedDefinition,
4799        scope: NameScope,
4800    ) -> Result<(), crate::engine::EditorError> {
4801        self.graph
4802            .validate_existing_name(name, scope)
4803            .map_err(crate::engine::EditorError::Excel)?;
4804        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4805        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4806        {
4807            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4808            editor.update_name(name, definition, scope)?;
4809        }
4810        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4811        if !demoted {
4812            self.mark_topology_edited();
4813        }
4814        Ok(())
4815    }
4816
4817    pub fn delete_name_with_logger(
4818        &mut self,
4819        log: &mut crate::engine::ChangeLog,
4820        name: &str,
4821        scope: NameScope,
4822    ) -> Result<(), crate::engine::EditorError> {
4823        self.graph
4824            .validate_existing_name(name, scope)
4825            .map_err(crate::engine::EditorError::Excel)?;
4826        let prepared = self.prepare_name_dependent_span_demotion([name])?;
4827        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
4828        {
4829            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
4830            editor.delete_name(name, scope)?;
4831        }
4832        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4833        if !demoted {
4834            self.mark_topology_edited();
4835        }
4836        Ok(())
4837    }
4838
4839    pub fn edit_with_logger<T>(
4840        &mut self,
4841        log: &mut crate::engine::ChangeLog,
4842        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
4843    ) -> Result<T, crate::engine::EditorError> {
4844        // Record starting log length so we can mirror only newly-recorded events.
4845        let start_len = log.len();
4846
4847        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
4848        // (graph value cache is intentionally empty in canonical mode).
4849        struct ArrowSpillReader<'a> {
4850            sheets: &'a crate::arrow_store::SheetStore,
4851        }
4852        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
4853            fn read_cell_value(
4854                &self,
4855                sheet: &str,
4856                row: u32,
4857                col: u32,
4858            ) -> Option<formualizer_common::LiteralValue> {
4859                use formualizer_common::LiteralValue;
4860                let asheet = self.sheets.sheet(sheet)?;
4861                let r0 = row.saturating_sub(1) as usize;
4862                let c0 = col.saturating_sub(1) as usize;
4863                let v = asheet.get_cell_value(r0, c0);
4864                if matches!(v, LiteralValue::Empty) {
4865                    None
4866                } else {
4867                    Some(v)
4868                }
4869            }
4870        }
4871
4872        let ret = {
4873            let spill_reader = ArrowSpillReader {
4874                sheets: &self.arrow_sheets,
4875            };
4876            let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
4877                &mut self.graph,
4878                log,
4879                &spill_reader,
4880            );
4881            f(&mut editor)
4882        };
4883
4884        let new_events = log.events()[start_len..].to_vec();
4885        if new_events.iter().any(|event| {
4886            matches!(
4887                event,
4888                ChangeEvent::DefineName { .. }
4889                    | ChangeEvent::UpdateName { .. }
4890                    | ChangeEvent::DeleteName { .. }
4891            )
4892        }) {
4893            self.rollback_from_change_events(&new_events)?;
4894            log.truncate(start_len);
4895            return Err(crate::engine::EditorError::TransactionUnsupported {
4896                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
4897            });
4898        }
4899
4900        // Mirror value-impacting graph events to Arrow for forward edits.
4901        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
4902        for ev in &new_events {
4903            self.mirror_forward_change_to_arrow(ev);
4904        }
4905        for ev in &new_events {
4906            self.record_formula_plane_change_for_event(ev);
4907        }
4908
4909        Ok(ret)
4910    }
4911
4912    pub(crate) fn preflight_replay_admission(
4913        &mut self,
4914        events: &[ChangeEvent],
4915        forward: bool,
4916    ) -> Result<(), crate::engine::EditorError> {
4917        if !self.graph_admission_enabled() {
4918            return Ok(());
4919        }
4920        let mut vertex_delta = 0i128;
4921        let mut edge_delta = 0i128;
4922        let mut added_vertices = 0usize;
4923        let mut added_edges = 0usize;
4924        let mut formula_cells = BTreeSet::new();
4925        for event in events {
4926            match event {
4927                ChangeEvent::AddVertex {
4928                    formula,
4929                    coord,
4930                    sheet_id,
4931                    ..
4932                } => {
4933                    let delta = if forward { 1 } else { -1 };
4934                    vertex_delta += delta;
4935                    if forward {
4936                        added_vertices = added_vertices.saturating_add(1);
4937                        if formula.is_some() {
4938                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4939                        }
4940                    }
4941                }
4942                ChangeEvent::RemoveVertex {
4943                    old_formula,
4944                    coord,
4945                    sheet_id,
4946                    ..
4947                } => {
4948                    let delta = if forward { -1 } else { 1 };
4949                    vertex_delta += delta;
4950                    if !forward {
4951                        added_vertices = added_vertices.saturating_add(1);
4952                        if old_formula.is_some()
4953                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
4954                        {
4955                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
4956                        }
4957                    }
4958                }
4959                ChangeEvent::EdgeAdded { .. } => {
4960                    let delta = if forward { 1 } else { -1 };
4961                    edge_delta += delta;
4962                    if forward {
4963                        added_edges = added_edges.saturating_add(1);
4964                    }
4965                }
4966                ChangeEvent::EdgeRemoved { .. } => {
4967                    let delta = if forward { -1 } else { 1 };
4968                    edge_delta += delta;
4969                    if !forward {
4970                        added_edges = added_edges.saturating_add(1);
4971                    }
4972                }
4973                ChangeEvent::SetFormula {
4974                    addr, old_formula, ..
4975                } => {
4976                    if forward || old_formula.is_some() {
4977                        formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
4978                    }
4979                }
4980                _ => {}
4981            }
4982        }
4983        let stats = self.graph.baseline_stats();
4984        let final_vertices = i128::try_from(stats.graph_vertex_count)
4985            .ok()
4986            .and_then(|count| count.checked_add(vertex_delta))
4987            .and_then(|count| usize::try_from(count).ok())
4988            .ok_or_else(|| {
4989                crate::engine::EditorError::Excel(
4990                    ExcelError::new(ExcelErrorKind::NImpl)
4991                        .with_message("replay vertex count overflow"),
4992                )
4993            })?;
4994        let final_edges = i128::try_from(stats.graph_edge_count)
4995            .ok()
4996            .and_then(|count| count.checked_add(edge_delta))
4997            .and_then(|count| usize::try_from(count).ok())
4998            .ok_or_else(|| {
4999                crate::engine::EditorError::Excel(
5000                    ExcelError::new(ExcelErrorKind::NImpl)
5001                        .with_message("replay edge count overflow"),
5002                )
5003            })?;
5004        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5005            final_vertices,
5006            final_edges,
5007            materialization_cells: formula_cells.len() as u64,
5008            added_vertices,
5009            added_edges,
5010        })
5011        .map_err(crate::engine::EditorError::Excel)
5012    }
5013
5014    pub fn undo_logged(
5015        &mut self,
5016        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5017        log: &mut crate::engine::ChangeLog,
5018    ) -> Result<(), crate::engine::EditorError> {
5019        let pending_events = log
5020            .last_group_indices()
5021            .into_iter()
5022            .map(|index| log.events()[index].clone())
5023            .collect::<Vec<_>>();
5024        self.preflight_replay_admission(&pending_events, false)?;
5025        let names = Self::name_event_names(&pending_events);
5026        let prepared =
5027            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5028        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5029        let batch = undo.undo(&mut self.graph, log)?;
5030        for item in batch.iter().rev() {
5031            self.apply_inverse_row_visibility_event(&item.event);
5032            self.apply_inverse_staged_formula_event(&item.event);
5033        }
5034        self.mirror_undo_batch_to_arrow(&batch);
5035        if !batch.is_empty() {
5036            for item in &batch {
5037                self.record_formula_plane_change_for_event(&item.event);
5038            }
5039            if !demoted && !names.is_empty() {
5040                self.mark_topology_edited();
5041            }
5042        }
5043        Ok(())
5044    }
5045
5046    pub fn redo_logged(
5047        &mut self,
5048        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5049        log: &mut crate::engine::ChangeLog,
5050    ) -> Result<(), crate::engine::EditorError> {
5051        let pending_events = undo.pending_redo_events();
5052        self.preflight_replay_admission(&pending_events, true)?;
5053        let names = Self::name_event_names(&pending_events);
5054        let prepared =
5055            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5056        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5057        let batch = undo.redo(&mut self.graph, log)?;
5058        for item in &batch {
5059            self.apply_forward_row_visibility_event(&item.event);
5060            self.apply_forward_staged_formula_event(&item.event);
5061        }
5062        self.mirror_redo_batch_to_arrow(&batch);
5063        if !batch.is_empty() {
5064            for item in &batch {
5065                self.record_formula_plane_change_for_event(&item.event);
5066            }
5067            if !demoted && !names.is_empty() {
5068                self.mark_topology_edited();
5069            }
5070        }
5071        Ok(())
5072    }
5073
5074    /// Undo the last committed atomic action using the journal stack.
5075    ///
5076    /// This path does not require a `ChangeLog`.
5077    pub fn undo_action(
5078        &mut self,
5079        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5080    ) -> Result<(), crate::engine::EditorError> {
5081        let Some(journal) = undo.pop_undo_action() else {
5082            return Ok(());
5083        };
5084        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5085            undo.push_done_action(journal);
5086            return Err(error);
5087        }
5088        let names = Self::name_event_names(&journal.graph.events);
5089        let prepared =
5090            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5091                Ok(prepared) => prepared,
5092                Err(error) => {
5093                    undo.push_done_action(journal);
5094                    return Err(error);
5095                }
5096            };
5097        let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5098            Ok(demoted) => demoted,
5099            Err(error) => {
5100                undo.push_done_action(journal);
5101                return Err(error);
5102            }
5103        };
5104
5105        journal.graph.undo(&mut self.graph)?;
5106        self.apply_inverse_row_visibility_events(&journal.graph.events);
5107        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5108        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5109            for event in &journal.graph.events {
5110                self.record_formula_plane_change_for_event(event);
5111            }
5112            if !demoted && !names.is_empty() {
5113                self.mark_topology_edited();
5114            } else {
5115                self.mark_data_edited();
5116            }
5117        }
5118
5119        undo.push_redo_action(journal);
5120        Ok(())
5121    }
5122
5123    /// Redo the last undone atomic action using the journal stack.
5124    ///
5125    /// This path does not require a `ChangeLog`.
5126    pub fn redo_action(
5127        &mut self,
5128        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5129    ) -> Result<(), crate::engine::EditorError> {
5130        let Some(journal) = undo.pop_redo_action() else {
5131            return Ok(());
5132        };
5133        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5134            undo.push_redo_action(journal);
5135            return Err(error);
5136        }
5137        let names = Self::name_event_names(&journal.graph.events);
5138        let prepared =
5139            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5140                Ok(prepared) => prepared,
5141                Err(error) => {
5142                    undo.push_redo_action(journal);
5143                    return Err(error);
5144                }
5145            };
5146        let demoted = match self.commit_name_dependent_span_demotion(prepared) {
5147            Ok(demoted) => demoted,
5148            Err(error) => {
5149                undo.push_redo_action(journal);
5150                return Err(error);
5151            }
5152        };
5153
5154        journal.graph.redo(&mut self.graph)?;
5155        self.apply_forward_row_visibility_events(&journal.graph.events);
5156        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5157        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5158            for event in &journal.graph.events {
5159                self.record_formula_plane_change_for_event(event);
5160            }
5161            if !demoted && !names.is_empty() {
5162                self.mark_topology_edited();
5163            } else {
5164                self.mark_data_edited();
5165            }
5166        }
5167
5168        undo.push_done_action(journal);
5169        Ok(())
5170    }
5171
5172    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5173        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5174        // Coord stores 0-based indices.
5175        let row = addr.coord.row() + 1;
5176        let col = addr.coord.col() + 1;
5177        (sheet, row, col)
5178    }
5179
5180    fn mirror_undo_batch_to_arrow(
5181        &mut self,
5182        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5183    ) {
5184        // Undo applies inverses in reverse order.
5185        for item in batch.iter().rev() {
5186            self.mirror_inverse_change_to_arrow(&item.event);
5187        }
5188    }
5189
5190    fn mirror_redo_batch_to_arrow(
5191        &mut self,
5192        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5193    ) {
5194        // Redo applies events in forward order.
5195        for item in batch.iter() {
5196            self.mirror_forward_change_to_arrow(&item.event);
5197        }
5198    }
5199
5200    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5201        use crate::engine::ChangeEvent;
5202        use formualizer_common::LiteralValue;
5203
5204        match ev {
5205            ChangeEvent::SetValue {
5206                addr,
5207                old_value,
5208                old_formula,
5209                ..
5210            } => {
5211                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5212                if old_formula.is_some() {
5213                    self.clear_delta_overlay_cell(&sheet, row, col);
5214                } else {
5215                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5216                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5217                }
5218            }
5219            ChangeEvent::SetFormula {
5220                addr,
5221                old_value,
5222                old_formula,
5223                ..
5224            } => {
5225                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5226                if old_formula.is_some() {
5227                    self.clear_delta_overlay_cell(&sheet, row, col);
5228                } else {
5229                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5230                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5231                }
5232            }
5233            ChangeEvent::SpillCommitted { old, new, .. } => {
5234                // Inverse: restore `old` (or clear if none).
5235                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5236                if let Some(snap) = old {
5237                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5238                }
5239            }
5240            ChangeEvent::SpillCleared { old, .. } => {
5241                // Inverse: restore prior spill.
5242                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5243            }
5244            ChangeEvent::SetRowVisibility { .. } => {
5245                // Engine-side metadata only; no Arrow overlay effect.
5246            }
5247            _ => {}
5248        }
5249    }
5250
5251    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5252        use crate::engine::ChangeEvent;
5253
5254        match ev {
5255            ChangeEvent::SetValue { addr, new, .. } => {
5256                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5257                self.mirror_value_to_overlay(&sheet, row, col, new);
5258            }
5259            ChangeEvent::SetFormula { addr, .. } => {
5260                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5261                self.clear_delta_overlay_cell(&sheet, row, col);
5262                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5263            }
5264            ChangeEvent::SpillCommitted { old, new, .. } => {
5265                if let Some(snap) = old {
5266                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5267                }
5268                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5269            }
5270            ChangeEvent::SpillCleared { old, .. } => {
5271                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5272            }
5273            ChangeEvent::SetRowVisibility { .. } => {
5274                // Engine-side metadata only; no Arrow overlay effect.
5275            }
5276            _ => {
5277                // Other graph structural operations do not have direct value effects in Arrow.
5278            }
5279        }
5280    }
5281
5282    fn mirror_spill_snapshot(
5283        &mut self,
5284        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5285        clear_only: bool,
5286    ) {
5287        use formualizer_common::LiteralValue;
5288
5289        let mut i = 0usize;
5290        for row in &snap.values {
5291            for v in row {
5292                if let Some(cell) = snap.target_cells.get(i) {
5293                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5294                    let out = if clear_only {
5295                        LiteralValue::Empty
5296                    } else {
5297                        v.clone()
5298                    };
5299                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5300                }
5301                i += 1;
5302            }
5303        }
5304        // If target_cells is longer than values (should not happen), clear remaining cells.
5305        if clear_only {
5306            for cell in snap.target_cells.iter().skip(i) {
5307                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5308                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5309            }
5310        }
5311    }
5312
5313    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5314        self.graph.set_default_sheet_by_name(name);
5315    }
5316
5317    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5318        self.graph.set_default_sheet_by_id(id);
5319    }
5320
5321    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5322        self.graph.set_sheet_index_mode(mode);
5323    }
5324
5325    fn clear_cached_static_schedule(&mut self) {
5326        self.cached_static_schedule = None;
5327    }
5328
5329    /// Mark data edited: bump snapshot and set edited flag.
5330    /// Value-only edits keep the stable-topology schedule cache alive.
5331    pub fn mark_data_edited(&mut self) {
5332        self.snapshot_id
5333            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5334        self.has_edited = true;
5335    }
5336
5337    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
5338    pub fn mark_topology_edited(&mut self) {
5339        self.snapshot_id
5340            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
5341        self.topology_epoch = self.topology_epoch.wrapping_add(1);
5342        self.graph.bump_topology_revision();
5343        self.clear_cached_static_schedule();
5344        self.cached_mixed_topology = None;
5345        if let Some(ledger) = self.active_resource_ledger.as_mut() {
5346            let released = ledger.account_mixed_cache(0);
5347            debug_assert!(released.is_ok());
5348        }
5349        self.has_edited = true;
5350    }
5351
5352    fn mark_all_formula_vertices_dirty(&mut self) {
5353        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
5354        for vertex in vertices {
5355            self.graph.mark_vertex_dirty(vertex);
5356        }
5357    }
5358
5359    fn mark_moved_formula_vertices_dirty(
5360        &mut self,
5361        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
5362    ) {
5363        for vertex in &summary.vertices_moved {
5364            if self.graph.get_formula_id(*vertex).is_some() {
5365                self.graph.mark_vertex_dirty(*vertex);
5366            }
5367        }
5368    }
5369
5370    /// Access Arrow sheet store (read-only)
5371    pub fn sheet_store(&self) -> &SheetStore {
5372        &self.arrow_sheets
5373    }
5374
5375    /// True when any sheet carries manual/filter row-visibility state.
5376    /// Used by load-time freshness checks (see `Engine::adopt_file_sheets`).
5377    pub(crate) fn has_row_visibility_state(&self) -> bool {
5378        !self.row_visibility.is_empty()
5379    }
5380
5381    /// Access Arrow sheet store (mutable)
5382    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
5383        &mut self.arrow_sheets
5384    }
5385
5386    pub fn has_staged_formulas(&self) -> bool {
5387        !self.staged_formulas.is_empty()
5388    }
5389
5390    pub fn staged_formula_count(&self) -> usize {
5391        self.staged_formulas.values().map(StagedSheet::len).sum()
5392    }
5393
5394    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
5395    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
5396        self.staged_formulas
5397            .entry(sheet.to_string())
5398            .or_default()
5399            .stage(row, col, text);
5400        self.staged_formula_index.stage(sheet, row, col);
5401    }
5402
5403    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
5404        for (row, col, _) in &staged.entries {
5405            self.staged_formula_index.remove(sheet, *row, *col);
5406        }
5407        if staged.deferred_package.is_some() {
5408            self.staged_formula_index.set_package(sheet, None);
5409        }
5410    }
5411
5412    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
5413        self.staged_formulas.insert(sheet, staged);
5414    }
5415
5416    #[doc(hidden)]
5417    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
5418        SourceFormulaIngress { engine: self }
5419    }
5420
5421    #[doc(hidden)]
5422    /// Test-only fault-injection seam. Not part of the supported API; it exists so
5423    /// integration tests in sibling crates can fail a commit at an exact point.
5424    #[doc(hidden)]
5425    #[cfg(any(test, feature = "test-support"))]
5426    pub fn set_before_prepared_span_commit_hook(
5427        &mut self,
5428        hook: impl FnOnce() + Send + Sync + 'static,
5429    ) {
5430        self.before_prepared_span_commit_hook = Some(Box::new(hook));
5431    }
5432
5433    #[doc(hidden)]
5434    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
5435    #[cfg(test)]
5436    pub(crate) fn set_before_target_preparation_commit_hook(
5437        &mut self,
5438        hook: impl FnOnce() + Send + Sync + 'static,
5439    ) {
5440        self.before_target_preparation_commit_hook = Some(Box::new(hook));
5441    }
5442
5443    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
5444        let sheet = package.sheet_name.clone();
5445        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
5446        debug_assert!(staged.deferred_package.is_none());
5447        staged.deferred_package = Some(package);
5448        staged.reconcile_attached_deferred_package();
5449        self.staged_formula_index
5450            .set_package(&sheet, staged.deferred_package.as_ref());
5451    }
5452
5453    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
5454        let mut removed = None;
5455        let mut remove_sheet = false;
5456        let mut had_package = false;
5457        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
5458            had_package = entries.deferred_package.is_some();
5459            removed = entries.remove(row, col);
5460            remove_sheet = entries.is_empty();
5461        }
5462        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
5463        if !ordinary_removed && had_package {
5464            self.staged_formula_index.touch_package(sheet);
5465        }
5466        if remove_sheet {
5467            self.staged_formulas.remove(sheet);
5468            self.staged_formula_index.set_package(sheet, None);
5469        }
5470        removed
5471    }
5472
5473    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
5474        if self.staged_formulas.remove(sheet).is_some() {
5475            self.staged_formula_index.clear_sheet(sheet);
5476        }
5477    }
5478
5479    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
5480        let Some(entries) = self.staged_formulas.remove(old) else {
5481            return;
5482        };
5483        self.staged_formula_index.clear_sheet(old);
5484        let (formulas, mut package) = entries.into_parts();
5485        for (row, col, text) in formulas {
5486            self.stage_formula_text(new, row, col, text);
5487        }
5488        if let Some(package) = package.as_mut() {
5489            package.sheet_name = new.to_string();
5490        }
5491        if let Some(package) = package {
5492            self.stage_deferred_formula_package(package);
5493        }
5494    }
5495
5496    /// Get a staged formula text for a given cell if present (cloned).
5497    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
5498        self.staged_formulas
5499            .get(sheet)
5500            .and_then(|v| v.get(row, col))
5501    }
5502
5503    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
5504        &self.formula_parse_diagnostics
5505    }
5506
5507    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
5508        std::mem::take(&mut self.formula_parse_diagnostics)
5509    }
5510
5511    pub fn clear_formula_parse_diagnostics(&mut self) {
5512        self.formula_parse_diagnostics.clear();
5513    }
5514
5515    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
5516        self.last_formula_ingest_report.as_ref()
5517    }
5518
5519    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
5520        &self.formula_ingest_report_total
5521    }
5522
5523    #[cfg(test)]
5524    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
5525        &mut self,
5526        hook: impl FnOnce() + Send + Sync + 'static,
5527    ) {
5528        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
5529    }
5530
5531    #[cfg(test)]
5532    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
5533        self.inject_target_semantic_stale_once_for_test = true;
5534    }
5535
5536    #[cfg(test)]
5537    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
5538        self.force_virtual_dep_changes_remaining_for_test = rounds;
5539    }
5540
5541    #[cfg(test)]
5542    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
5543        self.fail_evaluation_commit_preflight_once_for_test = true;
5544    }
5545
5546    #[cfg(test)]
5547    pub(crate) fn set_target_preparation_fault_for_test(
5548        &mut self,
5549        fault: crate::engine::target_preparation::TargetPreparationFault,
5550    ) {
5551        self.target_preparation_fault_for_test = Some(fault);
5552    }
5553
5554    #[cfg(test)]
5555    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
5556        self.staged_formula_index.revision()
5557    }
5558
5559    #[cfg(test)]
5560    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
5561        let ordinary_storage = self
5562            .staged_formulas
5563            .values()
5564            .map(|sheet| sheet.entries.len())
5565            .sum::<usize>();
5566        let package_storage = self
5567            .staged_formulas
5568            .values()
5569            .filter(|sheet| sheet.deferred_package.is_some())
5570            .count();
5571        ordinary_storage == self.staged_formula_index.ordinary_count()
5572            && package_storage == self.staged_formula_index.package_count()
5573            && self.staged_formulas.iter().all(|(name, sheet)| {
5574                sheet.entries.iter().all(|(row, col, _)| {
5575                    let leases = self
5576                        .staged_formula_index
5577                        .leases_in_region(name, *row, *col, *row, *col);
5578                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
5579                })
5580            })
5581    }
5582
5583    #[cfg(test)]
5584    pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
5585        self.last_formula_plane_span_eval_report.as_ref()
5586    }
5587
5588    #[cfg(test)]
5589    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
5590        self.evaluation_request_begin_count_for_test
5591    }
5592
5593    #[cfg(test)]
5594    pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
5595        self.force_source_family_fallback = force;
5596    }
5597
5598    #[cfg(test)]
5599    pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
5600        self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
5601    }
5602
5603    #[cfg(test)]
5604    pub(crate) fn set_fragmented_commit_fault_for_test(
5605        &mut self,
5606        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5607    ) {
5608        self.fragmented_commit_fault_for_test = Some(fault);
5609    }
5610
5611    #[cfg(test)]
5612    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
5613        &mut self,
5614        hook: impl FnOnce() + Send + Sync + 'static,
5615    ) {
5616        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
5617    }
5618
5619    #[cfg(test)]
5620    pub(crate) fn set_after_eager_proposal_commit_hook(
5621        &mut self,
5622        hook: impl FnOnce() + Send + Sync + 'static,
5623    ) {
5624        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
5625    }
5626
5627    #[cfg(test)]
5628    pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
5629        self.graph.formula_authority().indexes_epoch()
5630    }
5631
5632    #[cfg(test)]
5633    pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
5634        self.formula_plane_capacity_bailouts
5635    }
5636
5637    #[cfg(test)]
5638    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
5639        self.topology_epoch
5640    }
5641
5642    #[cfg(test)]
5643    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
5644        self.graph.topology_revision()
5645    }
5646
5647    #[cfg(test)]
5648    pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
5649        self.cached_mixed_topology.is_some()
5650    }
5651
5652    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
5653        self.formula_ingest_report_total.mode = report.mode;
5654        self.formula_ingest_report_total.accumulate(&report);
5655        self.last_formula_ingest_report = Some(report);
5656    }
5657
5658    fn prepare_source_formula_family(
5659        &mut self,
5660        sheet_id: SheetId,
5661        family: &crate::engine::SourceFormulaFamily,
5662        allow_function_closure: bool,
5663        planning_provider: Option<&dyn crate::traits::FunctionProvider>,
5664    ) -> Result<
5665        (
5666            crate::formula_plane::placement::PreparedAnchorOncePlacement,
5667            bool,
5668        ),
5669        SourceFamilyPreparationError,
5670    > {
5671        let transport = family
5672            .validated_complete_domain(&self.workbook_load_limits)
5673            .map_err(SourceFamilyPreparationError::contract)?;
5674        let domain = match transport {
5675            crate::engine::PlacementDomainTransport::RowRun {
5676                row_start,
5677                row_end,
5678                col,
5679            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5680            crate::engine::PlacementDomainTransport::ColRun {
5681                row,
5682                col_start,
5683                col_end,
5684            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5685            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5686                sheet_id,
5687                rect.start.row,
5688                rect.end.row,
5689                rect.start.col,
5690                rect.end.col,
5691            ),
5692        };
5693        let formula = if family.anchor_text.starts_with('=') {
5694            family.anchor_text.to_string()
5695        } else {
5696            format!("={}", family.anchor_text)
5697        };
5698        let ast = formualizer_parse::parser::parse(&formula)
5699            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5700        let relocation = if allow_function_closure {
5701            validate_anchor_once_shadow_relocation(
5702                &ast,
5703                family.anchor_coord0.row,
5704                family.anchor_coord0.col,
5705                &domain,
5706                planning_provider.unwrap_or(&self.resolver),
5707            )
5708        } else {
5709            validate_anchor_once_syntax(
5710                &ast,
5711                family.anchor_coord0.row,
5712                family.anchor_coord0.col,
5713                &domain,
5714            )
5715        };
5716        relocation.map_err(SourceFamilyPreparationError::ast)?;
5717        let ast_id = self.intern_formula_ast(&ast);
5718        let placement = CellRef::new(
5719            sheet_id,
5720            Coord::from_excel(
5721                family.anchor_coord0.row + 1,
5722                family.anchor_coord0.col + 1,
5723                true,
5724                true,
5725            ),
5726        );
5727        let ingested = {
5728            let mut pipeline = match planning_provider {
5729                Some(provider) => self.graph.ingest_pipeline(provider),
5730                None => self.ingest_pipeline(),
5731            };
5732            if allow_function_closure {
5733                pipeline = pipeline.enable_function_semantics();
5734            }
5735            pipeline.ingest_formula(
5736                FormulaAstInput::RawArena(ast_id),
5737                placement,
5738                Some(family.anchor_text.clone()),
5739            )
5740        }
5741        .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5742        let function_semantics_used = ingested
5743            .labels
5744            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5745        let candidate = FormulaPlacementCandidate::new(
5746            sheet_id,
5747            family.anchor_coord0.row,
5748            family.anchor_coord0.col,
5749            ingested.ast_id,
5750            Some(family.anchor_text.clone()),
5751        );
5752        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
5753            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
5754        prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
5755            .map(|prepared| (prepared, function_semantics_used))
5756            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
5757    }
5758
5759    fn placement_domain_from_transport(
5760        sheet_id: SheetId,
5761        transport: crate::engine::PlacementDomainTransport,
5762    ) -> PlacementDomain {
5763        match transport {
5764            crate::engine::PlacementDomainTransport::RowRun {
5765                row_start,
5766                row_end,
5767                col,
5768            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
5769            crate::engine::PlacementDomainTransport::ColRun {
5770                row,
5771                col_start,
5772                col_end,
5773            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
5774            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
5775                sheet_id,
5776                rect.start.row,
5777                rect.end.row,
5778                rect.start.col,
5779                rect.end.col,
5780            ),
5781        }
5782    }
5783
5784    #[cfg(test)]
5785    pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
5786        &mut self,
5787        sheet_name: &str,
5788        source_id: crate::engine::SourceFamilyId,
5789        expected: crate::engine::PartitionLegacyMember,
5790        replay: crate::engine::DeferredReplayFormula,
5791    ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
5792    {
5793        let sheet_id = self.graph.sheet_id_mut(sheet_name);
5794        self.graph
5795            .analyze_fragmented_exact_replay_record(
5796                &self.resolver,
5797                sheet_id,
5798                source_id,
5799                expected,
5800                replay,
5801            )
5802            .map_err(|error| format!("{error:?}"))
5803    }
5804
5805    pub(crate) fn prepare_fragmented_source_transaction(
5806        &mut self,
5807        source: &crate::engine::PartitionedSourceFormulaFamily,
5808        disposition: &crate::engine::FormulaReplayDisposition,
5809        prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5810        legacy_formulas: Vec<
5811            crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
5812        >,
5813    ) -> Result<
5814        crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5815        crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
5816    > {
5817        let transaction = self.graph.prepare_fragmented_source_transaction(
5818            &self.source_formula_token,
5819            source,
5820            disposition,
5821            prepared,
5822            legacy_formulas,
5823        )?;
5824        self.prepared_legacy_admission(
5825            transaction.legacy_graph(),
5826            transaction.materialization_cells(),
5827        )
5828        .map_err(
5829            crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
5830        )?;
5831        Ok(transaction)
5832    }
5833
5834    pub(crate) fn commit_fragmented_source_transaction(
5835        &mut self,
5836        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5837        disposition: &crate::engine::FormulaReplayDisposition,
5838    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5839        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5840        let provider_revision = self.resolver.planning_semantic_revision();
5841        let epoch = registry_guard.epoch();
5842        let resolver = &self.resolver;
5843        let decision = self.graph.commit_fragmented_source_transaction(
5844            prepared,
5845            &self.source_formula_token,
5846            disposition,
5847            epoch,
5848            provider_revision,
5849            || resolver.planning_semantic_revision(),
5850            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5851        );
5852        drop(registry_guard);
5853        decision
5854    }
5855
5856    #[cfg(test)]
5857    pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
5858        &mut self,
5859        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5860        disposition: &crate::engine::FormulaReplayDisposition,
5861        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
5862    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5863        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5864        let provider_revision = self.resolver.planning_semantic_revision();
5865        let epoch = registry_guard.epoch();
5866        let resolver = &self.resolver;
5867        let decision = self.graph.commit_fragmented_source_transaction(
5868            prepared,
5869            &self.source_formula_token,
5870            disposition,
5871            epoch,
5872            provider_revision,
5873            || resolver.planning_semantic_revision(),
5874            fault,
5875        );
5876        drop(registry_guard);
5877        decision
5878    }
5879
5880    #[cfg(test)]
5881    pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
5882        &mut self,
5883        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5884        disposition: &crate::engine::FormulaReplayDisposition,
5885        before_second_sample: impl FnOnce(),
5886    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5887        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
5888        let provider_revision = self.resolver.planning_semantic_revision();
5889        let epoch = registry_guard.epoch();
5890        let resolver = &self.resolver;
5891        let decision = self.graph.commit_fragmented_source_transaction(
5892            prepared,
5893            &self.source_formula_token,
5894            disposition,
5895            epoch,
5896            provider_revision,
5897            || {
5898                before_second_sample();
5899                resolver.planning_semantic_revision()
5900            },
5901            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5902        );
5903        drop(registry_guard);
5904        decision
5905    }
5906
5907    #[cfg(test)]
5908    pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
5909        &mut self,
5910        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
5911        disposition: &crate::engine::FormulaReplayDisposition,
5912        function_semantic_epoch: u64,
5913        function_provider_revision: Option<u64>,
5914    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
5915        self.graph.commit_fragmented_source_transaction(
5916            prepared,
5917            &self.source_formula_token,
5918            disposition,
5919            function_semantic_epoch,
5920            function_provider_revision,
5921            || function_provider_revision,
5922            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
5923        )
5924    }
5925
5926    pub(crate) fn analyze_partitioned_source_family_for_transaction(
5927        &mut self,
5928        sheet_name: &str,
5929        family: &crate::engine::PartitionedSourceFormulaFamily,
5930    ) -> Result<
5931        crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
5932        SourceFamilyPreparationError,
5933    > {
5934        family
5935            .validate(&self.workbook_load_limits)
5936            .map_err(SourceFamilyPreparationError::contract)?;
5937        let formula = if family.template_text.starts_with('=') {
5938            family.template_text.to_string()
5939        } else {
5940            format!("={}", family.template_text)
5941        };
5942        let ast = formualizer_parse::parser::parse(&formula)
5943            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
5944        let mut requests = Vec::new();
5945        Self::collect_planning_function_requests(&ast, &mut requests);
5946        let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
5947            &self.resolver,
5948            requests,
5949        )
5950        .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5951        let sheet_id = self.graph.sheet_id_mut(sheet_name);
5952        let domains: Vec<_> = family
5953            .fragments
5954            .iter()
5955            .copied()
5956            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
5957            .collect();
5958        for domain in &domains {
5959            validate_anchor_once_shadow_relocation(
5960                &ast,
5961                family.template_origin0.row,
5962                family.template_origin0.col,
5963                domain,
5964                &snapshot,
5965            )
5966            .map_err(SourceFamilyPreparationError::ast)?;
5967        }
5968        let ast_id = self.intern_formula_ast(&ast);
5969        let placement = CellRef::new(
5970            sheet_id,
5971            Coord::from_excel(
5972                family.template_origin0.row + 1,
5973                family.template_origin0.col + 1,
5974                true,
5975                true,
5976            ),
5977        );
5978        let ingested = self
5979            .graph
5980            .ingest_pipeline(&snapshot)
5981            .enable_function_semantics()
5982            .ingest_formula(
5983                FormulaAstInput::RawArena(ast_id),
5984                placement,
5985                Some(family.template_text.clone()),
5986            )
5987            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
5988        let function_semantics_used = ingested
5989            .labels
5990            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
5991        let name_assumptions = self
5992            .graph
5993            .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
5994            .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
5995        let candidate = FormulaPlacementCandidate::new(
5996            sheet_id,
5997            family.template_origin0.row,
5998            family.template_origin0.col,
5999            ingested.ast_id,
6000            Some(family.template_text.clone()),
6001        );
6002        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6003            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6004        let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
6005            total
6006                .checked_add(domain.cell_count())
6007                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
6008        })?;
6009        let mut placements = Vec::with_capacity(domains.len());
6010        for domain in domains {
6011            let member_count = domain.cell_count();
6012            placements.push(
6013                prepare_anchor_once_fragment(
6014                    candidate.clone(),
6015                    analysis.clone(),
6016                    domain,
6017                    member_count,
6018                    direct_cells,
6019                )
6020                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
6021            );
6022        }
6023        let fallback_cells = family.legacy_members.shared_member_count() as u64;
6024        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6025            return Err(SourceFamilyPreparationError::analysis(
6026                "PartitionMemberCountMismatch",
6027            ));
6028        }
6029        Ok(
6030            crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
6031                engine_token: Arc::clone(&self.source_formula_token),
6032                sheet_id,
6033                sheet_name: Arc::from(sheet_name),
6034                source: family.clone(),
6035                placements,
6036                function_semantic_epoch: snapshot.epoch(),
6037                function_provider_revision: snapshot.provider_revision(),
6038                function_semantics_used,
6039                name_assumptions,
6040                direct_cells,
6041            },
6042        )
6043    }
6044
6045    fn prepare_partitioned_source_formula_family_shadow(
6046        &mut self,
6047        sheet_id: SheetId,
6048        family: &crate::engine::PartitionedSourceFormulaFamily,
6049    ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
6050        family
6051            .validate(&self.workbook_load_limits)
6052            .map_err(SourceFamilyPreparationError::contract)?;
6053        let formula = if family.template_text.starts_with('=') {
6054            family.template_text.to_string()
6055        } else {
6056            format!("={}", family.template_text)
6057        };
6058        let ast = formualizer_parse::parser::parse(&formula)
6059            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6060        let domains: Vec<_> = family
6061            .fragments
6062            .iter()
6063            .copied()
6064            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6065            .collect();
6066        for domain in &domains {
6067            validate_anchor_once_shadow_relocation(
6068                &ast,
6069                family.template_origin0.row,
6070                family.template_origin0.col,
6071                domain,
6072                &self.resolver,
6073            )
6074            .map_err(SourceFamilyPreparationError::ast)?;
6075        }
6076        let ast_id = self.intern_formula_ast(&ast);
6077        let placement = CellRef::new(
6078            sheet_id,
6079            Coord::from_excel(
6080                family.template_origin0.row + 1,
6081                family.template_origin0.col + 1,
6082                true,
6083                true,
6084            ),
6085        );
6086        let ingested = self
6087            .ingest_pipeline()
6088            .enable_function_semantics()
6089            .ingest_formula(
6090                FormulaAstInput::RawArena(ast_id),
6091                placement,
6092                Some(family.template_text.clone()),
6093            )
6094            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6095        let function_semantics_used = ingested
6096            .labels
6097            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6098        let candidate = FormulaPlacementCandidate::new(
6099            sheet_id,
6100            family.template_origin0.row,
6101            family.template_origin0.col,
6102            ingested.ast_id,
6103            Some(family.template_text.clone()),
6104        );
6105        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6106            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6107        let fragment_count = domains.len() as u64;
6108        let mut direct_cells = 0u64;
6109        for domain in domains {
6110            let member_count = domain.cell_count();
6111            validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6112                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6113            direct_cells = direct_cells
6114                .checked_add(member_count)
6115                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6116        }
6117        let fallback_cells = family.legacy_members.shared_member_count() as u64;
6118        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6119            return Err(SourceFamilyPreparationError::analysis(
6120                "PartitionMemberCountMismatch",
6121            ));
6122        }
6123        Ok(PreparedPartitionShadow {
6124            fragment_count,
6125            direct_cells,
6126            fallback_cells,
6127            function_semantics_used,
6128        })
6129    }
6130
6131    fn analyze_partitioned_formula_plane_shadow(
6132        &mut self,
6133        batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6134    ) -> FormulaIngestReport {
6135        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6136        for (sheet_name, families) in batches {
6137            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6138            for family in families {
6139                report.shadow_candidate_cells = report
6140                    .shadow_candidate_cells
6141                    .saturating_add(family.surviving_member_count);
6142                match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6143                    Ok(prepared) => {
6144                        let fragment_count = prepared.fragment_count;
6145                        let descendants = family.surviving_member_count.saturating_sub(1);
6146                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6147                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6148                        report.source_anchor_analyses =
6149                            report.source_anchor_analyses.saturating_add(1);
6150                        report.source_partition_analyses_reused = report
6151                            .source_partition_analyses_reused
6152                            .saturating_add(fragment_count.saturating_sub(1));
6153                        report.source_partitioned_families_prepared = report
6154                            .source_partitioned_families_prepared
6155                            .saturating_add(1);
6156                        report.source_partition_fragments_prepared = report
6157                            .source_partition_fragments_prepared
6158                            .saturating_add(fragment_count);
6159                        report.source_partition_span_cells_prepared = report
6160                            .source_partition_span_cells_prepared
6161                            .saturating_add(prepared.direct_cells);
6162                        report.source_partition_fallback_cells = report
6163                            .source_partition_fallback_cells
6164                            .saturating_add(prepared.fallback_cells);
6165                        report.shadow_accepted_span_cells = report
6166                            .shadow_accepted_span_cells
6167                            .saturating_add(prepared.direct_cells);
6168                        report.shadow_fallback_cells = report
6169                            .shadow_fallback_cells
6170                            .saturating_add(prepared.fallback_cells);
6171                        report.source_descendant_strings_avoided = report
6172                            .source_descendant_strings_avoided
6173                            .saturating_add(descendants);
6174                        report.source_descendant_events_avoided = report
6175                            .source_descendant_events_avoided
6176                            .saturating_add(descendants);
6177                        report.source_descendant_analyses_avoided = report
6178                            .source_descendant_analyses_avoided
6179                            .saturating_add(descendants);
6180                        report.graph_formula_vertices_avoided_shadow = report
6181                            .graph_formula_vertices_avoided_shadow
6182                            .saturating_add(prepared.direct_cells);
6183                        report.ast_roots_avoided_shadow = report
6184                            .ast_roots_avoided_shadow
6185                            .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6186                        report.edge_rows_avoided_shadow = report
6187                            .edge_rows_avoided_shadow
6188                            .saturating_add(prepared.direct_cells);
6189                        if prepared.function_semantics_used {
6190                            report.source_partition_function_semantics =
6191                                report.source_partition_function_semantics.saturating_add(1);
6192                        }
6193                    }
6194                    Err(error) => {
6195                        if error.parse_attempted {
6196                            report.source_anchor_parses =
6197                                report.source_anchor_parses.saturating_add(1);
6198                        }
6199                        if error.ast_created {
6200                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6201                        }
6202                        if error.analysis_created {
6203                            report.source_anchor_analyses =
6204                                report.source_anchor_analyses.saturating_add(1);
6205                        }
6206                        report.shadow_fallback_cells = report
6207                            .shadow_fallback_cells
6208                            .saturating_add(family.surviving_member_count);
6209                        report.source_partitioned_families_rejected = report
6210                            .source_partitioned_families_rejected
6211                            .saturating_add(1);
6212                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
6213                    }
6214                }
6215            }
6216        }
6217        report
6218    }
6219
6220    fn analyze_compressed_formula_plane_shadow(
6221        &mut self,
6222        batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6223    ) -> FormulaIngestReport {
6224        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6225        let _active_epoch = self.graph.formula_authority().plane.epoch();
6226
6227        for (sheet_name, families) in batches {
6228            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6229            for family in families {
6230                let descendants = family.member_count.saturating_sub(1);
6231                report.shadow_candidate_cells = report
6232                    .shadow_candidate_cells
6233                    .saturating_add(family.member_count);
6234
6235                match self.prepare_source_formula_family(sheet_id, family, true, None) {
6236                    Ok(_) => {
6237                        report.source_descendant_strings_avoided = report
6238                            .source_descendant_strings_avoided
6239                            .saturating_add(descendants);
6240                        report.source_descendant_events_avoided = report
6241                            .source_descendant_events_avoided
6242                            .saturating_add(descendants);
6243                        report.source_descendant_analyses_avoided = report
6244                            .source_descendant_analyses_avoided
6245                            .saturating_add(descendants);
6246                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6247                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6248                        report.source_anchor_analyses =
6249                            report.source_anchor_analyses.saturating_add(1);
6250                        report.shadow_accepted_span_cells = report
6251                            .shadow_accepted_span_cells
6252                            .saturating_add(family.member_count);
6253                        report.source_compressed_families_prepared =
6254                            report.source_compressed_families_prepared.saturating_add(1);
6255                        report.source_compressed_cells_prepared = report
6256                            .source_compressed_cells_prepared
6257                            .saturating_add(family.member_count);
6258                        report.graph_formula_vertices_avoided_shadow = report
6259                            .graph_formula_vertices_avoided_shadow
6260                            .saturating_add(family.member_count);
6261                        report.ast_roots_avoided_shadow =
6262                            report.ast_roots_avoided_shadow.saturating_add(descendants);
6263                        report.edge_rows_avoided_shadow = report
6264                            .edge_rows_avoided_shadow
6265                            .saturating_add(family.member_count);
6266                    }
6267                    Err(error) => {
6268                        if error.parse_attempted {
6269                            report.source_anchor_parses =
6270                                report.source_anchor_parses.saturating_add(1);
6271                        }
6272                        if error.ast_created {
6273                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6274                        }
6275                        if error.analysis_created {
6276                            report.source_anchor_analyses =
6277                                report.source_anchor_analyses.saturating_add(1);
6278                        }
6279                        report.shadow_fallback_cells = report
6280                            .shadow_fallback_cells
6281                            .saturating_add(family.member_count);
6282                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
6283                    }
6284                }
6285            }
6286        }
6287        report
6288    }
6289
6290    fn analyze_formula_plane_shadow_candidates(
6291        &mut self,
6292        batches: &[FormulaIngestBatch],
6293    ) -> FormulaIngestReport {
6294        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6295        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6296
6297        // Touch graph-owned authority deliberately: Tranche 3 shadow analysis uses
6298        // scratch state, but FormulaPlane ownership now lives on DependencyGraph.
6299        let _active_epoch = self.graph.formula_authority().plane.epoch();
6300
6301        let batch_sheet_ids: Vec<SheetId> = batches
6302            .iter()
6303            .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
6304            .collect();
6305        let mut groups: BTreeMap<
6306            (SheetId, u64, u32),
6307            Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
6308        > = BTreeMap::new();
6309        {
6310            let mut pipeline = self.ingest_pipeline();
6311            for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
6312                for record in &batch.formulas {
6313                    if record.row == 0 || record.col == 0 {
6314                        report.shadow_candidate_cells =
6315                            report.shadow_candidate_cells.saturating_add(1);
6316                        report.shadow_fallback_cells =
6317                            report.shadow_fallback_cells.saturating_add(1);
6318                        Self::record_shadow_fallback_reason(
6319                            &mut report,
6320                            PlacementFallbackReason::UnsupportedShapeOrGaps,
6321                            1,
6322                        );
6323                        continue;
6324                    }
6325
6326                    let placement = CellRef::new(
6327                        sheet_id,
6328                        Coord::from_excel(record.row, record.col, true, true),
6329                    );
6330                    let ingested = match pipeline.ingest_formula(
6331                        FormulaAstInput::RawArena(record.ast_id),
6332                        placement,
6333                        record.formula_text.clone(),
6334                    ) {
6335                        Ok(ingested) => ingested,
6336                        Err(_) => {
6337                            report.shadow_candidate_cells =
6338                                report.shadow_candidate_cells.saturating_add(1);
6339                            report.shadow_fallback_cells =
6340                                report.shadow_fallback_cells.saturating_add(1);
6341                            Self::record_shadow_fallback_reason(
6342                                &mut report,
6343                                PlacementFallbackReason::UnsupportedCanonicalTemplate,
6344                                1,
6345                            );
6346                            continue;
6347                        }
6348                    };
6349                    let candidate = FormulaPlacementCandidate::new(
6350                        sheet_id,
6351                        record.row - 1,
6352                        record.col - 1,
6353                        ingested.ast_id,
6354                        record.formula_text.clone(),
6355                    );
6356                    let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
6357                        Ok(analysis) => analysis,
6358                        Err(reason) => {
6359                            report.shadow_candidate_cells =
6360                                report.shadow_candidate_cells.saturating_add(1);
6361                            report.shadow_fallback_cells =
6362                                report.shadow_fallback_cells.saturating_add(1);
6363                            Self::record_shadow_fallback_reason(&mut report, reason, 1);
6364                            continue;
6365                        }
6366                    };
6367                    groups
6368                        .entry((
6369                            sheet_id,
6370                            ingested.parameterized_canonical_hash,
6371                            candidate.col,
6372                        ))
6373                        .or_default()
6374                        .push((candidate, analysis));
6375                }
6376            }
6377        }
6378
6379        let mut scratch_plane = FormulaPlane::default();
6380        for entries in groups.into_values() {
6381            let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
6382            for (component, component_analyses) in
6383                Self::split_candidate_components_with_analyses(candidates, analyses)
6384            {
6385                let placement_report = place_candidate_family_with_analyses(
6386                    &mut scratch_plane,
6387                    component,
6388                    component_analyses,
6389                );
6390                let counters = placement_report.counters;
6391                report.shadow_candidate_cells = report
6392                    .shadow_candidate_cells
6393                    .saturating_add(counters.formula_cells_seen);
6394                report.shadow_accepted_span_cells = report
6395                    .shadow_accepted_span_cells
6396                    .saturating_add(counters.accepted_span_cells);
6397                report.shadow_fallback_cells = report
6398                    .shadow_fallback_cells
6399                    .saturating_add(counters.legacy_cells);
6400                report.shadow_templates_interned = report
6401                    .shadow_templates_interned
6402                    .saturating_add(counters.templates_interned);
6403                report.shadow_spans_created = report
6404                    .shadow_spans_created
6405                    .saturating_add(counters.spans_created);
6406                report.graph_formula_vertices_avoided_shadow = report
6407                    .graph_formula_vertices_avoided_shadow
6408                    .saturating_add(counters.formula_vertices_avoided);
6409                report.ast_roots_avoided_shadow = report
6410                    .ast_roots_avoided_shadow
6411                    .saturating_add(counters.ast_roots_avoided);
6412                report.edge_rows_avoided_shadow = report
6413                    .edge_rows_avoided_shadow
6414                    .saturating_add(counters.edge_rows_avoided);
6415                for (reason, count) in counters.fallback_reasons {
6416                    Self::record_shadow_fallback_reason(&mut report, reason, count);
6417                }
6418            }
6419        }
6420        report
6421    }
6422
6423    fn record_shadow_fallback_reason(
6424        report: &mut FormulaIngestReport,
6425        reason: PlacementFallbackReason,
6426        count: u64,
6427    ) {
6428        *report
6429            .fallback_reasons
6430            .entry(format!("{reason:?}"))
6431            .or_default() += count;
6432    }
6433
6434    fn analyze_formula_plane_authoritative_ingest(
6435        &mut self,
6436        batches: &[FormulaIngestBatch],
6437    ) -> (
6438        FormulaIngestReport,
6439        Vec<FormulaIngestBatch>,
6440        PlannedFormulaMaterialize,
6441    ) {
6442        let existing_span_refs = self
6443            .graph
6444            .formula_authority()
6445            .active_span_refs()
6446            .into_iter()
6447            .collect::<rustc_hash::FxHashSet<_>>();
6448        let mut report =
6449            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
6450        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
6451
6452        let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
6453        let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
6454        let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
6455
6456        for batch in batches {
6457            let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
6458            for record in &batch.formulas {
6459                if record.row == 0 || record.col == 0 {
6460                    report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
6461                    report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
6462                    Self::record_shadow_fallback_reason(
6463                        &mut report,
6464                        PlacementFallbackReason::UnsupportedShapeOrGaps,
6465                        1,
6466                    );
6467                    fallback
6468                        .entry(batch.sheet_name.clone())
6469                        .or_default()
6470                        .push(record.clone());
6471                    continue;
6472                }
6473
6474                pending_candidates.push((
6475                    batch.sheet_name.clone(),
6476                    FormulaPlacementCandidate::new(
6477                        sheet_id,
6478                        record.row - 1,
6479                        record.col - 1,
6480                        record.ast_id,
6481                        record.formula_text.clone(),
6482                    ),
6483                ));
6484            }
6485        }
6486
6487        let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
6488        let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
6489            (0..pending_candidates.len()).map(|_| None).collect();
6490        let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
6491            (0..pending_candidates.len()).map(|_| None).collect();
6492        {
6493            let mut pipeline = self.ingest_pipeline();
6494            for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
6495                let placement = CellRef::new(
6496                    candidate.sheet_id,
6497                    Coord::from_excel(
6498                        candidate.row.saturating_add(1),
6499                        candidate.col.saturating_add(1),
6500                        true,
6501                        true,
6502                    ),
6503                );
6504                let ingested = pipeline.ingest_formula(
6505                    FormulaAstInput::RawArena(candidate.ast_id),
6506                    placement,
6507                    candidate.formula_text.clone(),
6508                );
6509                match ingested {
6510                    Ok(ingested) => {
6511                        candidate.ast_id = ingested.ast_id;
6512                        let dep_plan = ingested.dep_plan.clone();
6513                        match CandidateAnalysis::from_ingested(candidate, &ingested) {
6514                            Ok(analysis) => {
6515                                analyses_by_index[idx] = Some(analysis);
6516                                plans_by_index[idx] = Some(dep_plan);
6517                            }
6518                            Err(reason) => {
6519                                report.shadow_candidate_cells =
6520                                    report.shadow_candidate_cells.saturating_add(1);
6521                                report.shadow_fallback_cells =
6522                                    report.shadow_fallback_cells.saturating_add(1);
6523                                Self::record_shadow_fallback_reason(&mut report, reason, 1);
6524                                planned_fallback
6525                                    .entry(sheet_name.clone())
6526                                    .or_default()
6527                                    .push((
6528                                        candidate.row.saturating_add(1),
6529                                        candidate.col.saturating_add(1),
6530                                        candidate.ast_id,
6531                                        dep_plan,
6532                                    ));
6533                            }
6534                        }
6535                    }
6536                    Err(_) => {
6537                        let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
6538                        report.shadow_candidate_cells =
6539                            report.shadow_candidate_cells.saturating_add(1);
6540                        report.shadow_fallback_cells =
6541                            report.shadow_fallback_cells.saturating_add(1);
6542                        Self::record_shadow_fallback_reason(&mut report, reason, 1);
6543                        fallback.entry(sheet_name.clone()).or_default().push(
6544                            FormulaIngestRecord::new(
6545                                candidate.row.saturating_add(1),
6546                                candidate.col.saturating_add(1),
6547                                candidate.ast_id,
6548                                candidate.formula_text.clone(),
6549                            ),
6550                        );
6551                    }
6552                }
6553            }
6554        }
6555
6556        for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
6557            if analyses_by_index[idx].is_none() {
6558                continue;
6559            }
6560            let canonical_hash = analyses_by_index[idx]
6561                .as_ref()
6562                .expect("remaining candidate has an analysis")
6563                .parameterized_canonical_hash();
6564            groups
6565                .entry((candidate.sheet_id, canonical_hash, candidate.col))
6566                .or_default()
6567                .push(idx);
6568        }
6569
6570        for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
6571            let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
6572            let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
6573            for idx in &candidate_indices {
6574                // Each candidate index belongs to exactly one group, so the
6575                // plan row can be moved out instead of deep-cloned.
6576                if let Some(plan) = plans_by_index[*idx].take() {
6577                    let candidate = &pending_candidates[*idx].1;
6578                    plans_by_coord
6579                        .entry((candidate.row, candidate.col))
6580                        .or_default()
6581                        .push(plan);
6582                }
6583            }
6584            let candidates: Vec<_> = candidate_indices
6585                .iter()
6586                .map(|idx| pending_candidates[*idx].1.clone())
6587                .collect();
6588            let components = Self::split_shadow_candidate_components(candidates);
6589            let analyzed_components =
6590                if components.len() == 1 && components[0].len() == candidate_indices.len() {
6591                    let component = components.into_iter().next().expect("one component");
6592                    let component_analyses = candidate_indices
6593                        .iter()
6594                        .map(|idx| {
6595                            analyses_by_index[*idx]
6596                                .take()
6597                                .expect("candidate analysis must be used once")
6598                        })
6599                        .collect();
6600                    vec![(component, component_analyses)]
6601                } else {
6602                    let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6603                    for idx in candidate_indices.iter().rev() {
6604                        let candidate = &pending_candidates[*idx].1;
6605                        indices_by_coord
6606                            .entry((candidate.row, candidate.col))
6607                            .or_default()
6608                            .push(*idx);
6609                    }
6610
6611                    components
6612                        .into_iter()
6613                        .map(|component| {
6614                            let mut component_analyses = Vec::with_capacity(component.len());
6615                            for candidate in &component {
6616                                let idx = indices_by_coord
6617                                    .get_mut(&(candidate.row, candidate.col))
6618                                    .and_then(Vec::pop)
6619                                    .expect("component candidate must have a precomputed analysis");
6620                                component_analyses.push(
6621                                    analyses_by_index[idx]
6622                                        .take()
6623                                        .expect("candidate analysis must be used once"),
6624                                );
6625                            }
6626                            (component, component_analyses)
6627                        })
6628                        .collect()
6629                };
6630
6631            for (component, component_analyses) in analyzed_components {
6632                for (component, component_analyses) in
6633                    split_candidate_affine_literal_runs(component, component_analyses)
6634                {
6635                    let placement_report = {
6636                        let authority = self.graph.formula_authority_mut();
6637                        place_candidate_family_with_analyses(
6638                            &mut authority.plane,
6639                            component.clone(),
6640                            component_analyses,
6641                        )
6642                    };
6643                    Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
6644
6645                    // Index candidates by placement once per component. The
6646                    // previous per-result linear `find` made this fallback
6647                    // mapping O(N²) for rejected families (e.g. an N-cell
6648                    // chain rejected with `InternalDependency`), dominating
6649                    // first-eval ingest cost on large rejected families.
6650                    // First insert wins, matching the old `Iterator::find`
6651                    // semantics for duplicate placements.
6652                    let mut candidate_by_placement: FxHashMap<
6653                        crate::formula_plane::runtime::PlacementCoord,
6654                        &FormulaPlacementCandidate,
6655                    > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
6656                    for candidate in &component {
6657                        candidate_by_placement
6658                            .entry(candidate.placement())
6659                            .or_insert(candidate);
6660                    }
6661                    for result in &placement_report.results {
6662                        let FormulaPlacementResult::Legacy { placement, .. } = result else {
6663                            continue;
6664                        };
6665                        if let Some(&candidate) = candidate_by_placement.get(placement) {
6666                            let plan = plans_by_coord
6667                                .get_mut(&(candidate.row, candidate.col))
6668                                .and_then(Vec::pop);
6669                            if let Some(plan) = plan {
6670                                planned_fallback
6671                                    .entry(sheet_name.clone())
6672                                    .or_default()
6673                                    .push((
6674                                        candidate.row.saturating_add(1),
6675                                        candidate.col.saturating_add(1),
6676                                        candidate.ast_id,
6677                                        plan,
6678                                    ));
6679                            } else {
6680                                fallback.entry(sheet_name.clone()).or_default().push(
6681                                    FormulaIngestRecord::new(
6682                                        candidate.row.saturating_add(1),
6683                                        candidate.col.saturating_add(1),
6684                                        candidate.ast_id,
6685                                        candidate.formula_text.clone(),
6686                                    ),
6687                                );
6688                            }
6689                        }
6690                    }
6691                }
6692            }
6693        }
6694
6695        let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
6696        let new_span_refs = self
6697            .graph
6698            .formula_authority()
6699            .active_span_refs()
6700            .into_iter()
6701            .filter(|span_ref| !existing_span_refs.contains(span_ref))
6702            .collect::<Vec<_>>();
6703        self.graph
6704            .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
6705
6706        let fallback_batches = fallback
6707            .into_iter()
6708            .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
6709            .collect();
6710        (report, fallback_batches, planned_fallback)
6711    }
6712
6713    fn accumulate_formula_plane_placement_report(
6714        report: &mut FormulaIngestReport,
6715        placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
6716    ) {
6717        let counters = &placement_report.counters;
6718        report.shadow_candidate_cells = report
6719            .shadow_candidate_cells
6720            .saturating_add(counters.formula_cells_seen);
6721        report.shadow_accepted_span_cells = report
6722            .shadow_accepted_span_cells
6723            .saturating_add(counters.accepted_span_cells);
6724        report.shadow_fallback_cells = report
6725            .shadow_fallback_cells
6726            .saturating_add(counters.legacy_cells);
6727        report.shadow_templates_interned = report
6728            .shadow_templates_interned
6729            .saturating_add(counters.templates_interned);
6730        report.shadow_spans_created = report
6731            .shadow_spans_created
6732            .saturating_add(counters.spans_created);
6733        report.graph_formula_vertices_avoided_shadow = report
6734            .graph_formula_vertices_avoided_shadow
6735            .saturating_add(counters.formula_vertices_avoided);
6736        report.ast_roots_avoided_shadow = report
6737            .ast_roots_avoided_shadow
6738            .saturating_add(counters.ast_roots_avoided);
6739        report.edge_rows_avoided_shadow = report
6740            .edge_rows_avoided_shadow
6741            .saturating_add(counters.edge_rows_avoided);
6742        for (reason, count) in &counters.fallback_reasons {
6743            Self::record_shadow_fallback_reason(report, *reason, *count);
6744        }
6745    }
6746
6747    fn split_candidate_components_with_analyses(
6748        candidates: Vec<FormulaPlacementCandidate>,
6749        mut analyses: Vec<CandidateAnalysis>,
6750    ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
6751        let components = Self::split_shadow_candidate_components(candidates.clone());
6752        let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
6753        for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
6754            analysis_by_coord
6755                .entry((candidate.row, candidate.col))
6756                .or_default()
6757                .push(analysis);
6758        }
6759        components
6760            .into_iter()
6761            .flat_map(|component| {
6762                let mut component_analyses = Vec::with_capacity(component.len());
6763                for candidate in &component {
6764                    let analysis = analysis_by_coord
6765                        .get_mut(&(candidate.row, candidate.col))
6766                        .and_then(Vec::pop)
6767                        .expect("component candidate must have a precomputed analysis");
6768                    component_analyses.push(analysis);
6769                }
6770                split_candidate_affine_literal_runs(component, component_analyses)
6771            })
6772            .collect()
6773    }
6774
6775    fn split_shadow_candidate_components(
6776        candidates: Vec<FormulaPlacementCandidate>,
6777    ) -> Vec<Vec<FormulaPlacementCandidate>> {
6778        if candidates.len() <= 1 {
6779            return vec![candidates];
6780        }
6781
6782        // Fast path: candidates already ordered as one contiguous
6783        // single-column (or single-row) run form exactly one 4-connected
6784        // component in their existing (row, col) order; skip the BFS.
6785        let is_row_run = candidates.windows(2).all(|w| {
6786            w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
6787        });
6788        let is_col_run = candidates.windows(2).all(|w| {
6789            w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
6790        });
6791        if is_row_run || is_col_run {
6792            return vec![candidates];
6793        }
6794
6795        let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
6796        for (idx, candidate) in candidates.iter().enumerate() {
6797            coord_to_indices
6798                .entry((candidate.row, candidate.col))
6799                .or_default()
6800                .push(idx);
6801        }
6802
6803        let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
6804        let mut components = Vec::new();
6805        while let Some(&start) = remaining.iter().next() {
6806            remaining.remove(&start);
6807            let mut queue = VecDeque::from([start]);
6808            let mut component_indices = Vec::new();
6809
6810            while let Some(idx) = queue.pop_front() {
6811                component_indices.push(idx);
6812                let candidate = &candidates[idx];
6813                let mut neighbor_coords = Vec::with_capacity(5);
6814                neighbor_coords.push((candidate.row, candidate.col));
6815                if let Some(row) = candidate.row.checked_sub(1) {
6816                    neighbor_coords.push((row, candidate.col));
6817                }
6818                neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
6819                if let Some(col) = candidate.col.checked_sub(1) {
6820                    neighbor_coords.push((candidate.row, col));
6821                }
6822                neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
6823
6824                for coord in neighbor_coords {
6825                    if let Some(indices) = coord_to_indices.get(&coord) {
6826                        for &neighbor in indices {
6827                            if remaining.remove(&neighbor) {
6828                                queue.push_back(neighbor);
6829                            }
6830                        }
6831                    }
6832                }
6833            }
6834
6835            component_indices.sort_by_key(|idx| {
6836                let candidate = &candidates[*idx];
6837                (candidate.row, candidate.col, *idx)
6838            });
6839            components.push(
6840                component_indices
6841                    .into_iter()
6842                    .map(|idx| candidates[idx].clone())
6843                    .collect(),
6844            );
6845        }
6846
6847        components
6848    }
6849
6850    fn collect_planning_function_requests(
6851        ast: &ASTNode,
6852        requests: &mut Vec<(String, String, usize)>,
6853    ) {
6854        match &ast.node_type {
6855            ASTNodeType::Function { name, args } => {
6856                requests.push((String::new(), name.clone(), args.len()));
6857                for arg in args {
6858                    Self::collect_planning_function_requests(arg, requests);
6859                }
6860            }
6861            ASTNodeType::BinaryOp { left, right, .. } => {
6862                Self::collect_planning_function_requests(left, requests);
6863                Self::collect_planning_function_requests(right, requests);
6864            }
6865            ASTNodeType::UnaryOp { expr, .. } => {
6866                Self::collect_planning_function_requests(expr, requests);
6867            }
6868            ASTNodeType::Call { callee, args } => {
6869                Self::collect_planning_function_requests(callee, requests);
6870                for arg in args {
6871                    Self::collect_planning_function_requests(arg, requests);
6872                }
6873            }
6874            ASTNodeType::Array(rows) => {
6875                for cell in rows.iter().flatten() {
6876                    Self::collect_planning_function_requests(cell, requests);
6877                }
6878            }
6879            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
6880        }
6881    }
6882
6883    fn prepared_placement_function_semantics_changed(
6884        &self,
6885        semantic_epoch: u64,
6886        prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
6887        guard: &crate::function_registry::SemanticEpochReadGuard,
6888    ) -> bool {
6889        if semantic_epoch == guard.epoch() {
6890            return false;
6891        }
6892        let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6893        let Some(ast) = self
6894            .graph
6895            .data_store()
6896            .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6897        else {
6898            // Missing planning evidence must never turn a semantic change into reuse.
6899            return true;
6900        };
6901        let mut requests = Vec::new();
6902        Self::collect_planning_function_requests(&ast, &mut requests);
6903        guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
6904    }
6905
6906    fn prepared_function_semantics_changed(
6907        &self,
6908        preparation: &crate::engine::FormulaCompressedPreparation,
6909        guard: &crate::function_registry::SemanticEpochReadGuard,
6910    ) -> bool {
6911        if preparation.function_semantic_epoch == guard.epoch() {
6912            return false;
6913        }
6914
6915        let mut requests = Vec::new();
6916        for (_, _, prepared) in &preparation.prepared {
6917            let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
6918            let Some(ast) = self
6919                .graph
6920                .data_store()
6921                .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
6922            else {
6923                return true;
6924            };
6925            Self::collect_planning_function_requests(&ast, &mut requests);
6926        }
6927        guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
6928    }
6929
6930    pub(crate) fn prepare_source_formula_families(
6931        &mut self,
6932        sheet_name: &str,
6933        families: &[crate::engine::SourceFormulaFamily],
6934    ) -> crate::engine::FormulaCompressedPreparation {
6935        let mut preparation = crate::engine::FormulaCompressedPreparation {
6936            engine_token: Arc::clone(&self.source_formula_token),
6937            function_semantic_epoch: crate::function_registry::semantic_epoch(),
6938            function_provider_revision: None,
6939            function_semantics_used: false,
6940            sheet_name: Arc::from(sheet_name),
6941            prepared: Vec::new(),
6942            rejected: BTreeMap::new(),
6943            fragmented: Vec::new(),
6944            fragmented_sources: BTreeMap::new(),
6945            eager_replay: Vec::new(),
6946            preparation_spool_replays: 0,
6947            clean_rejected_anchor_counts: [0; 3],
6948            fragmented_rejected_anchor_counts: [0; 3],
6949            exact_replay: None,
6950            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
6951        };
6952        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
6953            return preparation;
6954        }
6955
6956        let mut requests = Vec::new();
6957        for family in families {
6958            let formula = if family.anchor_text.starts_with('=') {
6959                family.anchor_text.to_string()
6960            } else {
6961                format!("={}", family.anchor_text)
6962            };
6963            if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
6964                Self::collect_planning_function_requests(&ast, &mut requests);
6965            }
6966        }
6967        requests.sort();
6968        requests.dedup();
6969        let snapshot =
6970            match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6971                &self.resolver,
6972                requests,
6973            ) {
6974                Ok(snapshot) => snapshot,
6975                Err(error) => {
6976                    let reason = match error {
6977                        crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
6978                            "FunctionSemanticSnapshotUnavailable"
6979                        }
6980                        crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
6981                            "FunctionProviderRevisionUnavailable"
6982                        }
6983                    };
6984                    for family in families {
6985                        preparation
6986                            .rejected
6987                            .insert(family.source_id, reason.to_string());
6988                    }
6989                    return preparation;
6990                }
6991            };
6992        preparation.function_semantic_epoch = snapshot.epoch();
6993        preparation.function_provider_revision = snapshot.provider_revision();
6994
6995        let sheet_id = self.graph.sheet_id_mut(sheet_name);
6996        for family in families {
6997            #[cfg(test)]
6998            let force_replay = self.force_source_family_fallback
6999                || cfg!(feature = "benchmark_internal")
7000                    && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7001            #[cfg(all(feature = "benchmark_internal", not(test)))]
7002            let force_replay =
7003                std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7004            #[cfg(any(test, feature = "benchmark_internal"))]
7005            if force_replay {
7006                preparation
7007                    .rejected
7008                    .insert(family.source_id, "ForcedReplay".to_string());
7009                continue;
7010            }
7011            match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
7012                Ok((prepared, function_semantics_used)) => {
7013                    preparation.function_semantics_used |= function_semantics_used;
7014                    preparation
7015                        .replay_disposition
7016                        .set_family_direct(family.source_id);
7017                    preparation
7018                        .prepared
7019                        .push((family.source_id, family.source_order, prepared));
7020                }
7021                Err(error) => {
7022                    preparation.clean_rejected_anchor_counts[0] = preparation
7023                        .clean_rejected_anchor_counts[0]
7024                        .saturating_add(u64::from(error.parse_attempted));
7025                    preparation.clean_rejected_anchor_counts[1] = preparation
7026                        .clean_rejected_anchor_counts[1]
7027                        .saturating_add(u64::from(error.ast_created));
7028                    preparation.clean_rejected_anchor_counts[2] = preparation
7029                        .clean_rejected_anchor_counts[2]
7030                        .saturating_add(u64::from(error.analysis_created));
7031                    preparation.rejected.insert(family.source_id, error.reason);
7032                }
7033            }
7034        }
7035        preparation
7036    }
7037
7038    fn prepare_source_formula_proposals(
7039        &mut self,
7040        sheet_name: &str,
7041        families: &[crate::engine::SourceFormulaFamily],
7042        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7043        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7044        formula_record_count: u64,
7045        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
7046        suppressed: &BTreeSet<(u32, u32)>,
7047    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
7048        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
7049        preparation.exact_replay = Some(Arc::clone(&replay));
7050        preparation
7051            .replay_disposition
7052            .extend_suppressed_excel_coords(suppressed.iter().copied());
7053        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7054            return Ok(preparation);
7055        }
7056        #[cfg(feature = "benchmark_internal")]
7057        let benchmark_forced_replay =
7058            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7059        #[cfg(not(feature = "benchmark_internal"))]
7060        let benchmark_forced_replay = false;
7061        let authority_partitions = if benchmark_forced_replay {
7062            for partition in authority_partitions {
7063                preparation
7064                    .rejected
7065                    .insert(partition.source_id, "ForcedReplay".to_string());
7066            }
7067            &[][..]
7068        } else {
7069            authority_partitions
7070        };
7071        let authority_ids: BTreeSet<_> = authority_partitions
7072            .iter()
7073            .map(|partition| partition.source_id)
7074            .collect();
7075        for partition in replay_partitions {
7076            if !authority_ids.contains(&partition.source_id) {
7077                preparation
7078                    .replay_disposition
7079                    .register_partition(partition, false)
7080                    .map_err(|reason| {
7081                        ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7082                    })?;
7083            }
7084        }
7085        if replay_partitions.is_empty()
7086            && preparation.rejected.is_empty()
7087            && preparation.direct_cell_count() == formula_record_count
7088        {
7089            return Ok(preparation);
7090        }
7091
7092        let mut analyzed = Vec::new();
7093        for source in authority_partitions {
7094            preparation
7095                .fragmented_sources
7096                .insert(source.source_id, source.clone());
7097            match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7098                Ok(prepared) => {
7099                    let mut candidate = preparation.replay_disposition.clone();
7100                    match candidate.register_partition(source, true) {
7101                        Ok(()) => {
7102                            preparation.replay_disposition = candidate;
7103                            analyzed.push((source.clone(), prepared));
7104                        }
7105                        Err(reason) => {
7106                            preparation
7107                                .rejected
7108                                .insert(source.source_id, reason.to_string());
7109                            preparation
7110                                .replay_disposition
7111                                .register_partition(source, false)
7112                                .map_err(|reason| {
7113                                    ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7114                                })?;
7115                        }
7116                    }
7117                }
7118                Err(error) => {
7119                    preparation.fragmented_rejected_anchor_counts[0] = preparation
7120                        .fragmented_rejected_anchor_counts[0]
7121                        .saturating_add(u64::from(error.parse_attempted));
7122                    preparation.fragmented_rejected_anchor_counts[1] = preparation
7123                        .fragmented_rejected_anchor_counts[1]
7124                        .saturating_add(u64::from(error.ast_created));
7125                    preparation.fragmented_rejected_anchor_counts[2] = preparation
7126                        .fragmented_rejected_anchor_counts[2]
7127                        .saturating_add(u64::from(error.analysis_created));
7128                    preparation.rejected.insert(source.source_id, error.reason);
7129                    preparation
7130                        .replay_disposition
7131                        .register_partition(source, false)
7132                        .map_err(|reason| {
7133                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7134                        })?;
7135                }
7136            }
7137        }
7138
7139        let initial_replay = replay
7140            .lock()
7141            .map_err(|_| {
7142                ExcelError::new(ExcelErrorKind::Value)
7143                    .with_message("compressed formula exact replay lock poisoned")
7144            })?
7145            .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7146            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7147        preparation.preparation_spool_replays = 1;
7148        let sheet_id = self.graph.sheet_id_mut(sheet_name);
7149        let mut preparation_failed = false;
7150        for (source, prepared_family) in analyzed {
7151            let result = (|| {
7152                let expected: BTreeMap<_, _> = source
7153                    .legacy_members
7154                    .as_slice()
7155                    .iter()
7156                    .map(|member| (member.coord, member.kind))
7157                    .collect();
7158                let mut actual = BTreeMap::new();
7159                for record in initial_replay
7160                    .iter()
7161                    .filter(|record| record.partition_owner == Some(source.source_id))
7162                {
7163                    let coord = record
7164                        .row
7165                        .checked_sub(1)
7166                        .zip(record.col.checked_sub(1))
7167                        .map(|(row, col)| crate::engine::SourceCoord { row, col })
7168                        .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7169                    let kind = match record.family {
7170                        Some(family) if family == source.source_id => {
7171                            crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7172                        }
7173                        None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7174                        Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7175                    };
7176                    if actual.insert(coord, (kind, record.clone())).is_some() {
7177                        return Err("ExactReplayRecordDuplicate".to_string());
7178                    }
7179                }
7180                let actual_kinds: BTreeMap<_, _> = actual
7181                    .iter()
7182                    .map(|(coord, (kind, _))| (*coord, *kind))
7183                    .collect();
7184                if actual_kinds != expected {
7185                    return Err("ExactReplayLegacySetMismatch".to_string());
7186                }
7187                let mut legacy = Vec::with_capacity(source.legacy_members.len());
7188                for expected in source.legacy_members.as_slice() {
7189                    let record = actual
7190                        .remove(&expected.coord)
7191                        .map(|(_, record)| record)
7192                        .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7193                    legacy.push(
7194                        self.graph
7195                            .analyze_fragmented_exact_replay_record(
7196                                &self.resolver,
7197                                sheet_id,
7198                                source.source_id,
7199                                *expected,
7200                                record,
7201                            )
7202                            .map_err(|error| format!("{error:?}"))?,
7203                    );
7204                }
7205                Ok::<_, String>((prepared_family, legacy))
7206            })();
7207            match result {
7208                Ok((prepared_family, legacy)) => preparation.fragmented.push(
7209                    crate::engine::formula_source::PreparedFragmentedSourceProposal {
7210                        source: source.clone(),
7211                        prepared: prepared_family,
7212                        legacy,
7213                        replay: Arc::clone(&replay),
7214                    },
7215                ),
7216                Err(reason) => {
7217                    preparation_failed = true;
7218                    for count in &mut preparation.fragmented_rejected_anchor_counts {
7219                        *count = count.saturating_add(1);
7220                    }
7221                    preparation.rejected.insert(source.source_id, reason);
7222                    preparation
7223                        .replay_disposition
7224                        .force_family_legacy(source.source_id);
7225                }
7226            }
7227        }
7228
7229        preparation.eager_replay = if preparation_failed {
7230            preparation.preparation_spool_replays = 2;
7231            replay
7232                .lock()
7233                .map_err(|_| {
7234                    ExcelError::new(ExcelErrorKind::Value)
7235                        .with_message("compressed formula exact replay lock poisoned")
7236                })?
7237                .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7238                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7239        } else {
7240            initial_replay
7241        };
7242        Ok(preparation)
7243    }
7244
7245    fn formula_batch_from_exact_replay(
7246        &mut self,
7247        sheet_name: &str,
7248        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7249    ) -> Result<FormulaIngestBatch, ExcelError> {
7250        let mut cache = rustc_hash::FxHashMap::default();
7251        let mut formulas = Vec::new();
7252        for record in replayed {
7253            let key = if record.text.starts_with('=') {
7254                record.text
7255            } else {
7256                format!("={}", record.text)
7257            };
7258            let ast_id = if let Some(cached) = cache.get(&key) {
7259                *cached
7260            } else {
7261                let parsed = match formualizer_parse::parser::parse(&key) {
7262                    Ok(parsed) => parsed,
7263                    Err(error) => {
7264                        let Some(parsed) = self.handle_formula_parse_error(
7265                            sheet_name,
7266                            record.row,
7267                            record.col,
7268                            &key,
7269                            error.to_string(),
7270                        )?
7271                        else {
7272                            continue;
7273                        };
7274                        parsed
7275                    }
7276                };
7277                let ast_id = self.intern_formula_ast(&parsed);
7278                cache.insert(key.clone(), ast_id);
7279                ast_id
7280            };
7281            formulas.push(
7282                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
7283                    .with_source_proof(record.source_order, record.family, record.partition_owner),
7284            );
7285        }
7286        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
7287    }
7288
7289    fn replay_prepared_families_exact_records(
7290        &mut self,
7291        preparation: &crate::engine::FormulaCompressedPreparation,
7292    ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
7293        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7294            ExcelError::new(ExcelErrorKind::Value)
7295                .with_message("stale compressed preparation has no exact replay owner")
7296        })?;
7297        let direct: BTreeSet<_> = preparation
7298            .prepared
7299            .iter()
7300            .map(|(family, _, _)| *family)
7301            .collect();
7302        let mut replay_disposition = preparation.replay_disposition.clone();
7303        for family in &direct {
7304            replay_disposition.force_family_legacy(*family);
7305        }
7306        let replayed = replay
7307            .lock()
7308            .map_err(|_| {
7309                ExcelError::new(ExcelErrorKind::Value)
7310                    .with_message("compressed formula exact replay lock poisoned")
7311            })?
7312            .replay(&replay_disposition)
7313            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7314        let mut records: Vec<_> = replayed
7315            .into_iter()
7316            .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
7317            .collect();
7318        records.sort_by_key(|record| record.source_order);
7319        let expected: u64 = preparation
7320            .prepared
7321            .iter()
7322            .map(|(_, _, prepared)| prepared.member_count)
7323            .sum();
7324        if records.len() as u64 != expected {
7325            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7326                "compressed formula exact replay incomplete: expected {expected}, got {}",
7327                records.len()
7328            )));
7329        }
7330        Ok(records)
7331    }
7332
7333    fn replay_one_prepared_family_exact(
7334        &mut self,
7335        preparation: &crate::engine::FormulaCompressedPreparation,
7336        family: crate::engine::SourceFamilyId,
7337        member_count: u64,
7338    ) -> Result<FormulaIngestBatch, ExcelError> {
7339        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
7340            ExcelError::new(ExcelErrorKind::Value)
7341                .with_message("stale compressed preparation has no exact replay owner")
7342        })?;
7343        let mut replay_disposition = preparation.replay_disposition.clone();
7344        replay_disposition.force_family_legacy(family);
7345        let replayed = replay
7346            .lock()
7347            .map_err(|_| {
7348                ExcelError::new(ExcelErrorKind::Value)
7349                    .with_message("compressed formula exact replay lock poisoned")
7350            })?
7351            .replay(&replay_disposition)
7352            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7353            .into_iter()
7354            .filter(|record| record.family == Some(family));
7355        let batch =
7356            self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
7357        if batch.formulas.len() as u64 != member_count {
7358            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
7359                "compressed formula exact replay incomplete: expected {member_count}, got {}",
7360                batch.formulas.len()
7361            )));
7362        }
7363        Ok(batch)
7364    }
7365
7366    fn exact_replay_record_is_prepared_partition_legacy(
7367        source: &crate::engine::PartitionedSourceFormulaFamily,
7368        record: &crate::engine::DeferredReplayFormula,
7369    ) -> bool {
7370        if record.partition_owner != Some(source.source_id) {
7371            return false;
7372        }
7373        let Some(coord) = record
7374            .row
7375            .checked_sub(1)
7376            .zip(record.col.checked_sub(1))
7377            .map(|(row, col)| crate::engine::SourceCoord { row, col })
7378        else {
7379            return false;
7380        };
7381        source.legacy_members.as_slice().iter().any(|member| {
7382            member.coord == coord
7383                && match member.kind {
7384                    crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
7385                        record.family == Some(source.source_id)
7386                    }
7387                    crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
7388                        record.family.is_none()
7389                    }
7390                }
7391        })
7392    }
7393
7394    fn validate_whole_partition_replay(
7395        source: &crate::engine::PartitionedSourceFormulaFamily,
7396        records: &[crate::engine::DeferredReplayFormula],
7397    ) -> Result<(), ExcelError> {
7398        let mut coordinates = BTreeSet::new();
7399        let mut shared = 0u64;
7400        let mut ordinary = 0u64;
7401        for record in records {
7402            let coord = record
7403                .row
7404                .checked_sub(1)
7405                .zip(record.col.checked_sub(1))
7406                .map(|(row, col)| crate::engine::SourceCoord { row, col })
7407                .ok_or_else(|| {
7408                    ExcelError::new(ExcelErrorKind::Value)
7409                        .with_message("whole-family replay contains an invalid coordinate")
7410                })?;
7411            if !coordinates.insert(coord) {
7412                return Err(ExcelError::new(ExcelErrorKind::Value)
7413                    .with_message("whole-family replay contains a duplicate coordinate"));
7414            }
7415            match (record.family, record.partition_owner) {
7416                (Some(family), Some(owner))
7417                    if family == source.source_id && owner == source.source_id =>
7418                {
7419                    let direct = source.fragments.iter().any(|fragment| {
7420                        let rect = fragment.rect();
7421                        coord.row >= rect.start.row
7422                            && coord.row <= rect.end.row
7423                            && coord.col >= rect.start.col
7424                            && coord.col <= rect.end.col
7425                    });
7426                    let legacy = source.legacy_members.as_slice().iter().any(|member| {
7427                        member.coord == coord
7428                            && member.kind
7429                                == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7430                    });
7431                    if !direct && !legacy {
7432                        return Err(ExcelError::new(ExcelErrorKind::Value)
7433                            .with_message("whole-family replay contains an extra shared member"));
7434                    }
7435                    shared = shared.saturating_add(1);
7436                }
7437                (None, Some(owner)) if owner == source.source_id => {
7438                    if !source.legacy_members.as_slice().iter().any(|member| {
7439                        member.coord == coord
7440                            && member.kind
7441                                == crate::engine::PartitionLegacyMemberKind::OrdinaryException
7442                    }) {
7443                        return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
7444                            "whole-family replay contains an extra ordinary exception",
7445                        ));
7446                    }
7447                    ordinary = ordinary.saturating_add(1);
7448                }
7449                _ => {
7450                    return Err(ExcelError::new(ExcelErrorKind::Value)
7451                        .with_message("whole-family replay ownership mismatch"));
7452                }
7453            }
7454        }
7455        if shared != source.surviving_member_count
7456            || ordinary != source.reconciliation.ordinary_exceptions
7457        {
7458            return Err(ExcelError::new(ExcelErrorKind::Value)
7459                .with_message("whole-family replay is incomplete"));
7460        }
7461        Ok(())
7462    }
7463
7464    fn prepare_target_combined_legacy_graph(
7465        &self,
7466        packages: &[PreparedTargetSourcePackage],
7467        ordinary: &[PreparedOrdinaryStagedFormula],
7468    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
7469        let mut planned_by_coord = BTreeMap::new();
7470        for package in packages {
7471            for (row, col, ast_id, plan) in &package.legacy {
7472                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
7473            }
7474        }
7475        for formula in ordinary {
7476            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
7477                planned_by_coord.insert(
7478                    (formula.sheet_id, formula.lease.row, formula.lease.col),
7479                    (ast_id, plan),
7480                );
7481            }
7482        }
7483        let planned = planned_by_coord
7484            .into_iter()
7485            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
7486            .collect::<Vec<_>>();
7487        let formula_count = planned.len();
7488        let graph = self
7489            .graph
7490            .prepare_legacy_graph_plan_multi_sheet(planned)
7491            .map_err(|error| {
7492                ExcelError::new(ExcelErrorKind::Value)
7493                    .with_message(format!("target graph preparation failed: {error}"))
7494            })?;
7495        Ok((graph, formula_count))
7496    }
7497
7498    fn materialize_target_package_direct_records(
7499        &mut self,
7500        package: &mut PreparedTargetSourcePackage,
7501        assumptions: &crate::engine::PreparationRevision,
7502        planning_requests: &mut BTreeSet<(String, String, usize)>,
7503        deadline: Option<std::time::Instant>,
7504    ) -> Result<(), ExcelError> {
7505        let existing = package
7506            .legacy
7507            .iter()
7508            .map(|(row, col, _, _)| (*row, *col))
7509            .collect::<BTreeSet<_>>();
7510        let mut missing = BTreeMap::new();
7511        for record in &package.replay_records {
7512            if !existing.contains(&(record.row, record.col)) {
7513                missing.insert((record.row, record.col), record.clone());
7514            }
7515        }
7516        let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
7517        for record in batch.formulas {
7518            self.target_preparation_checkpoint(deadline, 1)?;
7519            let ast = self
7520                .graph
7521                .data_store()
7522                .retrieve_ast(record.ast_id, self.graph.sheet_reg())
7523                .ok_or_else(|| {
7524                    ExcelError::new(ExcelErrorKind::Value)
7525                        .with_message("target fallback AST is unavailable")
7526                })?;
7527            let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
7528            if let Some(reason) =
7529                Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
7530            {
7531                return Err(Self::preparation_stale(
7532                    reason,
7533                    "target fallback planning snapshot became stale during materialization",
7534                ));
7535            }
7536            let placement = CellRef::new(
7537                package.sheet_id,
7538                Coord::from_excel(record.row, record.col, true, true),
7539            );
7540            let ingested = self
7541                .graph
7542                .ingest_pipeline(&snapshot)
7543                .enable_function_semantics()
7544                .ingest_formula(
7545                    FormulaAstInput::RawArena(record.ast_id),
7546                    placement,
7547                    record.formula_text,
7548                )?;
7549            package
7550                .legacy
7551                .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7552        }
7553        package.direct_families = 0;
7554        package.direct_cells = 0;
7555        package.direct_fragments = 0;
7556        package.direct_complete_families = 0;
7557        package.direct_complete_cells = 0;
7558        package.direct_partition_families = 0;
7559        package.direct_partition_cells = 0;
7560        Ok(())
7561    }
7562
7563    fn prepare_target_source_package(
7564        &mut self,
7565        sheet: &str,
7566        lease: StagedPackageLease,
7567        deadline: Option<std::time::Instant>,
7568    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
7569        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
7570            ExcelError::new(ExcelErrorKind::Ref)
7571                .with_message(format!("deferred source sheet not found: {sheet}"))
7572        })?;
7573        let (
7574            mut source_report,
7575            families,
7576            partitions,
7577            replay,
7578            invalidated,
7579            suppressed,
7580            reconciliation_replay,
7581        ) = {
7582            let package = self
7583                .staged_formulas
7584                .get(sheet)
7585                .and_then(|staged| staged.deferred_package.as_ref())
7586                .ok_or_else(|| {
7587                    ExcelError::new(ExcelErrorKind::Value)
7588                        .with_message("staged deferred source package is unavailable")
7589                })?;
7590            if package.sheet_name != sheet {
7591                return Err(ExcelError::new(ExcelErrorKind::Value)
7592                    .with_message("deferred formula package sheet mismatch"));
7593            }
7594            (
7595                package.report.clone(),
7596                package.families.clone(),
7597                package.partitioned_families.clone(),
7598                Arc::clone(&package.replay),
7599                package.invalidated.clone(),
7600                package.suppressed.clone(),
7601                package.reconciliation_replay.clone(),
7602            )
7603        };
7604
7605        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
7606        for partition in &partitions {
7607            replay_disposition
7608                .register_partition(partition, false)
7609                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
7610        }
7611        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
7612        self.target_preparation_checkpoint(deadline, 1)?;
7613        let mut replay_records = if let Some(mut records) = reconciliation_replay {
7614            records.retain(|record| {
7615                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
7616                else {
7617                    return true;
7618                };
7619                let coord = crate::engine::SourceCoord { row, col };
7620                let disposition = record.family.map_or_else(
7621                    || replay_disposition.ordinary_disposition(coord).0,
7622                    |family| replay_disposition.shared_disposition(family, coord),
7623                );
7624                !matches!(
7625                    disposition,
7626                    crate::engine::FormulaReplayCoordinateDisposition::Direct
7627                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
7628                )
7629            });
7630            records
7631        } else {
7632            replay
7633                .lock()
7634                .map_err(|_| {
7635                    ExcelError::new(ExcelErrorKind::Value)
7636                        .with_message("deferred formula spool lock poisoned")
7637                })?
7638                .replay_partitioned(&replay_disposition, &partitions)
7639                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7640        };
7641        replay_records.sort_by_key(|record| record.source_order);
7642        for chunk in replay_records.chunks(256) {
7643            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
7644        }
7645        if replay_records
7646            .windows(2)
7647            .any(|records| records[0].source_order == records[1].source_order)
7648        {
7649            return Err(ExcelError::new(ExcelErrorKind::Value)
7650                .with_message("duplicate deferred source-order proof"));
7651        }
7652
7653        let mut disposition = replay_disposition;
7654        let mut ordered_placements = Vec::new();
7655        let mut direct_families = 0usize;
7656        let mut direct_cells = 0u64;
7657        let mut direct_fragments = 0u64;
7658        let mut direct_complete_families = 0u64;
7659        let mut direct_complete_cells = 0u64;
7660        let mut direct_partition_families = 0u64;
7661        let mut direct_partition_cells = 0u64;
7662        let mut anchor_parses = 0u64;
7663        let mut anchor_asts = 0u64;
7664        let mut anchor_analyses = 0u64;
7665        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
7666            for family in &families {
7667                self.target_preparation_checkpoint(deadline, 1)?;
7668                if invalidated.contains(&family.source_id) {
7669                    continue;
7670                }
7671                match self.prepare_source_formula_family(sheet_id, family, true, None) {
7672                    Ok((prepared, _)) => {
7673                        anchor_parses = anchor_parses.saturating_add(1);
7674                        anchor_asts = anchor_asts.saturating_add(1);
7675                        anchor_analyses = anchor_analyses.saturating_add(1);
7676                        direct_families = direct_families.saturating_add(1);
7677                        direct_cells = direct_cells.saturating_add(prepared.member_count);
7678                        direct_complete_families = direct_complete_families.saturating_add(1);
7679                        direct_complete_cells =
7680                            direct_complete_cells.saturating_add(prepared.member_count);
7681                        if self.config.formula_plane_mode
7682                            == FormulaPlaneMode::AuthoritativeExperimental
7683                        {
7684                            disposition.set_family_direct(family.source_id);
7685                        }
7686                        ordered_placements.push((family.source_order, vec![prepared]));
7687                    }
7688                    Err(error) => {
7689                        anchor_parses =
7690                            anchor_parses.saturating_add(u64::from(error.parse_attempted));
7691                        anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7692                        anchor_analyses =
7693                            anchor_analyses.saturating_add(u64::from(error.analysis_created));
7694                        *source_report
7695                            .fallback_reasons
7696                            .entry(error.reason)
7697                            .or_default() += 1;
7698                    }
7699                }
7700            }
7701
7702            for source in &partitions {
7703                self.target_preparation_checkpoint(deadline, 1)?;
7704                if invalidated.contains(&source.source_id) {
7705                    continue;
7706                }
7707                let prepared =
7708                    match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
7709                        Ok(prepared) => {
7710                            anchor_parses = anchor_parses.saturating_add(1);
7711                            anchor_asts = anchor_asts.saturating_add(1);
7712                            anchor_analyses = anchor_analyses.saturating_add(1);
7713                            prepared
7714                        }
7715                        Err(error) => {
7716                            anchor_parses =
7717                                anchor_parses.saturating_add(u64::from(error.parse_attempted));
7718                            anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
7719                            anchor_analyses =
7720                                anchor_analyses.saturating_add(u64::from(error.analysis_created));
7721                            *source_report
7722                                .fallback_reasons
7723                                .entry(error.reason)
7724                                .or_default() += 1;
7725                            continue;
7726                        }
7727                    };
7728                let fragment_regions: Vec<_> = prepared
7729                    .placements
7730                    .iter()
7731                    .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
7732                    .collect();
7733                let crosses_fragment =
7734                    prepared
7735                        .placements
7736                        .iter()
7737                        .enumerate()
7738                        .any(|(fragment_index, placement)| {
7739                            placement
7740                                .fragment_dependency_proof()
7741                                .1
7742                                .dependencies
7743                                .iter()
7744                                .any(|dependency| {
7745                                    fragment_regions.iter().enumerate().any(
7746                                        |(candidate_index, candidate)| {
7747                                            candidate_index != fragment_index
7748                                                && dependency.read_region.intersects(candidate)
7749                                        },
7750                                    )
7751                                })
7752                        });
7753                let family_records = replay_records
7754                    .iter()
7755                    .filter(|record| {
7756                        record.family == Some(source.source_id)
7757                            || record.partition_owner == Some(source.source_id)
7758                    })
7759                    .cloned()
7760                    .collect::<Vec<_>>();
7761                if crosses_fragment {
7762                    *source_report
7763                        .fallback_reasons
7764                        .entry("CrossFragmentDependency".to_string())
7765                        .or_default() += 1;
7766                    continue;
7767                }
7768                if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
7769                    *source_report
7770                        .fallback_reasons
7771                        .entry(format!("TargetPartitionReplay:{error}"))
7772                        .or_default() += 1;
7773                    continue;
7774                }
7775                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
7776                    let mut candidate = disposition.clone();
7777                    if let Err(reason) = candidate.register_partition(source, true) {
7778                        *source_report
7779                            .fallback_reasons
7780                            .entry(reason.to_string())
7781                            .or_default() += 1;
7782                        continue;
7783                    }
7784                    disposition = candidate;
7785                }
7786                direct_families = direct_families.saturating_add(1);
7787                direct_cells = direct_cells.saturating_add(prepared.direct_cells);
7788                direct_partition_families = direct_partition_families.saturating_add(1);
7789                direct_partition_cells =
7790                    direct_partition_cells.saturating_add(prepared.direct_cells);
7791                direct_fragments =
7792                    direct_fragments.saturating_add(prepared.placements.len() as u64);
7793                ordered_placements.push((source.source_order, prepared.placements));
7794            }
7795        }
7796        ordered_placements.sort_by_key(|(source_order, _)| *source_order);
7797        let placements = ordered_placements
7798            .into_iter()
7799            .flat_map(|(_, placements)| placements)
7800            .collect();
7801
7802        Ok(PreparedTargetSourcePackage {
7803            sheet: sheet.to_string(),
7804            sheet_id,
7805            lease,
7806            source_report,
7807            replay_records,
7808            disposition,
7809            placements,
7810            legacy: Vec::new(),
7811            direct_families,
7812            direct_cells,
7813            direct_fragments,
7814            direct_complete_families,
7815            direct_complete_cells,
7816            direct_partition_families,
7817            direct_partition_cells,
7818            anchor_parses,
7819            anchor_asts,
7820            anchor_analyses,
7821        })
7822    }
7823
7824    fn fallback_planning_snapshot(
7825        &self,
7826        batch: &FormulaIngestBatch,
7827    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
7828        let mut requests = Vec::new();
7829        for formula in &batch.formulas {
7830            let ast = self
7831                .graph
7832                .data_store()
7833                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
7834                .ok_or_else(|| {
7835                    ExcelError::new(ExcelErrorKind::Value)
7836                        .with_message("ordered fallback AST is unavailable")
7837                })?;
7838            Self::collect_planning_function_requests(&ast, &mut requests);
7839        }
7840        requests.sort();
7841        requests.dedup();
7842        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7843            &self.resolver,
7844            requests,
7845        )
7846        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
7847    }
7848
7849    fn prepare_legacy_batch_fallback(
7850        &mut self,
7851        batch: FormulaIngestBatch,
7852        function_provider: &dyn crate::traits::FunctionProvider,
7853    ) -> Result<
7854        (
7855            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
7856            u64,
7857        ),
7858        ExcelError,
7859    > {
7860        let formula_count = batch.formulas.len() as u64;
7861        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7862        let mut planned = Vec::with_capacity(batch.formulas.len());
7863        for record in batch.formulas {
7864            let placement = CellRef::new(
7865                sheet_id,
7866                Coord::from_excel(record.row, record.col, true, true),
7867            );
7868            let ingested = self
7869                .graph
7870                .ingest_pipeline(function_provider)
7871                .enable_function_semantics()
7872                .ingest_formula(
7873                    FormulaAstInput::RawArena(record.ast_id),
7874                    placement,
7875                    record.formula_text,
7876                )
7877                .map_err(|error| {
7878                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
7879                })?;
7880            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
7881        }
7882        let plan = self
7883            .graph
7884            .prepare_legacy_graph_plan(sheet_id, planned)
7885            .map_err(|error| {
7886                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
7887            })?;
7888        Ok((plan, formula_count))
7889    }
7890
7891    fn publish_compressed_partial_report(
7892        &mut self,
7893        report: &FormulaIngestReport,
7894        direct_report: &FormulaIngestReport,
7895    ) {
7896        if direct_report.source_family_promoted == 0
7897            && direct_report.graph_formula_cells_materialized == 0
7898        {
7899            return;
7900        }
7901        let mut published = report.clone();
7902        published.accumulate(direct_report);
7903        self.record_formula_ingest_report(published);
7904    }
7905
7906    fn finish_compressed_formula_sources(
7907        &mut self,
7908        batches: Vec<(
7909            FormulaIngestBatch,
7910            crate::engine::FormulaCompressedSourceReport,
7911            crate::engine::FormulaCompressedPreparation,
7912        )>,
7913    ) -> Result<FormulaIngestReport, ExcelError> {
7914        self.observe_function_semantic_epoch()?;
7915        if batches.iter().any(|(_, _, preparation)| {
7916            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
7917        }) {
7918            return Err(ExcelError::new(ExcelErrorKind::Value)
7919                .with_message("compressed source preparation belongs to another engine"));
7920        }
7921        if batches.iter().any(|(fallback, _, preparation)| {
7922            preparation.sheet_name.as_ref() != fallback.sheet_name
7923        }) {
7924            return Err(ExcelError::new(ExcelErrorKind::Value)
7925                .with_message("compressed source preparation sheet mismatch"));
7926        }
7927        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
7928        let initial_provider_revision = self.resolver.planning_semantic_revision();
7929        let mut fallback_batches = Vec::with_capacity(batches.len());
7930        let mut stale_fallback_batches = Vec::new();
7931        let mut pending_preparations = Vec::new();
7932        for (mut fallback, mut source, mut preparation) in batches {
7933            for formula in fallback.formulas.drain(..) {
7934                let source_order = formula.source_order.ok_or_else(|| {
7935                    ExcelError::new(ExcelErrorKind::Value).with_message(
7936                        "compressed source supplied formulas without source-order proof",
7937                    )
7938                })?;
7939                let text = formula.formula_text.ok_or_else(|| {
7940                    ExcelError::new(ExcelErrorKind::Value).with_message(
7941                        "ordered compressed fallback formula has no exact source text",
7942                    )
7943                })?;
7944                preparation
7945                    .eager_replay
7946                    .push(crate::engine::DeferredReplayFormula {
7947                        source_order,
7948                        row: formula.row,
7949                        col: formula.col,
7950                        text: text.to_string(),
7951                        family: formula.source_family,
7952                        partition_owner: formula.partition_owner,
7953                    });
7954            }
7955            preparation
7956                .eager_replay
7957                .sort_by_key(|record| record.source_order);
7958            source.source_spool_replays = source
7959                .source_spool_replays
7960                .saturating_add(preparation.preparation_spool_replays);
7961            let stale_reason = preparation
7962                .function_semantics_used
7963                .then(|| {
7964                    if preparation.function_provider_revision != initial_provider_revision {
7965                        Some("FunctionProviderRevisionChanged")
7966                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
7967                    {
7968                        Some("FunctionSemanticEpochChanged")
7969                    } else {
7970                        None
7971                    }
7972                })
7973                .flatten();
7974            let stale_semantics = stale_reason.is_some();
7975            if let Some(reason) = stale_reason {
7976                for (family, _, _) in &preparation.prepared {
7977                    preparation.rejected.insert(*family, reason.to_string());
7978                }
7979            }
7980            for reason in preparation.rejected.values() {
7981                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
7982            }
7983            let clean_direct_cells: u64 = preparation
7984                .prepared
7985                .iter()
7986                .map(|(_, _, prepared)| prepared.member_count)
7987                .sum();
7988            let fragmented_direct_cells: u64 = preparation
7989                .fragmented
7990                .iter()
7991                .map(|fragment| fragment.source.surviving_member_count)
7992                .sum();
7993            let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
7994            source.replay_families = source.families_seen.saturating_sub(
7995                preparation
7996                    .prepared
7997                    .len()
7998                    .saturating_add(preparation.fragmented.len()) as u64,
7999            );
8000            source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
8001            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
8002                fallback.sheet_name.clone(),
8003                source,
8004            );
8005            if stale_semantics {
8006                stale_fallback_batches.push((fallback, compressed));
8007            } else {
8008                fallback_batches.push((fallback, compressed));
8009            }
8010            pending_preparations.push((preparation, stale_semantics));
8011        }
8012        drop(initial_guard);
8013
8014        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
8015        let configured_mode = self.config.formula_plane_mode;
8016        self.config.formula_plane_mode = FormulaPlaneMode::Off;
8017        let stale_result =
8018            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
8019        self.config.formula_plane_mode = configured_mode;
8020        let mut report = stale_result?;
8021        report.mode = configured_mode;
8022
8023        self.config.formula_plane_mode = FormulaPlaneMode::Off;
8024        let fallback_result =
8025            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
8026        self.config.formula_plane_mode = configured_mode;
8027        match fallback_result {
8028            Ok(fallback_report) => report.accumulate(&fallback_report),
8029            Err(error) => {
8030                self.record_formula_ingest_report(report);
8031                return Err(error);
8032            }
8033        }
8034
8035        let mut direct_report =
8036            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
8037        for (preparation, _) in &pending_preparations {
8038            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
8039                preparation.clean_rejected_anchor_counts[0]
8040                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
8041            );
8042            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
8043                preparation.clean_rejected_anchor_counts[1]
8044                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
8045            );
8046            direct_report.source_anchor_analyses =
8047                direct_report.source_anchor_analyses.saturating_add(
8048                    preparation.clean_rejected_anchor_counts[2]
8049                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
8050                );
8051            let rejected_partitions: Vec<_> = preparation
8052                .rejected
8053                .keys()
8054                .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
8055                .collect();
8056            direct_report.source_partitioned_families_rejected = direct_report
8057                .source_partitioned_families_rejected
8058                .saturating_add(rejected_partitions.len() as u64);
8059            direct_report.source_partition_failures = direct_report
8060                .source_partition_failures
8061                .saturating_add(rejected_partitions.len() as u64);
8062            direct_report.source_partition_fallback_cells = direct_report
8063                .source_partition_fallback_cells
8064                .saturating_add(
8065                    rejected_partitions
8066                        .iter()
8067                        .map(|source| source.surviving_member_count)
8068                        .sum::<u64>(),
8069                );
8070        }
8071        loop {
8072            #[cfg(any(test, feature = "test-support"))]
8073            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8074                hook();
8075            }
8076            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8077            let commit_provider_revision = self.resolver.planning_semantic_revision();
8078            let mut newly_stale = Vec::new();
8079            let mut current = Vec::new();
8080            for pending in pending_preparations.drain(..) {
8081                let stale_reason = pending
8082                    .0
8083                    .function_semantics_used
8084                    .then(|| {
8085                        if pending.0.function_provider_revision != commit_provider_revision {
8086                            Some("FunctionProviderRevisionChanged")
8087                        } else if self
8088                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
8089                        {
8090                            Some("FunctionSemanticEpochChanged")
8091                        } else {
8092                            None
8093                        }
8094                    })
8095                    .flatten();
8096                if let Some(reason) = stale_reason {
8097                    newly_stale.push((pending, reason));
8098                } else {
8099                    current.push(pending);
8100                }
8101            }
8102            if !newly_stale.is_empty() {
8103                drop(commit_guard);
8104                for ((mut preparation, was_initially_stale), reason) in newly_stale {
8105                    for (family, _, _) in &preparation.prepared {
8106                        preparation.rejected.insert(*family, reason.to_string());
8107                    }
8108                    let exact = match self.replay_prepared_families_exact_records(&preparation) {
8109                        Ok(exact) => exact,
8110                        Err(error) => {
8111                            self.publish_compressed_partial_report(&report, &direct_report);
8112                            return Err(error);
8113                        }
8114                    };
8115                    preparation.eager_replay.extend(exact);
8116                    preparation
8117                        .eager_replay
8118                        .sort_by_key(|record| record.source_order);
8119                    direct_report.source_spool_replays =
8120                        direct_report.source_spool_replays.saturating_add(1);
8121                    if !was_initially_stale {
8122                        *direct_report
8123                            .fallback_reasons
8124                            .entry(reason.to_string())
8125                            .or_default() += preparation.prepared.len() as u64;
8126                    }
8127                    direct_report.source_family_fallback = direct_report
8128                        .source_family_fallback
8129                        .saturating_add(preparation.prepared.len() as u64);
8130                    direct_report.source_family_fallback_cells =
8131                        direct_report.source_family_fallback_cells.saturating_add(
8132                            preparation
8133                                .prepared
8134                                .iter()
8135                                .map(|(_, _, prepared)| prepared.member_count)
8136                                .sum(),
8137                        );
8138                    preparation.prepared.clear();
8139                    preparation.function_semantics_used = false;
8140                    if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8141                        current.push((preparation, false));
8142                    }
8143                }
8144                pending_preparations = current;
8145                continue;
8146            }
8147            drop(commit_guard);
8148
8149            // Preserve source order across clean and fragmented families. Every clean
8150            // placement uses E3's incremental append, so no global index rebuild is needed.
8151            enum SourceProposal {
8152                KnownFallback {
8153                    source_order: crate::engine::SourceFormulaOrder,
8154                    records: Vec<crate::engine::DeferredReplayFormula>,
8155                },
8156                Clean(
8157                    crate::engine::SourceFamilyId,
8158                    crate::engine::SourceFormulaOrder,
8159                    crate::formula_plane::placement::PreparedAnchorOncePlacement,
8160                ),
8161                Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8162            }
8163
8164            impl SourceProposal {
8165                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8166                    match self {
8167                        Self::KnownFallback { source_order, .. } => *source_order,
8168                        Self::Clean(_, source_order, _) => *source_order,
8169                        Self::Fragment(fragment) => fragment.source.source_order,
8170                    }
8171                }
8172            }
8173
8174            for (mut preparation, _) in current {
8175                let mut proposals = Vec::with_capacity(
8176                    preparation
8177                        .prepared
8178                        .len()
8179                        .saturating_add(preparation.fragmented.len())
8180                        .saturating_add(preparation.eager_replay.len()),
8181                );
8182                proposals.extend(preparation.prepared.drain(..).map(
8183                    |(source_id, source_order, prepared)| {
8184                        SourceProposal::Clean(source_id, source_order, prepared)
8185                    },
8186                ));
8187
8188                let accepted_fragments: BTreeMap<_, _> = preparation
8189                    .fragmented
8190                    .iter()
8191                    .map(|fragment| (fragment.source.source_id, &fragment.source))
8192                    .collect();
8193                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8194                for record in preparation.eager_replay.drain(..).filter(|record| {
8195                    !record.partition_owner.is_some_and(|owner| {
8196                        accepted_fragments.get(&owner).is_some_and(|source| {
8197                            Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8198                        })
8199                    })
8200                }) {
8201                    if let Some(owner) = record.partition_owner.or(record.family) {
8202                        family_fallbacks.entry(owner).or_default().push(record);
8203                    } else {
8204                        proposals.push(SourceProposal::KnownFallback {
8205                            source_order: record.source_order,
8206                            records: vec![record],
8207                        });
8208                    }
8209                }
8210                for (_, mut records) in family_fallbacks {
8211                    records.sort_by_key(|record| record.source_order);
8212                    if records
8213                        .windows(2)
8214                        .any(|window| window[0].source_order == window[1].source_order)
8215                    {
8216                        self.publish_compressed_partial_report(&report, &direct_report);
8217                        return Err(ExcelError::new(ExcelErrorKind::Value)
8218                            .with_message("duplicate exact-replay source-order proof"));
8219                    }
8220                    let Some(source_order) = records.first().map(|record| record.source_order)
8221                    else {
8222                        self.publish_compressed_partial_report(&report, &direct_report);
8223                        return Err(ExcelError::new(ExcelErrorKind::Value)
8224                            .with_message("empty exact-replay fallback family"));
8225                    };
8226                    proposals.push(SourceProposal::KnownFallback {
8227                        source_order,
8228                        records,
8229                    });
8230                }
8231                proposals.extend(
8232                    preparation
8233                        .fragmented
8234                        .drain(..)
8235                        .map(SourceProposal::Fragment),
8236                );
8237                proposals.sort_by_key(SourceProposal::source_order);
8238                if proposals
8239                    .windows(2)
8240                    .any(|window| window[0].source_order() == window[1].source_order())
8241                {
8242                    self.publish_compressed_partial_report(&report, &direct_report);
8243                    return Err(ExcelError::new(ExcelErrorKind::Value)
8244                        .with_message("ambiguous compressed source-order proof"));
8245                }
8246
8247                for proposal in proposals {
8248                    match proposal {
8249                        SourceProposal::KnownFallback { records, .. } => {
8250                            let batch = match self.formula_batch_from_exact_replay(
8251                                preparation.sheet_name.as_ref(),
8252                                records,
8253                            ) {
8254                                Ok(batch) => batch,
8255                                Err(error) => {
8256                                    self.publish_compressed_partial_report(&report, &direct_report);
8257                                    return Err(error);
8258                                }
8259                            };
8260                            if batch.is_empty() {
8261                                continue;
8262                            }
8263                            let snapshot = match self.fallback_planning_snapshot(&batch) {
8264                                Ok(snapshot) => snapshot,
8265                                Err(error) => {
8266                                    self.publish_compressed_partial_report(&report, &direct_report);
8267                                    return Err(error);
8268                                }
8269                            };
8270                            let commit_guard =
8271                                crate::function_registry::semantic_epoch_read_guard();
8272                            let provider_revision_initial =
8273                                self.resolver.planning_semantic_revision();
8274                            if (commit_guard.epoch() != snapshot.epoch()
8275                                && snapshot.semantic_changes_affect_requests_since_guarded(
8276                                    &commit_guard,
8277                                    snapshot.epoch(),
8278                                ))
8279                                || snapshot.provider_revision().is_some_and(|revision| {
8280                                    Some(revision) != provider_revision_initial
8281                                })
8282                            {
8283                                drop(commit_guard);
8284                                self.publish_compressed_partial_report(&report, &direct_report);
8285                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8286                                    "ordered fallback planning snapshot became stale",
8287                                ));
8288                            }
8289                            let (plan, formula_count) = match self
8290                                .prepare_legacy_batch_fallback(batch, &snapshot)
8291                            {
8292                                Ok(plan) => plan,
8293                                Err(error) => {
8294                                    drop(commit_guard);
8295                                    self.publish_compressed_partial_report(&report, &direct_report);
8296                                    return Err(error);
8297                                }
8298                            };
8299                            let provider_revision_after =
8300                                self.resolver.planning_semantic_revision();
8301                            if provider_revision_after != provider_revision_initial {
8302                                drop(commit_guard);
8303                                self.publish_compressed_partial_report(&report, &direct_report);
8304                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8305                                    "function provider changed while preparing ordered fallback",
8306                                ));
8307                            }
8308                            let graph_vertices = plan.new_vertex_count();
8309                            let Some(graph_edges) = plan.planned_edge_count() else {
8310                                drop(commit_guard);
8311                                self.publish_compressed_partial_report(&report, &direct_report);
8312                                return Err(ExcelError::new(ExcelErrorKind::Value)
8313                                    .with_message("prepared ordered fallback size overflow"));
8314                            };
8315                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
8316                            {
8317                                drop(commit_guard);
8318                                self.publish_compressed_partial_report(&report, &direct_report);
8319                                return Err(error);
8320                            }
8321                            if let Err(error) =
8322                                self.graph.validate_prepared_legacy_graph_plan(&plan)
8323                            {
8324                                drop(commit_guard);
8325                                self.publish_compressed_partial_report(&report, &direct_report);
8326                                return Err(ExcelError::new(ExcelErrorKind::Value)
8327                                    .with_message(error.to_string()));
8328                            }
8329                            #[cfg(test)]
8330                            if let Some(hook) = self
8331                                .before_legacy_fallback_final_provider_sample_hook
8332                                .take()
8333                            {
8334                                hook();
8335                            }
8336                            let provider_revision_final =
8337                                self.resolver.planning_semantic_revision();
8338                            if provider_revision_final != provider_revision_initial {
8339                                drop(commit_guard);
8340                                self.publish_compressed_partial_report(&report, &direct_report);
8341                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8342                                    "function provider changed after ordered fallback validation",
8343                                ));
8344                            }
8345                            let graph_formulas =
8346                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
8347                            direct_report.formula_cells_seen = direct_report
8348                                .formula_cells_seen
8349                                .saturating_add(formula_count);
8350                            direct_report.graph_formula_cells_materialized = direct_report
8351                                .graph_formula_cells_materialized
8352                                .saturating_add(graph_formulas as u64);
8353                            direct_report.graph_vertices_created = direct_report
8354                                .graph_vertices_created
8355                                .saturating_add(graph_vertices as u64);
8356                            direct_report.graph_edges_created = direct_report
8357                                .graph_edges_created
8358                                .saturating_add(graph_edges as u64);
8359                        }
8360                        SourceProposal::Clean(source_id, _, prepared) => {
8361                            let commit_guard =
8362                                crate::function_registry::semantic_epoch_read_guard();
8363                            let commit_provider_revision =
8364                                self.resolver.planning_semantic_revision();
8365                            let semantic_changed = preparation.function_semantics_used
8366                                && self.prepared_placement_function_semantics_changed(
8367                                    preparation.function_semantic_epoch,
8368                                    &prepared,
8369                                    &commit_guard,
8370                                );
8371                            let cells = prepared.member_count;
8372                            let stats = self.graph.baseline_stats();
8373                            self.preflight_graph_admission(
8374                                crate::engine::resource_ledger::GraphAdmission {
8375                                    final_vertices: stats.graph_vertex_count,
8376                                    final_edges: stats.graph_edge_count,
8377                                    materialization_cells: 0,
8378                                    added_vertices: 0,
8379                                    added_edges: 0,
8380                                },
8381                            )?;
8382                            let append = self
8383                                .graph
8384                                .formula_authority()
8385                                .prepare_formula_plane_append(
8386                                    vec![prepared],
8387                                    self.graph.data_store(),
8388                                    self.graph.sheet_reg(),
8389                                )
8390                                .map_err(|error| error.to_string())
8391                                .and_then(|append| {
8392                                    if preparation.function_semantics_used
8393                                        && preparation.function_provider_revision
8394                                            != commit_provider_revision
8395                                    {
8396                                        return Err("FunctionProviderRevisionChanged".to_string());
8397                                    }
8398                                    if preparation.function_semantics_used
8399                                        && self.resolver.planning_semantic_revision()
8400                                            != commit_provider_revision
8401                                    {
8402                                        return Err("FunctionProviderRevisionChanged".to_string());
8403                                    }
8404                                    if semantic_changed {
8405                                        return Err("FunctionSemanticEpochChanged".to_string());
8406                                    }
8407                                    self.graph
8408                                        .formula_authority()
8409                                        .validate_prepared_formula_plane_append(
8410                                            &append,
8411                                            self.graph.data_store(),
8412                                            self.graph.sheet_reg(),
8413                                        )
8414                                        .map_err(|error| error.to_string())?;
8415                                    Ok(append)
8416                                });
8417                            match append {
8418                                Ok(append) => {
8419                                    let placement_report = self
8420                                        .graph
8421                                        .formula_authority_mut()
8422                                        .apply_prevalidated_formula_plane_append(append);
8423                                    self.graph.mark_formula_spans_dirty(
8424                                        placement_report.spans.iter().copied(),
8425                                        WholeSpanDirtyReason::NewSpan,
8426                                    );
8427                                    direct_report.formula_cells_seen =
8428                                        direct_report.formula_cells_seen.saturating_add(cells);
8429                                    direct_report.shadow_candidate_cells =
8430                                        direct_report.shadow_candidate_cells.saturating_add(cells);
8431                                    direct_report.shadow_accepted_span_cells = direct_report
8432                                        .shadow_accepted_span_cells
8433                                        .saturating_add(cells);
8434                                    direct_report.shadow_templates_interned =
8435                                        direct_report.shadow_templates_interned.saturating_add(
8436                                            placement_report.work.templates_to_append as u64,
8437                                        );
8438                                    direct_report.shadow_spans_created = direct_report
8439                                        .shadow_spans_created
8440                                        .saturating_add(placement_report.spans.len() as u64);
8441                                    direct_report.graph_formula_vertices_avoided_shadow =
8442                                        direct_report
8443                                            .graph_formula_vertices_avoided_shadow
8444                                            .saturating_add(cells);
8445                                    direct_report.ast_roots_avoided_shadow = direct_report
8446                                        .ast_roots_avoided_shadow
8447                                        .saturating_add(cells.saturating_sub(1));
8448                                    direct_report.edge_rows_avoided_shadow = direct_report
8449                                        .edge_rows_avoided_shadow
8450                                        .saturating_add(cells);
8451                                    direct_report.source_family_promoted =
8452                                        direct_report.source_family_promoted.saturating_add(1);
8453                                    direct_report.source_family_promoted_cells = direct_report
8454                                        .source_family_promoted_cells
8455                                        .saturating_add(cells);
8456                                    direct_report.source_anchor_parses =
8457                                        direct_report.source_anchor_parses.saturating_add(1);
8458                                    direct_report.source_anchor_asts =
8459                                        direct_report.source_anchor_asts.saturating_add(1);
8460                                    direct_report.source_anchor_analyses =
8461                                        direct_report.source_anchor_analyses.saturating_add(1);
8462                                    let descendants = cells.saturating_sub(1);
8463                                    direct_report.source_descendant_strings_avoided = direct_report
8464                                        .source_descendant_strings_avoided
8465                                        .saturating_add(descendants);
8466                                    direct_report.source_descendant_events_avoided = direct_report
8467                                        .source_descendant_events_avoided
8468                                        .saturating_add(descendants);
8469                                    direct_report.source_descendant_analyses_avoided =
8470                                        direct_report
8471                                            .source_descendant_analyses_avoided
8472                                            .saturating_add(descendants);
8473                                    direct_report.source_compressed_families_prepared =
8474                                        direct_report
8475                                            .source_compressed_families_prepared
8476                                            .saturating_add(1);
8477                                    direct_report.source_compressed_cells_prepared = direct_report
8478                                        .source_compressed_cells_prepared
8479                                        .saturating_add(cells);
8480                                }
8481                                Err(error) => {
8482                                    drop(commit_guard);
8483                                    let reason = format!("CleanFormulaPlaneAppend:{error}");
8484                                    let batch = match self.replay_one_prepared_family_exact(
8485                                        &preparation,
8486                                        source_id,
8487                                        cells,
8488                                    ) {
8489                                        Ok(batch) => batch,
8490                                        Err(error) => {
8491                                            self.publish_compressed_partial_report(
8492                                                &report,
8493                                                &direct_report,
8494                                            );
8495                                            return Err(error);
8496                                        }
8497                                    };
8498                                    let snapshot = match self.fallback_planning_snapshot(&batch) {
8499                                        Ok(snapshot) => snapshot,
8500                                        Err(error) => {
8501                                            self.publish_compressed_partial_report(
8502                                                &report,
8503                                                &direct_report,
8504                                            );
8505                                            return Err(error);
8506                                        }
8507                                    };
8508                                    let commit_guard =
8509                                        crate::function_registry::semantic_epoch_read_guard();
8510                                    let provider_revision =
8511                                        self.resolver.planning_semantic_revision();
8512                                    if (commit_guard.epoch() != snapshot.epoch()
8513                                        && snapshot.semantic_changes_affect_requests_since_guarded(
8514                                            &commit_guard,
8515                                            snapshot.epoch(),
8516                                        ))
8517                                        || snapshot.provider_revision().is_some_and(|revision| {
8518                                            Some(revision) != provider_revision
8519                                        })
8520                                    {
8521                                        drop(commit_guard);
8522                                        self.publish_compressed_partial_report(
8523                                            &report,
8524                                            &direct_report,
8525                                        );
8526                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8527                                            .with_message(
8528                                                "clean-family fallback planning snapshot became stale",
8529                                            ));
8530                                    }
8531                                    let (plan, formula_count) = match self
8532                                        .prepare_legacy_batch_fallback(batch, &snapshot)
8533                                    {
8534                                        Ok(plan) => plan,
8535                                        Err(error) => {
8536                                            drop(commit_guard);
8537                                            self.publish_compressed_partial_report(
8538                                                &report,
8539                                                &direct_report,
8540                                            );
8541                                            return Err(error);
8542                                        }
8543                                    };
8544                                    let provider_revision_after =
8545                                        self.resolver.planning_semantic_revision();
8546                                    if provider_revision_after != provider_revision {
8547                                        drop(commit_guard);
8548                                        self.publish_compressed_partial_report(
8549                                            &report,
8550                                            &direct_report,
8551                                        );
8552                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8553                                            .with_message(
8554                                                "function provider changed while preparing clean-family fallback",
8555                                            ));
8556                                    }
8557                                    let graph_vertices = plan.new_vertex_count();
8558                                    let Some(graph_edges) = plan.planned_edge_count() else {
8559                                        drop(commit_guard);
8560                                        self.publish_compressed_partial_report(
8561                                            &report,
8562                                            &direct_report,
8563                                        );
8564                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8565                                            .with_message(
8566                                                "prepared clean-family fallback size overflow",
8567                                            ));
8568                                    };
8569                                    if let Err(error) =
8570                                        self.prepared_legacy_admission(&plan, formula_count)
8571                                    {
8572                                        drop(commit_guard);
8573                                        self.publish_compressed_partial_report(
8574                                            &report,
8575                                            &direct_report,
8576                                        );
8577                                        return Err(error);
8578                                    }
8579                                    if let Err(error) =
8580                                        self.graph.validate_prepared_legacy_graph_plan(&plan)
8581                                    {
8582                                        drop(commit_guard);
8583                                        self.publish_compressed_partial_report(
8584                                            &report,
8585                                            &direct_report,
8586                                        );
8587                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8588                                            .with_message(error.to_string()));
8589                                    }
8590                                    #[cfg(test)]
8591                                    if let Some(hook) = self
8592                                        .before_legacy_fallback_final_provider_sample_hook
8593                                        .take()
8594                                    {
8595                                        hook();
8596                                    }
8597                                    let provider_revision_final =
8598                                        self.resolver.planning_semantic_revision();
8599                                    if provider_revision_final != provider_revision {
8600                                        drop(commit_guard);
8601                                        self.publish_compressed_partial_report(
8602                                            &report,
8603                                            &direct_report,
8604                                        );
8605                                        return Err(ExcelError::new(ExcelErrorKind::Value)
8606                                            .with_message(
8607                                                "function provider changed after clean-family fallback validation",
8608                                            ));
8609                                    }
8610                                    let graph_formulas =
8611                                        self.graph.apply_prevalidated_legacy_graph_plan(plan);
8612                                    direct_report.formula_cells_seen = direct_report
8613                                        .formula_cells_seen
8614                                        .saturating_add(formula_count);
8615                                    direct_report.graph_formula_cells_materialized = direct_report
8616                                        .graph_formula_cells_materialized
8617                                        .saturating_add(graph_formulas as u64);
8618                                    direct_report.graph_vertices_created = direct_report
8619                                        .graph_vertices_created
8620                                        .saturating_add(graph_vertices as u64);
8621                                    direct_report.graph_edges_created = direct_report
8622                                        .graph_edges_created
8623                                        .saturating_add(graph_edges as u64);
8624                                    direct_report.source_spool_replays =
8625                                        direct_report.source_spool_replays.saturating_add(1);
8626                                    direct_report.source_family_fallback =
8627                                        direct_report.source_family_fallback.saturating_add(1);
8628                                    direct_report.source_family_fallback_cells = direct_report
8629                                        .source_family_fallback_cells
8630                                        .saturating_add(cells);
8631                                    direct_report.source_anchor_parses =
8632                                        direct_report.source_anchor_parses.saturating_add(1);
8633                                    direct_report.source_anchor_asts =
8634                                        direct_report.source_anchor_asts.saturating_add(1);
8635                                    direct_report.source_anchor_analyses =
8636                                        direct_report.source_anchor_analyses.saturating_add(1);
8637                                    *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8638                                }
8639                            }
8640                        }
8641                        SourceProposal::Fragment(fragment) => {
8642                            let commit_guard =
8643                                crate::function_registry::semantic_epoch_read_guard();
8644                            let commit_epoch = commit_guard.epoch();
8645                            let commit_provider_revision =
8646                                self.resolver.planning_semantic_revision();
8647                            let source = fragment.source;
8648                            let source_id = source.source_id;
8649                            #[cfg(test)]
8650                            let commit_fault = self
8651                                .fragmented_commit_fault_for_test
8652                                .take()
8653                                .unwrap_or(
8654                                crate::engine::fragmented_transaction::FragmentedCommitFault::None,
8655                            );
8656                            #[cfg(not(test))]
8657                            let commit_fault =
8658                                crate::engine::fragmented_transaction::FragmentedCommitFault::None;
8659                            let outcome = self
8660                                .prepare_fragmented_source_transaction(
8661                                    &source,
8662                                    &preparation.replay_disposition,
8663                                    fragment.prepared,
8664                                    fragment.legacy,
8665                                )
8666                                .map(|transaction| {
8667                                    let resolver = &self.resolver;
8668                                    self.graph.commit_fragmented_source_transaction(
8669                                        transaction,
8670                                        &self.source_formula_token,
8671                                        &preparation.replay_disposition,
8672                                        commit_epoch,
8673                                        commit_provider_revision,
8674                                        || resolver.planning_semantic_revision(),
8675                                        commit_fault,
8676                                    )
8677                                });
8678                            let fallback_reason = match outcome {
8679                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
8680                                    let mut delta = success.report_delta;
8681                                    delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
8682                                    delta.source_family_promoted = 1;
8683                                    delta.source_family_promoted_cells =
8684                                        source.surviving_member_count;
8685                                    delta.source_partitioned_families_seen = 0;
8686                                    delta.source_partition_holes = 0;
8687                                    delta.source_partition_ordinary_exceptions = 0;
8688                                    delta.source_partition_surviving_cells = 0;
8689                                    delta.shadow_templates_interned =
8690                                        success.plane.work.templates_to_append as u64;
8691                                    delta.shadow_spans_created = success.plane.spans.len() as u64;
8692                                    let descendants =
8693                                        source.surviving_member_count.saturating_sub(1);
8694                                    delta.source_descendant_strings_avoided = descendants;
8695                                    delta.source_descendant_events_avoided = descendants;
8696                                    delta.source_descendant_analyses_avoided = descendants;
8697                                    direct_report.accumulate(&delta);
8698                                    None
8699                                }
8700                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
8701                                    Some(format!("{reason:?}"))
8702                                }
8703                                Err(error) => Some(format!("{error:?}")),
8704                            };
8705                            if let Some(reason) = fallback_reason {
8706                                drop(commit_guard);
8707                                let mut replay_disposition = preparation.replay_disposition.clone();
8708                                replay_disposition.force_family_legacy(source_id);
8709                                let replayed = (|| {
8710                                    let records = fragment
8711                                        .replay
8712                                        .lock()
8713                                        .map_err(|_| {
8714                                            ExcelError::new(ExcelErrorKind::Value).with_message(
8715                                                "compressed formula exact replay lock poisoned",
8716                                            )
8717                                        })?
8718                                        .replay_partitioned(
8719                                            &replay_disposition,
8720                                            std::slice::from_ref(&source),
8721                                        )
8722                                        .map_err(|message| {
8723                                            ExcelError::new(ExcelErrorKind::Value)
8724                                                .with_message(message)
8725                                        })?
8726                                        .into_iter()
8727                                        .filter(|record| {
8728                                            record.family == Some(source_id)
8729                                                || (record.family.is_none()
8730                                                    && record.partition_owner == Some(source_id))
8731                                        })
8732                                        .collect::<Vec<_>>();
8733                                    Self::validate_whole_partition_replay(&source, &records)?;
8734                                    Ok::<_, ExcelError>(records)
8735                                })();
8736                                let replayed = match replayed {
8737                                    Ok(replayed) => replayed,
8738                                    Err(error) => {
8739                                        self.publish_compressed_partial_report(
8740                                            &report,
8741                                            &direct_report,
8742                                        );
8743                                        return Err(error);
8744                                    }
8745                                };
8746                                let batch = match self.formula_batch_from_exact_replay(
8747                                    preparation.sheet_name.as_ref(),
8748                                    replayed,
8749                                ) {
8750                                    Ok(batch) => batch,
8751                                    Err(error) => {
8752                                        self.publish_compressed_partial_report(
8753                                            &report,
8754                                            &direct_report,
8755                                        );
8756                                        return Err(error);
8757                                    }
8758                                };
8759                                let snapshot = match self.fallback_planning_snapshot(&batch) {
8760                                    Ok(snapshot) => snapshot,
8761                                    Err(error) => {
8762                                        self.publish_compressed_partial_report(
8763                                            &report,
8764                                            &direct_report,
8765                                        );
8766                                        return Err(error);
8767                                    }
8768                                };
8769                                let commit_guard =
8770                                    crate::function_registry::semantic_epoch_read_guard();
8771                                let provider_revision = self.resolver.planning_semantic_revision();
8772                                if (commit_guard.epoch() != snapshot.epoch()
8773                                    && snapshot.semantic_changes_affect_requests_since_guarded(
8774                                        &commit_guard,
8775                                        snapshot.epoch(),
8776                                    ))
8777                                    || snapshot
8778                                        .provider_revision()
8779                                        .is_some_and(|revision| Some(revision) != provider_revision)
8780                                {
8781                                    self.publish_compressed_partial_report(&report, &direct_report);
8782                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8783                                        format!(
8784                                            "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
8785                                            snapshot.epoch(),
8786                                            commit_guard.epoch(),
8787                                            snapshot.provider_revision(),
8788                                            provider_revision,
8789                                        ),
8790                                    ));
8791                                }
8792                                let (plan, formula_count) =
8793                                    match self.prepare_legacy_batch_fallback(batch, &snapshot) {
8794                                        Ok(plan) => plan,
8795                                        Err(error) => {
8796                                            drop(commit_guard);
8797                                            self.publish_compressed_partial_report(
8798                                                &report,
8799                                                &direct_report,
8800                                            );
8801                                            return Err(error);
8802                                        }
8803                                    };
8804                                let provider_revision_after =
8805                                    self.resolver.planning_semantic_revision();
8806                                if provider_revision_after != provider_revision {
8807                                    drop(commit_guard);
8808                                    self.publish_compressed_partial_report(&report, &direct_report);
8809                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8810                                        .with_message(
8811                                            "function provider changed while preparing whole-family fallback",
8812                                        ));
8813                                }
8814                                let graph_vertices = plan.new_vertex_count();
8815                                let Some(graph_edges) = plan.planned_edge_count() else {
8816                                    drop(commit_guard);
8817                                    self.publish_compressed_partial_report(&report, &direct_report);
8818                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8819                                        .with_message(
8820                                            "prepared whole-family fallback size overflow",
8821                                        ));
8822                                };
8823                                if let Err(error) =
8824                                    self.prepared_legacy_admission(&plan, formula_count)
8825                                {
8826                                    drop(commit_guard);
8827                                    self.publish_compressed_partial_report(&report, &direct_report);
8828                                    return Err(error);
8829                                }
8830                                if let Err(error) =
8831                                    self.graph.validate_prepared_legacy_graph_plan(&plan)
8832                                {
8833                                    drop(commit_guard);
8834                                    self.publish_compressed_partial_report(&report, &direct_report);
8835                                    return Err(ExcelError::new(ExcelErrorKind::Value)
8836                                        .with_message(error.to_string()));
8837                                }
8838                                #[cfg(test)]
8839                                if let Some(hook) = self
8840                                    .before_legacy_fallback_final_provider_sample_hook
8841                                    .take()
8842                                {
8843                                    hook();
8844                                }
8845                                let provider_revision_final =
8846                                    self.resolver.planning_semantic_revision();
8847                                if provider_revision_final != provider_revision {
8848                                    drop(commit_guard);
8849                                    self.publish_compressed_partial_report(&report, &direct_report);
8850                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8851                                        "function provider changed after whole-family fallback validation",
8852                                    ));
8853                                }
8854                                let graph_formulas =
8855                                    self.graph.apply_prevalidated_legacy_graph_plan(plan);
8856                                direct_report.formula_cells_seen = direct_report
8857                                    .formula_cells_seen
8858                                    .saturating_add(formula_count);
8859                                direct_report.graph_formula_cells_materialized = direct_report
8860                                    .graph_formula_cells_materialized
8861                                    .saturating_add(graph_formulas as u64);
8862                                direct_report.graph_vertices_created = direct_report
8863                                    .graph_vertices_created
8864                                    .saturating_add(graph_vertices as u64);
8865                                direct_report.graph_edges_created = direct_report
8866                                    .graph_edges_created
8867                                    .saturating_add(graph_edges as u64);
8868                                direct_report.source_spool_replays =
8869                                    direct_report.source_spool_replays.saturating_add(1);
8870                                direct_report.source_family_fallback =
8871                                    direct_report.source_family_fallback.saturating_add(1);
8872                                direct_report.source_family_fallback_cells = direct_report
8873                                    .source_family_fallback_cells
8874                                    .saturating_add(source.surviving_member_count);
8875                                direct_report.source_partitioned_families_rejected = direct_report
8876                                    .source_partitioned_families_rejected
8877                                    .saturating_add(1);
8878                                direct_report.source_partition_failures =
8879                                    direct_report.source_partition_failures.saturating_add(1);
8880                                direct_report.source_partition_fallback_cells = direct_report
8881                                    .source_partition_fallback_cells
8882                                    .saturating_add(source.surviving_member_count);
8883                                direct_report.source_anchor_parses =
8884                                    direct_report.source_anchor_parses.saturating_add(1);
8885                                direct_report.source_anchor_asts =
8886                                    direct_report.source_anchor_asts.saturating_add(1);
8887                                direct_report.source_anchor_analyses =
8888                                    direct_report.source_anchor_analyses.saturating_add(1);
8889                                *direct_report.fallback_reasons.entry(reason).or_default() += 1;
8890                            }
8891                        }
8892                    }
8893                    #[cfg(test)]
8894                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
8895                        hook();
8896                    }
8897                }
8898            }
8899            break;
8900        }
8901        report.accumulate(&direct_report);
8902        self.record_formula_ingest_report(report.clone());
8903        Ok(report)
8904    }
8905    /// Ingest replayed per-cell formulas while preserving compressed source counters.
8906    pub(crate) fn ingest_compressed_formula_source_batches(
8907        &mut self,
8908        batches: Vec<(
8909            FormulaIngestBatch,
8910            crate::engine::FormulaCompressedSourceBatch,
8911        )>,
8912    ) -> Result<FormulaIngestReport, ExcelError> {
8913        self.ingest_compressed_formula_source_batches_inner(batches, true)
8914    }
8915
8916    fn ingest_compressed_formula_source_batches_inner(
8917        &mut self,
8918        batches: Vec<(
8919            FormulaIngestBatch,
8920            crate::engine::FormulaCompressedSourceBatch,
8921        )>,
8922        publish_report: bool,
8923    ) -> Result<FormulaIngestReport, ExcelError> {
8924        let mut source_counts = [0_u64; 11];
8925        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
8926        let mut formula_batches = Vec::with_capacity(batches.len());
8927        let mut compressed_families = Vec::new();
8928        let mut partitioned_families = Vec::new();
8929        for (batch, compressed_batch) in batches {
8930            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
8931            if sheet_name.as_ref() != batch.sheet_name {
8932                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8933                    "compressed formula source sheet does not match its replay batch",
8934                ));
8935            }
8936            compressed_families.push((batch.sheet_name.clone(), families));
8937            partitioned_families.push((batch.sheet_name.clone(), partitions));
8938            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
8939            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
8940            source_counts[2] =
8941                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
8942            source_counts[3] =
8943                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
8944            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
8945            source_counts[5] =
8946                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
8947            source_counts[6] =
8948                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
8949            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
8950            source_counts[8] =
8951                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
8952            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
8953            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
8954            source_report.families_seen = source_report
8955                .families_seen
8956                .saturating_add(compressed.families_seen);
8957            source_report.family_cells_seen = source_report
8958                .family_cells_seen
8959                .saturating_add(compressed.family_cells_seen);
8960            source_report.source_clean_families = source_report
8961                .source_clean_families
8962                .saturating_add(compressed.source_clean_families);
8963            source_report.source_clean_cells = source_report
8964                .source_clean_cells
8965                .saturating_add(compressed.source_clean_cells);
8966            source_report.source_fragmentable_families = source_report
8967                .source_fragmentable_families
8968                .saturating_add(compressed.source_fragmentable_families);
8969            source_report.source_fragmentable_cells = source_report
8970                .source_fragmentable_cells
8971                .saturating_add(compressed.source_fragmentable_cells);
8972            source_report.source_fragment_count = source_report
8973                .source_fragment_count
8974                .saturating_add(compressed.source_fragment_count);
8975            source_report.source_isolated_fallback_cells = source_report
8976                .source_isolated_fallback_cells
8977                .saturating_add(compressed.source_isolated_fallback_cells);
8978            source_report.source_hole_exclusions = source_report
8979                .source_hole_exclusions
8980                .saturating_add(compressed.source_hole_exclusions);
8981            source_report.source_ordinary_exclusions = source_report
8982                .source_ordinary_exclusions
8983                .saturating_add(compressed.source_ordinary_exclusions);
8984            source_report.source_partition_failures = source_report
8985                .source_partition_failures
8986                .saturating_add(compressed.source_partition_failures);
8987            source_report.replay_families = source_report
8988                .replay_families
8989                .saturating_add(compressed.replay_families);
8990            source_report.replay_cells = source_report
8991                .replay_cells
8992                .saturating_add(compressed.replay_cells);
8993            source_report.forward_descendants = source_report
8994                .forward_descendants
8995                .saturating_add(compressed.forward_descendants);
8996            source_report.evidence_limit_fallbacks = source_report
8997                .evidence_limit_fallbacks
8998                .saturating_add(compressed.evidence_limit_fallbacks);
8999            source_report.evidence_peak_bytes = source_report
9000                .evidence_peak_bytes
9001                .max(compressed.evidence_peak_bytes);
9002            for (reason, count) in compressed.fallback_reasons {
9003                *source_report.fallback_reasons.entry(reason).or_default() += count;
9004            }
9005            formula_batches.push(batch);
9006        }
9007        self.ingest_formula_batches_inner(
9008            formula_batches,
9009            source_counts,
9010            Some(source_report),
9011            compressed_families,
9012            partitioned_families,
9013            publish_report,
9014        )
9015    }
9016
9017    pub fn ingest_formula_batches(
9018        &mut self,
9019        batches: Vec<FormulaIngestBatch>,
9020    ) -> Result<FormulaIngestReport, ExcelError> {
9021        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
9022        let report = self.ingest_formula_batches_inner(
9023            batches,
9024            [0; 11],
9025            None,
9026            Vec::new(),
9027            Vec::new(),
9028            true,
9029        )?;
9030        if has_formulas {
9031            self.mark_topology_edited();
9032        }
9033        Ok(report)
9034    }
9035
9036    fn ingest_formula_batches_unpublished(
9037        &mut self,
9038        batches: Vec<FormulaIngestBatch>,
9039    ) -> Result<FormulaIngestReport, ExcelError> {
9040        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
9041    }
9042
9043    fn ingest_formula_batches_inner(
9044        &mut self,
9045        batches: Vec<FormulaIngestBatch>,
9046        source_counts: [u64; 11],
9047        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
9048        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
9049        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
9050        publish_report: bool,
9051    ) -> Result<FormulaIngestReport, ExcelError> {
9052        self.observe_function_semantic_epoch()?;
9053        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
9054        #[cfg(feature = "benchmark_internal")]
9055        let benchmark_forced_replay =
9056            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9057        #[cfg(not(feature = "benchmark_internal"))]
9058        let benchmark_forced_replay = false;
9059        let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9060            && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9061        {
9062            let mut report =
9063                FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9064            report
9065                .fallback_reasons
9066                .insert("ForcedReplay".to_string(), formula_cells_seen);
9067            (report, batches, BTreeMap::new())
9068        } else {
9069            match self.config.formula_plane_mode {
9070                FormulaPlaneMode::Off => (
9071                    FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9072                    batches,
9073                    BTreeMap::new(),
9074                ),
9075                FormulaPlaneMode::Shadow => (
9076                    if partitioned_families
9077                        .iter()
9078                        .all(|(_, families)| families.is_empty())
9079                    {
9080                        if compressed_families.is_empty() {
9081                            self.analyze_formula_plane_shadow_candidates(&batches)
9082                        } else {
9083                            self.analyze_compressed_formula_plane_shadow(&compressed_families)
9084                        }
9085                    } else {
9086                        let mut partition_report =
9087                            self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9088                        if compressed_families
9089                            .iter()
9090                            .any(|(_, families)| !families.is_empty())
9091                        {
9092                            partition_report.accumulate(
9093                                &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9094                            );
9095                        }
9096                        partition_report
9097                    },
9098                    batches,
9099                    BTreeMap::new(),
9100                ),
9101                FormulaPlaneMode::AuthoritativeExperimental => {
9102                    self.analyze_formula_plane_authoritative_ingest(&batches)
9103                }
9104            }
9105        };
9106        report.formula_cells_seen = formula_cells_seen;
9107        report.source_formula_events = source_counts[0];
9108        report.source_ordinary_events = source_counts[1];
9109        report.source_shared_anchor_events = source_counts[2];
9110        report.source_shared_descendant_events = source_counts[3];
9111        report.source_unknown_events = source_counts[4];
9112        report.source_formula_records_spooled = source_counts[5];
9113        report.source_spool_encoded_bytes = source_counts[6];
9114        report.source_spool_peak_memory_bytes = source_counts[7];
9115        report.source_spool_spilled_bytes = source_counts[8];
9116        report.source_spool_spill_files = source_counts[9];
9117        report.source_spool_replays = source_counts[10];
9118        if let Some(source) = source_report {
9119            report.source_families_seen = source.families_seen;
9120            report.source_family_cells_seen = source.family_cells_seen;
9121            report.source_family_shadow_eligible = source.source_clean_families;
9122            report.source_family_shadow_eligible_cells = source.source_clean_cells;
9123            report.source_partitioned_families_seen = source.source_fragmentable_families;
9124            report.source_partition_holes = source.source_hole_exclusions;
9125            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9126            report.source_partition_failures = source.source_partition_failures;
9127            report.source_partition_surviving_cells = source.source_fragmentable_cells;
9128            report.source_family_fallback = report
9129                .source_family_fallback
9130                .saturating_add(source.replay_families);
9131            report.source_family_fallback_cells = report
9132                .source_family_fallback_cells
9133                .saturating_add(source.replay_cells);
9134            report.source_forward_descendants = source.forward_descendants;
9135            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9136            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9137            for (reason, count) in source.fallback_reasons {
9138                let total = report.fallback_reasons.entry(reason).or_default();
9139                *total = total.saturating_add(count);
9140            }
9141        }
9142
9143        if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9144            || !planned_materialize.is_empty()
9145        {
9146            let mut prepared_by_sheet = planned_materialize;
9147            for batch in materialize_batches {
9148                if batch.is_empty() {
9149                    continue;
9150                }
9151                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9152                    ExcelError::new(ExcelErrorKind::Ref)
9153                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9154                })?;
9155                let mut pipeline = self.ingest_pipeline();
9156                let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9157                    let placement = CellRef::new(
9158                        sheet_id,
9159                        Coord::from_excel(record.row, record.col, true, true),
9160                    );
9161                    (
9162                        FormulaAstInput::RawArena(record.ast_id),
9163                        placement,
9164                        record.formula_text,
9165                    )
9166                }))?;
9167                prepared_by_sheet
9168                    .entry(batch.sheet_name)
9169                    .or_default()
9170                    .extend(ingested.into_iter().map(|formula| {
9171                        (
9172                            formula.placement.coord.row() + 1,
9173                            formula.placement.coord.col() + 1,
9174                            formula.ast_id,
9175                            formula.dep_plan,
9176                        )
9177                    }));
9178            }
9179            let admission_preflighted = self.graph_admission_enabled();
9180            if admission_preflighted {
9181                let mut preview = Vec::new();
9182                for (sheet_name, formulas) in &prepared_by_sheet {
9183                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9184                        ExcelError::new(ExcelErrorKind::Ref)
9185                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
9186                    })?;
9187                    preview.extend(
9188                        formulas
9189                            .iter()
9190                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9191                    );
9192                }
9193                let admission = self.graph.preview_formula_mutations(&preview)?;
9194                self.preflight_graph_admission(admission)?;
9195            }
9196
9197            let mut builder = self.begin_bulk_ingest();
9198            if admission_preflighted {
9199                builder.mark_admission_preflighted();
9200            }
9201            for (sheet_name, formulas) in prepared_by_sheet {
9202                if formulas.is_empty() {
9203                    continue;
9204                }
9205                let sheet_id = builder.add_sheet(&sheet_name);
9206                builder.add_formula_plans(sheet_id, formulas);
9207            }
9208            let summary = builder.finish()?;
9209            report.graph_formula_cells_materialized = summary.formulas as u64;
9210            report.graph_vertices_created = summary.vertices as u64;
9211            report.graph_edges_created = summary.edges as u64;
9212        }
9213
9214        if publish_report {
9215            self.record_formula_ingest_report(report.clone());
9216        }
9217        Ok(report)
9218    }
9219
9220    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9221        let mut unique = Vec::new();
9222        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9223            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9224                prior.sheet == diagnostic.sheet
9225                    && prior.row == diagnostic.row
9226                    && prior.col == diagnostic.col
9227                    && prior.formula == diagnostic.formula
9228                    && prior.policy == diagnostic.policy
9229            });
9230            if !duplicate {
9231                unique.push(diagnostic);
9232            }
9233        }
9234        self.formula_parse_diagnostics.extend(unique);
9235    }
9236
9237    pub fn handle_formula_parse_error(
9238        &mut self,
9239        sheet: &str,
9240        row: u32,
9241        col: u32,
9242        formula: &str,
9243        message: String,
9244    ) -> Result<Option<ASTNode>, ExcelError> {
9245        let policy = self.config.formula_parse_policy;
9246
9247        if policy == FormulaParsePolicy::Strict {
9248            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9249            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9250                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9251            )));
9252        }
9253
9254        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9255            sheet: sheet.to_string(),
9256            row,
9257            col,
9258            formula: formula.to_string(),
9259            message: message.clone(),
9260            policy,
9261        });
9262
9263        match policy {
9264            FormulaParsePolicy::Strict => unreachable!(),
9265            FormulaParsePolicy::KeepCachedValue => Ok(None),
9266            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9267                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9268                None,
9269            ))),
9270            FormulaParsePolicy::CoerceToError => {
9271                let err = ExcelError::new(ExcelErrorKind::Error)
9272                    .with_message(format!("Malformed formula: {message}"));
9273                Ok(Some(ASTNode::new(
9274                    ASTNodeType::Literal(LiteralValue::Error(err)),
9275                    None,
9276                )))
9277            }
9278        }
9279    }
9280
9281    #[cfg(test)]
9282    fn target_preparation_fault(
9283        &mut self,
9284        seam: crate::engine::target_preparation::TargetPreparationFault,
9285    ) -> Result<(), ExcelError> {
9286        if self.target_preparation_fault_for_test == Some(seam) {
9287            self.target_preparation_fault_for_test = None;
9288            Err(ExcelError::new(ExcelErrorKind::Value)
9289                .with_message(format!("injected target preparation fault: {seam:?}")))
9290        } else {
9291            Ok(())
9292        }
9293    }
9294
9295    fn preparation_stale(
9296        reason: formualizer_common::PreparationStaleReason,
9297        message: impl Into<String>,
9298    ) -> ExcelError {
9299        ExcelError::new(ExcelErrorKind::Value)
9300            .with_message(message)
9301            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
9302    }
9303
9304    fn preparation_revision_stale_reason(
9305        assumptions: &crate::engine::PreparationRevision,
9306        current: &crate::engine::PreparationRevision,
9307        planning_requests: &BTreeSet<(String, String, usize)>,
9308        staged_leases_match: bool,
9309    ) -> Option<formualizer_common::PreparationStaleReason> {
9310        if assumptions.graph != current.graph {
9311            Some(formualizer_common::PreparationStaleReason::Graph)
9312        } else if assumptions.authority != current.authority
9313            || assumptions.authority_indexes != current.authority_indexes
9314            || assumptions.authority_indexed_plane != current.authority_indexed_plane
9315        {
9316            Some(formualizer_common::PreparationStaleReason::Authority)
9317        } else if assumptions.staged != current.staged || !staged_leases_match {
9318            Some(formualizer_common::PreparationStaleReason::Staged)
9319        } else if assumptions.symbols != current.symbols {
9320            Some(formualizer_common::PreparationStaleReason::Symbols)
9321        } else if assumptions.provider != current.provider {
9322            Some(formualizer_common::PreparationStaleReason::Provider)
9323        } else if assumptions.semantic != current.semantic
9324            && crate::function_registry::semantic_changes_affect_requests_since(
9325                assumptions.semantic,
9326                planning_requests.iter().cloned(),
9327            )
9328        {
9329            Some(formualizer_common::PreparationStaleReason::Semantic)
9330        } else {
9331            None
9332        }
9333    }
9334
9335    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
9336        let (authority, authority_indexes, authority_indexed_plane) =
9337            self.graph.authority_revisions();
9338        crate::engine::PreparationRevision {
9339            graph: self.graph.topology_revision(),
9340            authority,
9341            authority_indexes,
9342            authority_indexed_plane,
9343            staged: self.staged_formula_index.revision(),
9344            symbols: self.graph.symbol_revision(),
9345            semantic: crate::function_registry::semantic_epoch(),
9346            provider: self.resolver.planning_semantic_revision(),
9347        }
9348    }
9349
9350    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
9351        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
9352        let provider = self.resolver.planning_semantic_revision();
9353        let semantic = registry_guard.epoch();
9354        let (authority, authority_indexes, authority_indexed_plane) =
9355            self.graph.authority_revisions();
9356        let mut span_refs = self.graph.formula_authority().active_span_refs();
9357        span_refs.sort_unstable_by_key(|span_ref| {
9358            (span_ref.id.0, span_ref.generation, span_ref.version)
9359        });
9360        PlanningRevisionSnapshot {
9361            engine_topology_epoch: self.topology_epoch,
9362            graph_topology_revision: self.graph.topology_revision(),
9363            authority,
9364            authority_indexes,
9365            authority_indexed_plane,
9366            staged: self.staged_formula_index.revision(),
9367            symbols: self.graph.symbol_revision(),
9368            semantic,
9369            provider,
9370            formula_plane_mode: self.config.formula_plane_mode,
9371            deterministic_mode: self.config.deterministic_mode.clone(),
9372            budgets: self.evaluation_resource_budgets.clone(),
9373            span_refs,
9374        }
9375    }
9376
9377    fn recalc_plan_key(&self) -> RecalcPlanKey {
9378        RecalcPlanKey {
9379            engine_token: Arc::clone(&self.recalc_plan_token),
9380            revisions: self.planning_revision_snapshot(),
9381        }
9382    }
9383
9384    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
9385        ExcelError::new(ExcelErrorKind::Value)
9386            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
9387            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
9388    }
9389
9390    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
9391        use formualizer_common::PlanStaleReason;
9392
9393        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
9394            return Err(Self::plan_stale(PlanStaleReason::Engine));
9395        }
9396
9397        let current = self.planning_revision_snapshot();
9398        let expected = &key.revisions;
9399        let stale = if expected.provider != current.provider {
9400            Some(PlanStaleReason::Provider)
9401        } else if expected.semantic != current.semantic {
9402            Some(PlanStaleReason::Semantic)
9403        } else if expected.budgets != current.budgets
9404            || expected.deterministic_mode != current.deterministic_mode
9405        {
9406            Some(PlanStaleReason::Budget)
9407        } else if expected.staged != current.staged {
9408            Some(PlanStaleReason::Staged)
9409        } else if expected.symbols != current.symbols {
9410            Some(PlanStaleReason::Symbols)
9411        } else if expected.formula_plane_mode != current.formula_plane_mode
9412            || expected.authority != current.authority
9413            || expected.authority_indexes != current.authority_indexes
9414            || expected.authority_indexed_plane != current.authority_indexed_plane
9415        {
9416            Some(PlanStaleReason::Authority)
9417        } else if expected.span_refs != current.span_refs {
9418            Some(PlanStaleReason::SpanGeneration)
9419        } else if expected.graph_topology_revision != current.graph_topology_revision
9420            || expected.engine_topology_epoch != current.engine_topology_epoch
9421        {
9422            Some(PlanStaleReason::Graph)
9423        } else {
9424            None
9425        };
9426        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
9427    }
9428
9429    fn target_preparation_checkpoint(
9430        &mut self,
9431        deadline: Option<std::time::Instant>,
9432        work: u64,
9433    ) -> Result<(), ExcelError> {
9434        if self
9435            .active_cancel_flag
9436            .as_ref()
9437            .is_some_and(|cancel| cancel.is_cancelled())
9438        {
9439            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
9440                .with_message("target graph preparation cancelled"));
9441        }
9442        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
9443            return Err(crate::engine::ResourceLedgerError::Exhausted(
9444                formualizer_common::ResourceExhaustionDetail {
9445                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
9446                    limit: 0,
9447                    observed: 1,
9448                    request_id: self
9449                        .active_evaluation_resource_request
9450                        .as_ref()
9451                        .map(|stats| stats.request_id),
9452                },
9453            )
9454            .into_excel_error());
9455        }
9456        self.charge_bounded_work(work)
9457    }
9458
9459    fn opaque_reason_in_ast(
9460        &self,
9461        ast: &ASTNode,
9462        provider: &dyn crate::traits::FunctionProvider,
9463    ) -> Option<crate::engine::OpaqueReason> {
9464        match &ast.node_type {
9465            ASTNodeType::Function { name, args } => {
9466                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
9467                if canonical == "INDIRECT" {
9468                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
9469                }
9470                let Some(function) = provider.get_function_for_planning("", name) else {
9471                    return Some(crate::engine::OpaqueReason::UnknownFunction);
9472                };
9473                let caps = function.caps();
9474                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9475                    || caps.contains(FnCaps::RETURNS_REFERENCE)
9476                {
9477                    return Some(crate::engine::OpaqueReason::DynamicReference);
9478                }
9479                args.iter()
9480                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
9481            }
9482            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
9483            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
9484            ASTNodeType::BinaryOp { left, right, .. } => self
9485                .opaque_reason_in_ast(left, provider)
9486                .or_else(|| self.opaque_reason_in_ast(right, provider)),
9487            ASTNodeType::Array(rows) => rows
9488                .iter()
9489                .flat_map(|row| row.iter())
9490                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
9491            ASTNodeType::Reference {
9492                reference:
9493                    ReferenceType::Cell {
9494                        sheet: Some(sheet), ..
9495                    }
9496                    | ReferenceType::Range {
9497                        sheet: Some(sheet), ..
9498                    },
9499                ..
9500            } if self.graph.sheet_id(sheet).is_none() => {
9501                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
9502            }
9503            ASTNodeType::Reference {
9504                reference:
9505                    ReferenceType::External(_)
9506                    | ReferenceType::Cell3D { .. }
9507                    | ReferenceType::Range3D { .. },
9508                ..
9509            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
9510            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
9511        }
9512    }
9513
9514    fn target_planning_snapshot(
9515        &mut self,
9516        ast: &ASTNode,
9517        planning_requests: &mut BTreeSet<(String, String, usize)>,
9518    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
9519        #[cfg(test)]
9520        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
9521            return Err(Self::preparation_stale(
9522                formualizer_common::PreparationStaleReason::Semantic,
9523                "injected target semantic preparation movement",
9524            ));
9525        }
9526        #[cfg(test)]
9527        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
9528            hook();
9529        }
9530        let mut requests = Vec::new();
9531        Self::collect_planning_function_requests(ast, &mut requests);
9532        requests.sort();
9533        requests.dedup();
9534        planning_requests.extend(requests.iter().cloned());
9535        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
9536            &self.resolver,
9537            requests,
9538        )
9539        .map_err(|error| {
9540            ExcelError::new(ExcelErrorKind::Value)
9541                .with_message(format!("target planning snapshot unavailable: {error:?}"))
9542        })
9543    }
9544
9545    fn target_planning_snapshot_stale_reason(
9546        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
9547        assumptions: &crate::engine::PreparationRevision,
9548    ) -> Option<formualizer_common::PreparationStaleReason> {
9549        if snapshot
9550            .provider_revision()
9551            .is_some_and(|revision| Some(revision) != assumptions.provider)
9552        {
9553            Some(formualizer_common::PreparationStaleReason::Provider)
9554        } else if snapshot.epoch() != assumptions.semantic
9555            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
9556        {
9557            Some(formualizer_common::PreparationStaleReason::Semantic)
9558        } else {
9559            None
9560        }
9561    }
9562
9563    fn widen_target_preparation(
9564        policy: crate::engine::OpaquePreparePolicy,
9565        scope: &mut crate::engine::PrepareScope,
9566        reasons: &mut Vec<crate::engine::OpaqueReason>,
9567        reason: crate::engine::OpaqueReason,
9568    ) -> Result<bool, ExcelError> {
9569        if policy == crate::engine::OpaquePreparePolicy::Error {
9570            return Err(ExcelError::new(ExcelErrorKind::NImpl)
9571                .with_message(format!("opaque target preparation semantics: {reason:?}")));
9572        }
9573        if !reasons.contains(&reason) {
9574            reasons.push(reason);
9575        }
9576        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
9577            *scope = crate::engine::PrepareScope::Workbook;
9578            Ok(true)
9579        } else {
9580            Ok(false)
9581        }
9582    }
9583
9584    fn widen_target_preparation_to_sheet(
9585        policy: crate::engine::OpaquePreparePolicy,
9586        scope: &mut crate::engine::PrepareScope,
9587        reasons: &mut Vec<crate::engine::OpaqueReason>,
9588        reason: crate::engine::OpaqueReason,
9589        sheet: &str,
9590    ) -> Result<bool, ExcelError> {
9591        if policy == crate::engine::OpaquePreparePolicy::Error {
9592            return Err(ExcelError::new(ExcelErrorKind::NImpl)
9593                .with_message(format!("opaque target preparation semantics: {reason:?}")));
9594        }
9595        if !reasons.contains(&reason) {
9596            reasons.push(reason);
9597        }
9598        match scope {
9599            crate::engine::PrepareScope::Exact => {
9600                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
9601                Ok(true)
9602            }
9603            crate::engine::PrepareScope::Sheets(sheets) => {
9604                if sheets.iter().any(|candidate| candidate == sheet) {
9605                    Ok(false)
9606                } else {
9607                    sheets.push(sheet.to_string());
9608                    sheets.sort();
9609                    Ok(true)
9610                }
9611            }
9612            crate::engine::PrepareScope::Workbook => Ok(false),
9613        }
9614    }
9615
9616    fn ast_has_proven_sheet_local_dynamic(
9617        ast: &ASTNode,
9618        provider: &dyn crate::traits::FunctionProvider,
9619    ) -> bool {
9620        fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
9621            match &ast.node_type {
9622                ASTNodeType::Function { name, args } => {
9623                    let Some(function) = provider.get_function_for_planning("", name) else {
9624                        return (false, false);
9625                    };
9626                    let caps = function.caps();
9627                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
9628                        || caps.contains(FnCaps::RETURNS_REFERENCE);
9629                    let canonical = name.rsplit('.').next().unwrap_or(name);
9630                    if dynamic
9631                        && !canonical.eq_ignore_ascii_case("OFFSET")
9632                        && !canonical.eq_ignore_ascii_case("INDEX")
9633                    {
9634                        return (false, true);
9635                    }
9636                    let mut has_dynamic = dynamic;
9637                    for arg in args {
9638                        let (safe, child_dynamic) = classify(arg, provider);
9639                        if !safe {
9640                            return (false, has_dynamic || child_dynamic);
9641                        }
9642                        has_dynamic |= child_dynamic;
9643                    }
9644                    (true, has_dynamic)
9645                }
9646                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
9647                ASTNodeType::BinaryOp { left, right, .. } => {
9648                    let (left_safe, left_dynamic) = classify(left, provider);
9649                    let (right_safe, right_dynamic) = classify(right, provider);
9650                    (left_safe && right_safe, left_dynamic || right_dynamic)
9651                }
9652                ASTNodeType::Array(rows) => {
9653                    let mut has_dynamic = false;
9654                    for item in rows.iter().flatten() {
9655                        let (safe, child_dynamic) = classify(item, provider);
9656                        if !safe {
9657                            return (false, has_dynamic || child_dynamic);
9658                        }
9659                        has_dynamic |= child_dynamic;
9660                    }
9661                    (true, has_dynamic)
9662                }
9663                ASTNodeType::Reference {
9664                    reference:
9665                        ReferenceType::Cell { sheet: None, .. }
9666                        | ReferenceType::Range { sheet: None, .. },
9667                    ..
9668                }
9669                | ASTNodeType::Literal(_)
9670                | ASTNodeType::Omitted => (true, false),
9671                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
9672            }
9673        }
9674
9675        let (safe, dynamic) = classify(ast, provider);
9676        safe && dynamic
9677    }
9678
9679    fn table_selection_region(
9680        &self,
9681        entry: &crate::engine::graph::TableEntry,
9682        selection: &crate::engine::TableSelection,
9683    ) -> Result<PreparationRegion, ExcelError> {
9684        let mut start_row = entry.range.start.coord.row() + 1;
9685        let mut end_row = entry.range.end.coord.row() + 1;
9686        let mut start_col = entry.range.start.coord.col() + 1;
9687        let mut end_col = entry.range.end.coord.col() + 1;
9688        match selection {
9689            crate::engine::TableSelection::Whole => {}
9690            crate::engine::TableSelection::Headers => {
9691                if !entry.header_row {
9692                    return Err(ExcelError::new(ExcelErrorKind::Value)
9693                        .with_message(format!("table {} has no header row", entry.name)));
9694                }
9695                end_row = start_row;
9696            }
9697            crate::engine::TableSelection::Data => {
9698                if entry.header_row {
9699                    start_row = start_row.saturating_add(1);
9700                }
9701                if entry.totals_row {
9702                    end_row = end_row.saturating_sub(1);
9703                }
9704            }
9705            crate::engine::TableSelection::Totals => {
9706                if !entry.totals_row {
9707                    return Err(ExcelError::new(ExcelErrorKind::Value)
9708                        .with_message(format!("table {} has no totals row", entry.name)));
9709                }
9710                start_row = end_row;
9711            }
9712            crate::engine::TableSelection::Column(column) => {
9713                let index = entry.col_index(column).ok_or_else(|| {
9714                    ExcelError::new(ExcelErrorKind::Name)
9715                        .with_message(format!("unknown table column: {column}"))
9716                })?;
9717                start_col = start_col.saturating_add(index as u32);
9718                end_col = start_col;
9719            }
9720            crate::engine::TableSelection::Columns { start, end } => {
9721                let first = entry.col_index(start).ok_or_else(|| {
9722                    ExcelError::new(ExcelErrorKind::Name)
9723                        .with_message(format!("unknown table column: {start}"))
9724                })?;
9725                let last = entry.col_index(end).ok_or_else(|| {
9726                    ExcelError::new(ExcelErrorKind::Name)
9727                        .with_message(format!("unknown table column: {end}"))
9728                })?;
9729                if first > last {
9730                    return Err(ExcelError::new(ExcelErrorKind::Value)
9731                        .with_message("table column selection is reversed"));
9732                }
9733                start_col = start_col.saturating_add(first as u32);
9734                end_col = entry
9735                    .range
9736                    .start
9737                    .coord
9738                    .col()
9739                    .saturating_add(last as u32)
9740                    .saturating_add(1);
9741            }
9742        }
9743        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
9744            start_row = entry.range.start.coord.row() + 1;
9745            end_row = start_row;
9746        }
9747        if start_row > end_row || start_col > end_col {
9748            return Err(
9749                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
9750            );
9751        }
9752        Ok(PreparationRegion {
9753            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
9754            sheet_id: entry.sheet_id(),
9755            start_row,
9756            start_col,
9757            end_row,
9758            end_col,
9759        })
9760    }
9761
9762    #[cfg(test)]
9763    pub(crate) fn reset_target_root_dedup_probes_for_test() {
9764        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
9765    }
9766
9767    #[cfg(test)]
9768    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
9769        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
9770    }
9771
9772    pub(crate) fn resolve_target_producers(
9773        &mut self,
9774        targets: &[crate::engine::EvaluationTarget],
9775    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9776        use crate::engine::target_preparation::TargetProducer;
9777
9778        let request_id = self
9779            .active_evaluation_resource_request
9780            .as_ref()
9781            .map(|request| request.request_id);
9782        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
9783            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
9784        let active_span_refs = self
9785            .graph
9786            .formula_authority()
9787            .active_span_refs()
9788            .into_iter()
9789            .map(|span_ref| (span_ref.id, span_ref))
9790            .collect::<BTreeMap<_, _>>();
9791        let resolve_region = |engine: &mut Self,
9792                              region: Region,
9793                              value_only: Option<CellRef>,
9794                              roots: &mut OrderedTargetProducers|
9795         -> Result<(), ExcelError> {
9796            let before = roots.len();
9797            let authority = engine.graph.formula_authority();
9798            for matched in authority.producer_results.query(region).matches {
9799                if let FormulaProducerId::Span(span_id) = matched.value.producer
9800                    && let Some(span_ref) = active_span_refs.get(&span_id).copied()
9801                    && let Some(demanded) = matched.value.result_region.intersection(region)
9802                {
9803                    roots
9804                        .push(TargetProducer::Span { span_ref, demanded })
9805                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9806                }
9807            }
9808            for anchor in engine.graph.spill_anchors_in_region(
9809                region.sheet_id(),
9810                region.axis_ranges().0.query_bounds().0,
9811                region.axis_ranges().1.query_bounds().0,
9812                region.axis_ranges().0.query_bounds().1,
9813                region.axis_ranges().1.query_bounds().1,
9814            ) {
9815                roots
9816                    .push(TargetProducer::Legacy(anchor))
9817                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9818            }
9819            for vertex in engine.graph.vertices_in_region(
9820                region.sheet_id(),
9821                region.axis_ranges().0.query_bounds().0,
9822                region.axis_ranges().0.query_bounds().1,
9823                region.axis_ranges().1.query_bounds().0,
9824                region.axis_ranges().1.query_bounds().1,
9825            ) {
9826                let vertex = engine
9827                    .graph
9828                    .get_cell_ref(vertex)
9829                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
9830                    .unwrap_or(vertex);
9831                match engine.graph.get_vertex_kind(vertex) {
9832                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
9833                        roots.push(TargetProducer::Legacy(vertex)).map_err(|_| {
9834                            target_root_allocation_error(roots.len() + 1, request_id)
9835                        })?;
9836                    }
9837                    VertexKind::NamedScalar
9838                    | VertexKind::NamedArray
9839                    | VertexKind::Range
9840                    | VertexKind::InfiniteRange
9841                    | VertexKind::Table => {
9842                        roots.push(TargetProducer::Symbol(vertex)).map_err(|_| {
9843                            target_root_allocation_error(roots.len() + 1, request_id)
9844                        })?;
9845                    }
9846                    VertexKind::Empty | VertexKind::Cell | VertexKind::External => {}
9847                }
9848            }
9849            if roots.len() == before
9850                && let Some(cell) = value_only
9851            {
9852                roots
9853                    .push(TargetProducer::ValueOnly(cell))
9854                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
9855            }
9856            Ok(())
9857        };
9858
9859        for target in targets {
9860            match target {
9861                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
9862                    if *row == 0 || *col == 0 {
9863                        return Err(ExcelError::new(ExcelErrorKind::Ref)
9864                            .with_message("target cell coordinates are one-based"));
9865                    }
9866                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
9867                        ExcelError::new(ExcelErrorKind::Ref)
9868                            .with_message(format!("target sheet not found: {sheet}"))
9869                    })?;
9870                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
9871                    resolve_region(
9872                        self,
9873                        Region::point(sheet_id, *row - 1, *col - 1),
9874                        Some(cell),
9875                        &mut roots,
9876                    )?;
9877                }
9878                crate::engine::EvaluationTarget::Range(range) => {
9879                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
9880                        ExcelError::new(ExcelErrorKind::Ref)
9881                            .with_message(format!("target sheet not found: {}", range.sheet))
9882                    })?;
9883                    resolve_region(
9884                        self,
9885                        Region::rect(
9886                            sheet_id,
9887                            range.start_row - 1,
9888                            range.end_row - 1,
9889                            range.start_col - 1,
9890                            range.end_col - 1,
9891                        ),
9892                        None,
9893                        &mut roots,
9894                    )?;
9895                }
9896                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
9897                    let scope = self.name_query_scope(scope_sheet.as_deref())?;
9898                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
9899                        roots
9900                            .push(TargetProducer::Symbol(entry.vertex))
9901                            .map_err(|_| {
9902                                target_root_allocation_error(roots.len() + 1, request_id)
9903                            })?;
9904                    }
9905                }
9906                crate::engine::EvaluationTarget::Table { name, .. } => {
9907                    if let Some(entry) = self.graph.resolve_table_entry(name) {
9908                        roots
9909                            .push(TargetProducer::Symbol(entry.vertex))
9910                            .map_err(|_| {
9911                                target_root_allocation_error(roots.len() + 1, request_id)
9912                            })?;
9913                    }
9914                }
9915            }
9916        }
9917        Ok(roots.into_vec())
9918    }
9919
9920    fn resolve_target_producers_from_existing_roots(
9921        &mut self,
9922        roots: &[crate::engine::target_preparation::TargetProducer],
9923    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
9924        use crate::engine::target_preparation::TargetProducer;
9925        let request_id = self
9926            .active_evaluation_resource_request
9927            .as_ref()
9928            .map(|request| request.request_id);
9929        let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
9930            .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
9931        for root in roots {
9932            match *root {
9933                TargetProducer::Span { demanded, .. } => {
9934                    let (rows, cols) = demanded.axis_ranges();
9935                    for vertex in self.graph.vertices_in_region(
9936                        demanded.sheet_id(),
9937                        rows.query_bounds().0,
9938                        rows.query_bounds().1,
9939                        cols.query_bounds().0,
9940                        cols.query_bounds().1,
9941                    ) {
9942                        if matches!(
9943                            self.graph.get_vertex_kind(vertex),
9944                            VertexKind::FormulaScalar | VertexKind::FormulaArray
9945                        ) {
9946                            refreshed
9947                                .push(TargetProducer::Legacy(vertex))
9948                                .map_err(|_| {
9949                                    target_root_allocation_error(refreshed.len() + 1, request_id)
9950                                })?;
9951                        }
9952                    }
9953                }
9954                other => {
9955                    refreshed.push(other).map_err(|_| {
9956                        target_root_allocation_error(refreshed.len() + 1, request_id)
9957                    })?;
9958                }
9959            }
9960        }
9961        Ok(refreshed.into_vec())
9962    }
9963
9964    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
9965    ///
9966    /// This method prepares graph topology only. It does not evaluate target values.
9967    pub fn prepare_graph_for_targets(
9968        &mut self,
9969        targets: &[crate::engine::EvaluationTarget],
9970        options: crate::engine::TargetEvalOptions<'_>,
9971    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
9972        let previous_budgets = self.evaluation_resource_budgets.clone();
9973        let diagnostics_len = self.formula_parse_diagnostics.len();
9974        let previous_report = self.last_formula_ingest_report.clone();
9975        if let Some(budgets) = options.budgets {
9976            self.evaluation_resource_budgets = budgets.clone();
9977        }
9978        let previous_graph_budget_override = self
9979            .graph
9980            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
9981        // Hoist the call's cancellation onto the engine for its duration, so the
9982        // preparation checkpoints observe it. This is a standalone entry point, so
9983        // the previous value is restored rather than cleared.
9984        let previous_cancel = self.active_cancel_flag.take();
9985        self.active_cancel_flag = options.cancel.clone();
9986        let result = self.observe_evaluation_resource_request(
9987            EvaluationRequestKind::TargetPreparation,
9988            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
9989        );
9990        self.active_cancel_flag = previous_cancel;
9991        self.graph
9992            .set_admission_budget_override(previous_graph_budget_override);
9993        self.evaluation_resource_budgets = previous_budgets;
9994        if result.is_err() {
9995            self.formula_parse_diagnostics.truncate(diagnostics_len);
9996            self.last_formula_ingest_report = previous_report;
9997        }
9998        result
9999    }
10000
10001    fn prepare_graph_for_targets_unobserved(
10002        &mut self,
10003        targets: &[crate::engine::EvaluationTarget],
10004        options: &crate::engine::TargetEvalOptions<'_>,
10005    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10006        let scratch_checkpoint = self
10007            .active_resource_ledger
10008            .as_ref()
10009            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
10010        let result = self.prepare_graph_for_targets_transaction(targets, options);
10011        let release = self
10012            .active_resource_ledger
10013            .as_mut()
10014            .map_or(Ok(()), |ledger| {
10015                ledger.release_scratch_to(scratch_checkpoint)
10016            });
10017        match (result, release) {
10018            (result, Ok(())) => result,
10019            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
10020        }
10021    }
10022
10023    fn prepare_graph_for_targets_transaction(
10024        &mut self,
10025        targets: &[crate::engine::EvaluationTarget],
10026        options: &crate::engine::TargetEvalOptions<'_>,
10027    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10028        use crate::engine::{
10029            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
10030            TableSelection,
10031        };
10032
10033        self.target_preparation_checkpoint(options.deadline, 0)?;
10034        self.observe_function_semantic_epoch()?;
10035        let assumptions = self.preparation_revisions();
10036        let ledger_at_start = self
10037            .active_resource_ledger
10038            .as_ref()
10039            .map(|ledger| ledger.snapshot());
10040        let diagnostics_len = self.formula_parse_diagnostics.len();
10041        let report_len = self.last_formula_ingest_report.clone();
10042        let request_id = options.request_id.or_else(|| {
10043            self.active_evaluation_resource_request
10044                .as_ref()
10045                .map(|stats| stats.request_id)
10046        });
10047
10048        let mut scope = PrepareScope::Exact;
10049        let mut reasons = Vec::new();
10050        let mut regions = VecDeque::new();
10051        let mut normalized = Vec::with_capacity(targets.len());
10052        let mut symbol_vertices = VecDeque::new();
10053        for target in targets {
10054            self.target_preparation_checkpoint(options.deadline, 1)?;
10055            match target {
10056                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10057                    if *row == 0 || *col == 0 {
10058                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10059                            .with_message("target cell coordinates are one-based"));
10060                    }
10061                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10062                        ExcelError::new(ExcelErrorKind::Ref)
10063                            .with_message(format!("target sheet not found: {sheet}"))
10064                    })?;
10065                    regions.push_back(PreparationRegion {
10066                        sheet: sheet.clone(),
10067                        sheet_id,
10068                        start_row: *row,
10069                        start_col: *col,
10070                        end_row: *row,
10071                        end_col: *col,
10072                    });
10073                    normalized.push(target.clone());
10074                }
10075                crate::engine::EvaluationTarget::Range(range) => {
10076                    if range.start_row == 0
10077                        || range.start_col == 0
10078                        || range.end_row < range.start_row
10079                        || range.end_col < range.start_col
10080                    {
10081                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10082                            .with_message("invalid target range"));
10083                    }
10084                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10085                        ExcelError::new(ExcelErrorKind::Ref)
10086                            .with_message(format!("target sheet not found: {}", range.sheet))
10087                    })?;
10088                    regions.push_back(PreparationRegion {
10089                        sheet: range.sheet.clone(),
10090                        sheet_id,
10091                        start_row: range.start_row,
10092                        start_col: range.start_col,
10093                        end_row: range.end_row,
10094                        end_col: range.end_col,
10095                    });
10096                    normalized.push(target.clone());
10097                }
10098                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10099                    let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
10100                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
10101                        symbol_vertices.push_back(entry.vertex);
10102                    } else {
10103                        Self::widen_target_preparation(
10104                            options.opaque_policy,
10105                            &mut scope,
10106                            &mut reasons,
10107                            OpaqueReason::UnresolvedName,
10108                        )?;
10109                    }
10110                    normalized.push(target.clone());
10111                }
10112                crate::engine::EvaluationTarget::Table { name, selection } => {
10113                    if let Some(entry) = self.graph.resolve_table_entry(name) {
10114                        let region = self.table_selection_region(entry, selection)?;
10115                        symbol_vertices.push_back(entry.vertex);
10116                        regions.push_back(region);
10117                    } else {
10118                        Self::widen_target_preparation(
10119                            options.opaque_policy,
10120                            &mut scope,
10121                            &mut reasons,
10122                            OpaqueReason::UnresolvedTable,
10123                        )?;
10124                    }
10125                    normalized.push(target.clone());
10126                }
10127            }
10128        }
10129
10130        let mut visited_regions = FxHashSet::default();
10131        let mut visited_vertices = FxHashSet::default();
10132        let mut selected = FxHashSet::default();
10133        let mut prepared = Vec::new();
10134        let mut pending_diagnostics = Vec::new();
10135        let mut planning_requests = BTreeSet::new();
10136        let mut selected_cells = Vec::new();
10137        let mut workbook_seeded = false;
10138        let mut sheet_scope_seeded = BTreeSet::new();
10139        let mut indexed_query_sheets = FxHashSet::default();
10140        let mut discovery_scratch_reserved = 0u64;
10141        let mut package_encountered = false;
10142        let mut selected_package_sheets = FxHashSet::default();
10143        let mut prepared_packages = Vec::new();
10144        let authoritative_with_ordinary = self.config.formula_plane_mode
10145            == FormulaPlaneMode::AuthoritativeExperimental
10146            && self.staged_formula_index.ordinary_count() != 0;
10147        let has_unknown_package_sheet = self
10148            .staged_formula_index
10149            .package_sheets()
10150            .any(|sheet| self.graph.sheet_id(sheet).is_none());
10151
10152        loop {
10153            if let PrepareScope::Sheets(sheets) = &scope {
10154                for sheet in sheets.clone() {
10155                    if !sheet_scope_seeded.insert(sheet.clone()) {
10156                        continue;
10157                    }
10158                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10159                        package_encountered = true;
10160                        Self::widen_target_preparation(
10161                            options.opaque_policy,
10162                            &mut scope,
10163                            &mut reasons,
10164                            OpaqueReason::UnsupportedSourceSemantics,
10165                        )?;
10166                        continue;
10167                    };
10168                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10169                        self.target_preparation_checkpoint(options.deadline, 1)?;
10170                        regions.push_back(PreparationRegion {
10171                            sheet: sheet.clone(),
10172                            sheet_id,
10173                            start_row: lease.row,
10174                            start_col: lease.col,
10175                            end_row: lease.row,
10176                            end_col: lease.col,
10177                        });
10178                    }
10179                    if self
10180                        .staged_formula_index
10181                        .package_lease_for_sheet(&sheet)
10182                        .is_some()
10183                    {
10184                        regions.push_back(PreparationRegion {
10185                            sheet: sheet.clone(),
10186                            sheet_id,
10187                            start_row: 1,
10188                            start_col: 1,
10189                            end_row: self.workbook_load_limits.max_sheet_rows,
10190                            end_col: self.workbook_load_limits.max_sheet_cols,
10191                        });
10192                    }
10193                }
10194            }
10195            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10196                workbook_seeded = true;
10197                for (sheet, lease) in self.staged_formula_index.all_leases() {
10198                    self.target_preparation_checkpoint(options.deadline, 1)?;
10199                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10200                        package_encountered = true;
10201                        Self::widen_target_preparation(
10202                            options.opaque_policy,
10203                            &mut scope,
10204                            &mut reasons,
10205                            OpaqueReason::UnsupportedSourceSemantics,
10206                        )?;
10207                        continue;
10208                    };
10209                    regions.push_back(PreparationRegion {
10210                        sheet,
10211                        sheet_id,
10212                        start_row: lease.row,
10213                        start_col: lease.col,
10214                        end_row: lease.row,
10215                        end_col: lease.col,
10216                    });
10217                }
10218                let package_sheets = self
10219                    .staged_formula_index
10220                    .package_sheets()
10221                    .map(str::to_string)
10222                    .collect::<Vec<_>>();
10223                for sheet in package_sheets {
10224                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10225                        package_encountered = true;
10226                        Self::widen_target_preparation(
10227                            options.opaque_policy,
10228                            &mut scope,
10229                            &mut reasons,
10230                            OpaqueReason::UnsupportedSourceSemantics,
10231                        )?;
10232                        continue;
10233                    };
10234                    regions.push_back(PreparationRegion {
10235                        sheet,
10236                        sheet_id,
10237                        start_row: 1,
10238                        start_col: 1,
10239                        end_row: self.workbook_load_limits.max_sheet_rows,
10240                        end_col: self.workbook_load_limits.max_sheet_cols,
10241                    });
10242                }
10243            }
10244
10245            let Some(region) = regions.pop_front() else {
10246                if let Some(vertex) = symbol_vertices.pop_front() {
10247                    self.target_preparation_checkpoint(options.deadline, 1)?;
10248                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10249                        continue;
10250                    }
10251                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10252                    if let Some(ast) = self.graph.get_formula(vertex) {
10253                        let snapshot =
10254                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
10255                        if let Some(reason) =
10256                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10257                        {
10258                            return Err(Self::preparation_stale(
10259                                reason,
10260                                "target planning snapshot became stale during discovery",
10261                            ));
10262                        }
10263                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10264                        if let Some(reason) =
10265                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10266                        {
10267                            if reason == OpaqueReason::DynamicReference
10268                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10269                            {
10270                                let sheet = self.graph.get_vertex_sheet_id(vertex);
10271                                let sheet = self.graph.sheet_name(sheet).to_string();
10272                                Self::widen_target_preparation_to_sheet(
10273                                    options.opaque_policy,
10274                                    &mut scope,
10275                                    &mut reasons,
10276                                    reason,
10277                                    &sheet,
10278                                )?;
10279                            } else {
10280                                Self::widen_target_preparation(
10281                                    options.opaque_policy,
10282                                    &mut scope,
10283                                    &mut reasons,
10284                                    reason,
10285                                )?;
10286                            }
10287                        }
10288                    } else if vertex_is_dynamic {
10289                        Self::widen_target_preparation(
10290                            options.opaque_policy,
10291                            &mut scope,
10292                            &mut reasons,
10293                            OpaqueReason::DynamicReference,
10294                        )?;
10295                    }
10296                    if let Some(anchor) = self
10297                        .graph
10298                        .get_cell_ref(vertex)
10299                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
10300                    {
10301                        symbol_vertices.push_back(anchor);
10302                    }
10303                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
10304                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
10305                        regions.push_back(PreparationRegion {
10306                            sheet,
10307                            sheet_id: cell.sheet_id,
10308                            start_row: cell.coord.row() + 1,
10309                            start_col: cell.coord.col() + 1,
10310                            end_row: cell.coord.row() + 1,
10311                            end_col: cell.coord.col() + 1,
10312                        });
10313                    }
10314                    for dependency in self.graph.get_dependencies(vertex) {
10315                        self.target_preparation_checkpoint(options.deadline, 1)?;
10316                        symbol_vertices.push_back(dependency);
10317                    }
10318                    if let Some(range_dependencies) = self
10319                        .graph
10320                        .formula_range_dependencies(vertex)
10321                        .map(<[_]>::to_vec)
10322                    {
10323                        for range in range_dependencies {
10324                            self.target_preparation_checkpoint(options.deadline, 1)?;
10325                            // `Current` is the sheet the formula lives on.
10326                            let context_sheet = self.graph.get_vertex_sheet_id(vertex);
10327                            let Ok(sheet_id) =
10328                                self.resolve_sheet_locator(&range.sheet, context_sheet)
10329                            else {
10330                                if Self::widen_target_preparation(
10331                                    options.opaque_policy,
10332                                    &mut scope,
10333                                    &mut reasons,
10334                                    OpaqueReason::UnresolvedCrossSheetBinding,
10335                                )? {
10336                                    break;
10337                                }
10338                                continue;
10339                            };
10340                            let sheet = self.graph.sheet_name(sheet_id).to_string();
10341                            regions.push_back(PreparationRegion {
10342                                sheet,
10343                                sheet_id,
10344                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10345                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10346                                end_row: range
10347                                    .end_row
10348                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10349                                        bound.index + 1
10350                                    }),
10351                                end_col: range
10352                                    .end_col
10353                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10354                                        bound.index + 1
10355                                    }),
10356                            });
10357                        }
10358                    }
10359                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
10360                        match &name.definition {
10361                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
10362                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
10363                                sheet_id: cell.sheet_id,
10364                                start_row: cell.coord.row() + 1,
10365                                start_col: cell.coord.col() + 1,
10366                                end_row: cell.coord.row() + 1,
10367                                end_col: cell.coord.col() + 1,
10368                            }),
10369                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
10370                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
10371                                sheet_id: range.start.sheet_id,
10372                                start_row: range.start.coord.row() + 1,
10373                                start_col: range.start.coord.col() + 1,
10374                                end_row: range.end.coord.row() + 1,
10375                                end_col: range.end.coord.col() + 1,
10376                            }),
10377                            NamedDefinition::Formula {
10378                                ast,
10379                                dependencies,
10380                                range_deps,
10381                            } => {
10382                                let snapshot =
10383                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
10384                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
10385                                    &snapshot,
10386                                    &assumptions,
10387                                ) {
10388                                    return Err(Self::preparation_stale(
10389                                        reason,
10390                                        "target planning snapshot became stale during discovery",
10391                                    ));
10392                                }
10393                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
10394                                    Self::widen_target_preparation(
10395                                        options.opaque_policy,
10396                                        &mut scope,
10397                                        &mut reasons,
10398                                        reason,
10399                                    )?;
10400                                }
10401                                for dependency in dependencies {
10402                                    self.target_preparation_checkpoint(options.deadline, 1)?;
10403                                    symbol_vertices.push_back(*dependency);
10404                                }
10405                                for range in range_deps {
10406                                    self.target_preparation_checkpoint(options.deadline, 1)?;
10407                                    // `Current` is the sheet this name's formula
10408                                    // was interpreted on, which is the sheet its
10409                                    // vertex is placed on -- the same derivation
10410                                    // the formula-vertex arm above uses. It is
10411                                    // never the workbook default sheet, and an
10412                                    // unresolvable `Name` widens instead of
10413                                    // silently landing on some other sheet.
10414                                    let context_sheet = self.graph.get_vertex_sheet_id(vertex);
10415                                    let Ok(sheet_id) =
10416                                        self.resolve_sheet_locator(&range.sheet, context_sheet)
10417                                    else {
10418                                        if Self::widen_target_preparation(
10419                                            options.opaque_policy,
10420                                            &mut scope,
10421                                            &mut reasons,
10422                                            OpaqueReason::UnresolvedCrossSheetBinding,
10423                                        )? {
10424                                            break;
10425                                        }
10426                                        continue;
10427                                    };
10428                                    regions.push_back(PreparationRegion {
10429                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
10430                                        sheet_id,
10431                                        start_row: range
10432                                            .start_row
10433                                            .map_or(1, |bound| bound.index + 1),
10434                                        start_col: range
10435                                            .start_col
10436                                            .map_or(1, |bound| bound.index + 1),
10437                                        end_row: range.end_row.map_or(
10438                                            self.workbook_load_limits.max_sheet_rows,
10439                                            |bound| bound.index + 1,
10440                                        ),
10441                                        end_col: range.end_col.map_or(
10442                                            self.workbook_load_limits.max_sheet_cols,
10443                                            |bound| bound.index + 1,
10444                                        ),
10445                                    });
10446                                }
10447                            }
10448                            NamedDefinition::Literal(_) => {}
10449                        }
10450                    }
10451                    if let Some(table) = self.graph.table_by_vertex(vertex) {
10452                        regions
10453                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
10454                    }
10455                    continue;
10456                }
10457                break;
10458            };
10459
10460            self.target_preparation_checkpoint(options.deadline, 1)?;
10461            if !visited_regions.insert(region.clone()) {
10462                continue;
10463            }
10464            if let PrepareScope::Sheets(sheets) = &mut scope
10465                && !sheets.iter().any(|sheet| sheet == &region.sheet)
10466            {
10467                sheets.push(region.sheet.clone());
10468                sheets.sort();
10469            }
10470            let package_match = self.staged_formula_index.package_for_region(
10471                &region.sheet,
10472                region.start_row,
10473                region.start_col,
10474                region.end_row,
10475                region.end_col,
10476            );
10477            let package_lease = match package_match {
10478                Some(Ok(lease)) => Some(lease),
10479                Some(Err(()))
10480                    if region.start_row == 1
10481                        && region.start_col == 1
10482                        && region.end_row == self.workbook_load_limits.max_sheet_rows
10483                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
10484                {
10485                    self.staged_formula_index
10486                        .package_lease_for_sheet(&region.sheet)
10487                }
10488                Some(Err(())) => {
10489                    Self::widen_target_preparation(
10490                        options.opaque_policy,
10491                        &mut scope,
10492                        &mut reasons,
10493                        OpaqueReason::DeferredSourcePackage,
10494                    )?;
10495                    None
10496                }
10497                None => None,
10498            };
10499            let compatibility_before_package_replay = package_lease.is_some()
10500                && (authoritative_with_ordinary || has_unknown_package_sheet);
10501            let package_lease = if compatibility_before_package_replay {
10502                package_encountered = true;
10503                Self::widen_target_preparation(
10504                    options.opaque_policy,
10505                    &mut scope,
10506                    &mut reasons,
10507                    OpaqueReason::UnsupportedSourceSemantics,
10508                )?;
10509                None
10510            } else {
10511                package_lease
10512            };
10513            if let Some(package_lease) = package_lease
10514                && selected_package_sheets.insert(region.sheet.clone())
10515            {
10516                self.target_preparation_checkpoint(options.deadline, 1)?;
10517                let mut package = self.prepare_target_source_package(
10518                    &region.sheet,
10519                    package_lease,
10520                    options.deadline,
10521                )?;
10522                for placement in &package.placements {
10523                    self.target_preparation_checkpoint(options.deadline, 1)?;
10524                    for dependency in &placement.fragment_dependency_proof().1.dependencies {
10525                        self.target_preparation_checkpoint(options.deadline, 1)?;
10526                        let (rows, cols) = dependency.read_region.axis_ranges();
10527                        let (start_row, end_row) = rows.query_bounds();
10528                        let (start_col, end_col) = cols.query_bounds();
10529                        let dependency_sheet = dependency.read_region.sheet_id();
10530                        regions.push_back(PreparationRegion {
10531                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10532                            sheet_id: dependency_sheet,
10533                            start_row: start_row.saturating_add(1),
10534                            start_col: start_col.saturating_add(1),
10535                            end_row: end_row
10536                                .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
10537                                .saturating_add(1),
10538                            end_col: end_col
10539                                .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
10540                                .saturating_add(1),
10541                        });
10542                    }
10543                }
10544
10545                let mut final_fallback = BTreeMap::new();
10546                for record in package.fallback_records() {
10547                    final_fallback.insert((record.row, record.col), record.clone());
10548                }
10549                let batch = self
10550                    .formula_batch_from_exact_replay(&region.sheet, final_fallback.into_values())?;
10551                for record in batch.formulas {
10552                    self.target_preparation_checkpoint(options.deadline, 1)?;
10553                    let ast = self
10554                        .graph
10555                        .data_store()
10556                        .retrieve_ast(record.ast_id, self.graph.sheet_reg())
10557                        .ok_or_else(|| {
10558                            ExcelError::new(ExcelErrorKind::Value)
10559                                .with_message("target fallback AST is unavailable")
10560                        })?;
10561                    let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10562                    if let Some(reason) =
10563                        Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10564                    {
10565                        return Err(Self::preparation_stale(
10566                            reason,
10567                            "target fallback planning snapshot became stale during discovery",
10568                        ));
10569                    }
10570                    let proven_sheet_local_dynamic =
10571                        Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10572                    if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10573                        if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
10574                            Self::widen_target_preparation_to_sheet(
10575                                options.opaque_policy,
10576                                &mut scope,
10577                                &mut reasons,
10578                                reason,
10579                                &region.sheet,
10580                            )?;
10581                        } else {
10582                            Self::widen_target_preparation(
10583                                options.opaque_policy,
10584                                &mut scope,
10585                                &mut reasons,
10586                                reason,
10587                            )?;
10588                        }
10589                    }
10590                    let placement = CellRef::new(
10591                        package.sheet_id,
10592                        Coord::from_excel(record.row, record.col, true, true),
10593                    );
10594                    let ingested = self
10595                        .graph
10596                        .ingest_pipeline(&snapshot)
10597                        .enable_function_semantics()
10598                        .ingest_formula(
10599                            FormulaAstInput::RawArena(record.ast_id),
10600                            placement,
10601                            record.formula_text,
10602                        )?;
10603                    if ingested.dep_plan.dynamic {
10604                        if proven_sheet_local_dynamic {
10605                            Self::widen_target_preparation_to_sheet(
10606                                options.opaque_policy,
10607                                &mut scope,
10608                                &mut reasons,
10609                                OpaqueReason::DynamicReference,
10610                                &region.sheet,
10611                            )?;
10612                        } else {
10613                            Self::widen_target_preparation(
10614                                options.opaque_policy,
10615                                &mut scope,
10616                                &mut reasons,
10617                                OpaqueReason::DynamicReference,
10618                            )?;
10619                        }
10620                    }
10621                    for dep in &ingested.dep_plan.direct_cell_deps {
10622                        self.target_preparation_checkpoint(options.deadline, 1)?;
10623                        regions.push_back(PreparationRegion {
10624                            sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10625                            sheet_id: dep.sheet_id,
10626                            start_row: dep.coord.row().saturating_add(1),
10627                            start_col: dep.coord.col().saturating_add(1),
10628                            end_row: dep.coord.row().saturating_add(1),
10629                            end_col: dep.coord.col().saturating_add(1),
10630                        });
10631                    }
10632                    for range in &ingested.dep_plan.range_deps {
10633                        self.target_preparation_checkpoint(options.deadline, 1)?;
10634                        // `Current` is the sheet the staged package's formula
10635                        // lives on.
10636                        let Ok(dependency_sheet) =
10637                            self.resolve_sheet_locator(&range.sheet, package.sheet_id)
10638                        else {
10639                            Self::widen_target_preparation(
10640                                options.opaque_policy,
10641                                &mut scope,
10642                                &mut reasons,
10643                                OpaqueReason::UnresolvedCrossSheetBinding,
10644                            )?;
10645                            continue;
10646                        };
10647                        regions.push_back(PreparationRegion {
10648                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10649                            sheet_id: dependency_sheet,
10650                            start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10651                            start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10652                            end_row: range
10653                                .end_row
10654                                .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10655                                    bound.index + 1
10656                                }),
10657                            end_col: range
10658                                .end_col
10659                                .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10660                                    bound.index + 1
10661                                }),
10662                        });
10663                    }
10664                    for name in ingested
10665                        .dep_plan
10666                        .resolved_named_refs
10667                        .iter()
10668                        .chain(&ingested.dep_plan.named_refs)
10669                    {
10670                        self.target_preparation_checkpoint(options.deadline, 1)?;
10671                        if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
10672                            symbol_vertices.push_back(entry.vertex);
10673                        } else if self.graph.resolve_source_scalar_entry(name).is_none()
10674                            && self.graph.resolve_source_table_entry(name).is_none()
10675                        {
10676                            Self::widen_target_preparation(
10677                                options.opaque_policy,
10678                                &mut scope,
10679                                &mut reasons,
10680                                OpaqueReason::UnresolvedName,
10681                            )?;
10682                        }
10683                    }
10684                    for table in &ingested.dep_plan.table_refs {
10685                        self.target_preparation_checkpoint(options.deadline, 1)?;
10686                        if let Some(entry) = self.graph.resolve_table_entry(table) {
10687                            symbol_vertices.push_back(entry.vertex);
10688                        } else if self.graph.resolve_source_table_entry(table).is_none() {
10689                            Self::widen_target_preparation(
10690                                options.opaque_policy,
10691                                &mut scope,
10692                                &mut reasons,
10693                                OpaqueReason::UnresolvedTable,
10694                            )?;
10695                        }
10696                    }
10697                    package.legacy.push((
10698                        record.row,
10699                        record.col,
10700                        ingested.ast_id,
10701                        ingested.dep_plan,
10702                    ));
10703                }
10704                prepared_packages.push(package);
10705            }
10706            let leases = self.staged_formula_index.leases_in_region(
10707                &region.sheet,
10708                region.start_row,
10709                region.start_col,
10710                region.end_row,
10711                region.end_col,
10712            );
10713            for lease in leases {
10714                self.target_preparation_checkpoint(options.deadline, 1)?;
10715                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
10716                    ExcelError::new(ExcelErrorKind::Ref)
10717                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
10718                })?;
10719                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
10720                if !selected.insert(key) {
10721                    continue;
10722                }
10723                let text = self
10724                    .staged_formulas
10725                    .get(&region.sheet)
10726                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
10727                    .ok_or_else(|| {
10728                        ExcelError::new(ExcelErrorKind::Value)
10729                            .with_message("staged formula index is stale")
10730                    })?
10731                    .to_string();
10732                let formula = if text.starts_with('=') {
10733                    text.clone()
10734                } else {
10735                    format!("={text}")
10736                };
10737                self.target_preparation_checkpoint(options.deadline, 1)?;
10738                let ast = match formualizer_parse::parser::parse(&formula) {
10739                    Ok(ast) => ast,
10740                    Err(error) => {
10741                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
10742                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
10743                                format!(
10744                                    "Formula parse error at {}!{}{}: {error}",
10745                                    region.sheet,
10746                                    col_letters_from_1based(lease.col)
10747                                        .unwrap_or_else(|_| "?".to_string()),
10748                                    lease.row
10749                                ),
10750                            ));
10751                        }
10752                        pending_diagnostics.push(FormulaParseDiagnostic {
10753                            sheet: region.sheet.clone(),
10754                            row: lease.row,
10755                            col: lease.col,
10756                            formula: formula.clone(),
10757                            message: error.to_string(),
10758                            policy: self.config.formula_parse_policy,
10759                        });
10760                        match self.config.formula_parse_policy {
10761                            FormulaParsePolicy::KeepCachedValue => {
10762                                selected_cells.push(
10763                                    formualizer_common::RangeAddress::new(
10764                                        region.sheet.clone(),
10765                                        lease.row,
10766                                        lease.col,
10767                                        lease.row,
10768                                        lease.col,
10769                                    )
10770                                    .expect("selected staged coordinates are valid"),
10771                                );
10772                                prepared.push(PreparedOrdinaryStagedFormula {
10773                                    sheet: region.sheet.clone(),
10774                                    sheet_id,
10775                                    lease,
10776                                    ast_id: None,
10777                                    plan: None,
10778                                });
10779                                continue;
10780                            }
10781                            FormulaParsePolicy::AsText => ASTNode::new(
10782                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
10783                                None,
10784                            ),
10785                            FormulaParsePolicy::CoerceToError => ASTNode::new(
10786                                ASTNodeType::Literal(LiteralValue::Error(
10787                                    ExcelError::new(ExcelErrorKind::Error)
10788                                        .with_message(format!("Malformed formula: {error}")),
10789                                )),
10790                                None,
10791                            ),
10792                            FormulaParsePolicy::Strict => unreachable!(),
10793                        }
10794                    }
10795                };
10796                self.target_preparation_checkpoint(options.deadline, 1)?;
10797                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
10798                self.target_preparation_checkpoint(options.deadline, 1)?;
10799                if let Some(reason) =
10800                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10801                {
10802                    return Err(Self::preparation_stale(
10803                        reason,
10804                        "target planning snapshot became stale during discovery",
10805                    ));
10806                }
10807                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
10808                    if reason == OpaqueReason::DynamicReference
10809                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10810                    {
10811                        Self::widen_target_preparation_to_sheet(
10812                            options.opaque_policy,
10813                            &mut scope,
10814                            &mut reasons,
10815                            reason,
10816                            &region.sheet,
10817                        )?;
10818                    } else {
10819                        Self::widen_target_preparation(
10820                            options.opaque_policy,
10821                            &mut scope,
10822                            &mut reasons,
10823                            reason,
10824                        )?;
10825                    }
10826                }
10827                let proven_sheet_local_dynamic =
10828                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
10829                let placement = CellRef::new(
10830                    sheet_id,
10831                    Coord::from_excel(lease.row, lease.col, true, true),
10832                );
10833                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
10834                    FormulaAstInput::Tree(ast),
10835                    placement,
10836                    Some(Arc::from(formula)),
10837                )?;
10838                self.target_preparation_checkpoint(options.deadline, 1)?;
10839                if ingested.dep_plan.dynamic {
10840                    if proven_sheet_local_dynamic {
10841                        Self::widen_target_preparation_to_sheet(
10842                            options.opaque_policy,
10843                            &mut scope,
10844                            &mut reasons,
10845                            OpaqueReason::DynamicReference,
10846                            &region.sheet,
10847                        )?;
10848                    } else {
10849                        Self::widen_target_preparation(
10850                            options.opaque_policy,
10851                            &mut scope,
10852                            &mut reasons,
10853                            OpaqueReason::DynamicReference,
10854                        )?;
10855                    }
10856                }
10857                for dep in &ingested.dep_plan.direct_cell_deps {
10858                    self.target_preparation_checkpoint(options.deadline, 1)?;
10859                    regions.push_back(PreparationRegion {
10860                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
10861                        sheet_id: dep.sheet_id,
10862                        start_row: dep.coord.row() + 1,
10863                        start_col: dep.coord.col() + 1,
10864                        end_row: dep.coord.row() + 1,
10865                        end_col: dep.coord.col() + 1,
10866                    });
10867                }
10868                for range in &ingested.dep_plan.range_deps {
10869                    self.target_preparation_checkpoint(options.deadline, 1)?;
10870                    // `Current` is the sheet the staged formula lives on.
10871                    let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
10872                    else {
10873                        Self::widen_target_preparation(
10874                            options.opaque_policy,
10875                            &mut scope,
10876                            &mut reasons,
10877                            OpaqueReason::UnresolvedCrossSheetBinding,
10878                        )?;
10879                        continue;
10880                    };
10881                    regions.push_back(PreparationRegion {
10882                        sheet: self.graph.sheet_name(dependency_sheet).to_string(),
10883                        sheet_id: dependency_sheet,
10884                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
10885                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
10886                        end_row: range
10887                            .end_row
10888                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
10889                                bound.index + 1
10890                            }),
10891                        end_col: range
10892                            .end_col
10893                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
10894                                bound.index + 1
10895                            }),
10896                    });
10897                }
10898                for name in ingested
10899                    .dep_plan
10900                    .resolved_named_refs
10901                    .iter()
10902                    .chain(&ingested.dep_plan.named_refs)
10903                {
10904                    self.target_preparation_checkpoint(options.deadline, 1)?;
10905                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
10906                        symbol_vertices.push_back(entry.vertex);
10907                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
10908                        && self.graph.resolve_source_table_entry(name).is_none()
10909                    {
10910                        Self::widen_target_preparation(
10911                            options.opaque_policy,
10912                            &mut scope,
10913                            &mut reasons,
10914                            OpaqueReason::UnresolvedName,
10915                        )?;
10916                    }
10917                }
10918                for table in &ingested.dep_plan.table_refs {
10919                    self.target_preparation_checkpoint(options.deadline, 1)?;
10920                    if let Some(entry) = self.graph.resolve_table_entry(table) {
10921                        symbol_vertices.push_back(entry.vertex);
10922                    } else if self.graph.resolve_source_table_entry(table).is_none() {
10923                        Self::widen_target_preparation(
10924                            options.opaque_policy,
10925                            &mut scope,
10926                            &mut reasons,
10927                            OpaqueReason::UnresolvedTable,
10928                        )?;
10929                    }
10930                }
10931                selected_cells.push(
10932                    formualizer_common::RangeAddress::new(
10933                        region.sheet.clone(),
10934                        lease.row,
10935                        lease.col,
10936                        lease.row,
10937                        lease.col,
10938                    )
10939                    .expect("selected staged coordinates are valid"),
10940                );
10941                prepared.push(PreparedOrdinaryStagedFormula {
10942                    sheet: region.sheet.clone(),
10943                    sheet_id,
10944                    lease,
10945                    ast_id: Some(ingested.ast_id),
10946                    plan: Some(ingested.dep_plan),
10947                });
10948            }
10949
10950            if indexed_query_sheets.insert(region.sheet_id) {
10951                self.graph.prepare_sheet_index_for_query(region.sheet_id);
10952                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
10953                    .saturating_mul(32);
10954                self.reserve_graph_source_scratch(bytes)?;
10955                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
10956            }
10957            let spill_anchors = self.graph.spill_anchors_in_region(
10958                region.sheet_id,
10959                region.start_row - 1,
10960                region.start_col - 1,
10961                region.end_row - 1,
10962                region.end_col - 1,
10963            );
10964            for anchor in spill_anchors {
10965                self.target_preparation_checkpoint(options.deadline, 1)?;
10966                symbol_vertices.push_back(anchor);
10967            }
10968            let vertices = self.graph.vertices_in_region(
10969                region.sheet_id,
10970                region.start_row - 1,
10971                region.end_row - 1,
10972                region.start_col - 1,
10973                region.end_col - 1,
10974            );
10975            for vertex in vertices {
10976                self.target_preparation_checkpoint(options.deadline, 1)?;
10977                symbol_vertices.push_back(vertex);
10978            }
10979        }
10980
10981        #[cfg(test)]
10982        self.target_preparation_fault(
10983            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
10984        )?;
10985
10986        if package_encountered
10987            || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
10988                && !prepared.is_empty())
10989        {
10990            Self::widen_target_preparation(
10991                options.opaque_policy,
10992                &mut scope,
10993                &mut reasons,
10994                OpaqueReason::UnsupportedSourceSemantics,
10995            )?;
10996            self.target_preparation_checkpoint(options.deadline, 0)?;
10997            let selected_count = self.staged_formula_count();
10998            let selected_packages = self
10999                .staged_formulas
11000                .values()
11001                .filter_map(|staged| staged.deferred_package.as_ref())
11002                .map(|package| package.families.len() + package.partitioned_families.len())
11003                .sum();
11004            self.formula_parse_diagnostics.truncate(diagnostics_len);
11005            self.last_formula_ingest_report = report_len;
11006            #[cfg(test)]
11007            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11008                hook();
11009            }
11010            self.target_preparation_checkpoint(options.deadline, 0)?;
11011            #[cfg(test)]
11012            self.target_preparation_fault(
11013                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11014            )?;
11015            let current_revisions = self.preparation_revisions();
11016            if let Some(reason) = Self::preparation_revision_stale_reason(
11017                &assumptions,
11018                &current_revisions,
11019                &planning_requests,
11020                true,
11021            ) {
11022                return Err(Self::preparation_stale(
11023                    reason,
11024                    "target compatibility preparation plan is stale",
11025                ));
11026            }
11027            #[cfg(test)]
11028            self.target_preparation_fault(
11029                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11030            )?;
11031            let commit_work_before = self
11032                .active_resource_ledger
11033                .as_ref()
11034                .map_or(0, |ledger| ledger.snapshot().work_charged);
11035            let commit_started = crate::instant::FzInstant::now();
11036            self.build_graph_all_unobserved()?;
11037            let commit_window = commit_started.elapsed();
11038            let ledger_after = self
11039                .active_resource_ledger
11040                .as_ref()
11041                .map(|ledger| ledger.snapshot());
11042            let actual_commit_work = ledger_after
11043                .map_or(0, |snapshot| snapshot.work_charged)
11044                .saturating_sub(commit_work_before);
11045            let observed_scratch_bytes = ledger_after
11046                .map_or(0, |snapshot| snapshot.scratch_peak)
11047                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11048            let revisions = assumptions.clone();
11049            let report = PreparedTargetGraphReport {
11050                request_id: request_id.unwrap_or_default(),
11051                requested_targets: targets.len(),
11052                normalized_regions: visited_regions.len(),
11053                normalized_target_list: normalized,
11054                selected_staged_cells: selected_count,
11055                selected_source_families: selected_packages,
11056                retained_staged_cells: self.staged_formula_count(),
11057                selected_cells,
11058                retained_cells: Vec::new(),
11059                widened_scope: PrepareScope::Workbook,
11060                widening_reasons: reasons,
11061                revisions,
11062                commit_window,
11063                estimated_scratch_bytes: discovery_scratch_reserved
11064                    .saturating_add((selected_count as u64).saturating_mul(256)),
11065                observed_scratch_bytes,
11066                estimated_commit_work: selected_count as u64,
11067                actual_commit_work,
11068                outcome: PreparationOutcome::CompatibilityPrepared,
11069            };
11070            self.observe_target_preparation_report(&report);
11071            return Ok(report);
11072        }
11073
11074        prepared.sort_by_key(|formula| formula.lease.insertion_order);
11075        for package in &prepared_packages {
11076            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11077                formualizer_common::RangeAddress::new(
11078                    package.sheet.clone(),
11079                    record.row,
11080                    record.col,
11081                    record.row,
11082                    record.col,
11083                )
11084                .ok()
11085            }));
11086        }
11087        let (mut legacy_graph, mut planned_formula_count) =
11088            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11089        let placements = prepared_packages
11090            .iter_mut()
11091            .flat_map(|package| std::mem::take(&mut package.placements))
11092            .collect::<Vec<_>>();
11093        let formula_plane = if placements.is_empty() {
11094            None
11095        } else {
11096            match self.graph.formula_authority().prepare_formula_plane_append(
11097                placements,
11098                self.graph.data_store(),
11099                self.graph.sheet_reg(),
11100            ) {
11101                Ok(append) => Some(append),
11102                Err(error) => {
11103                    let reason = format!("TargetFormulaPlaneAppend:{error}");
11104                    for package in &mut prepared_packages {
11105                        *package
11106                            .source_report
11107                            .fallback_reasons
11108                            .entry(reason.clone())
11109                            .or_default() += package.direct_families as u64;
11110                        if self.config.formula_plane_mode
11111                            == FormulaPlaneMode::AuthoritativeExperimental
11112                        {
11113                            self.materialize_target_package_direct_records(
11114                                package,
11115                                &assumptions,
11116                                &mut planning_requests,
11117                                options.deadline,
11118                            )?;
11119                        } else {
11120                            package.direct_families = 0;
11121                            package.direct_cells = 0;
11122                            package.direct_fragments = 0;
11123                            package.direct_complete_families = 0;
11124                            package.direct_complete_cells = 0;
11125                            package.direct_partition_families = 0;
11126                            package.direct_partition_cells = 0;
11127                        }
11128                    }
11129                    (legacy_graph, planned_formula_count) =
11130                        self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11131                    None
11132                }
11133            }
11134        };
11135        let new_vertices = legacy_graph.new_vertex_count();
11136        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11137            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11138        })?;
11139        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11140            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11141        })?;
11142        let current = self.graph.baseline_stats();
11143        let final_vertices = current
11144            .graph_vertex_count
11145            .checked_add(new_vertices)
11146            .ok_or_else(|| {
11147                crate::engine::ResourceLedgerError::Exhausted(
11148                    formualizer_common::ResourceExhaustionDetail {
11149                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11150                        limit: u64::MAX,
11151                        observed: u64::MAX,
11152                        request_id,
11153                    },
11154                )
11155                .into_excel_error()
11156            })?;
11157        let final_edges = current
11158            .graph_edge_count
11159            .checked_sub(removed_edges)
11160            .and_then(|count| count.checked_add(new_edges))
11161            .ok_or_else(|| {
11162                crate::engine::ResourceLedgerError::Exhausted(
11163                    formualizer_common::ResourceExhaustionDetail {
11164                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11165                        limit: u64::MAX,
11166                        observed: u64::MAX,
11167                        request_id,
11168                    },
11169                )
11170                .into_excel_error()
11171            })?;
11172        #[cfg(test)]
11173        self.target_preparation_fault(
11174            crate::engine::target_preparation::TargetPreparationFault::Admission,
11175        )?;
11176        let resource = |reason, limit: u64, observed: u64| {
11177            crate::engine::ResourceLedgerError::Exhausted(
11178                formualizer_common::ResourceExhaustionDetail {
11179                    reason,
11180                    limit,
11181                    observed,
11182                    request_id,
11183                },
11184            )
11185            .into_excel_error()
11186        };
11187        let admission = crate::engine::resource_ledger::GraphAdmission {
11188            final_vertices,
11189            final_edges,
11190            materialization_cells: planned_formula_count as u64,
11191            added_vertices: new_vertices,
11192            added_edges: new_edges,
11193        };
11194        let materialized_bytes = admission
11195            .materialized_graph_bytes()
11196            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11197        if let Err(error) = self.preflight_graph_admission(admission) {
11198            if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11199                self.observe_target_admission_failure(detail.reason);
11200            }
11201            return Err(error);
11202        }
11203        let selected_package_records = prepared_packages
11204            .iter()
11205            .map(|package| package.replay_records.len() as u64)
11206            .sum::<u64>();
11207        let planned_working_bytes = (prepared.len() as u64)
11208            .saturating_add(selected_package_records)
11209            .saturating_mul(256)
11210            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11211            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11212            .saturating_add((selected.len() as u64).saturating_mul(96))
11213            .saturating_add(materialized_bytes);
11214        let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11215        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11216        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11217            self.observe_target_admission_failure(
11218                formualizer_common::ResourceExhaustionReason::ScratchMemory,
11219            );
11220            return Err(error);
11221        }
11222
11223        let estimated_commit_duration = std::time::Duration::from_nanos(
11224            (new_vertices as u64)
11225                .saturating_add(new_edges as u64)
11226                .saturating_add(prepared.len() as u64)
11227                .saturating_add(selected_package_records)
11228                .max(1)
11229                .saturating_mul(100),
11230        );
11231        if options.deadline.is_some_and(|deadline| {
11232            std::time::Instant::now()
11233                .checked_add(estimated_commit_duration)
11234                .is_none_or(|finish| finish > deadline)
11235        }) {
11236            self.observe_target_admission_failure(
11237                formualizer_common::ResourceExhaustionReason::Deadline,
11238            );
11239            return Err(resource(
11240                formualizer_common::ResourceExhaustionReason::Deadline,
11241                0,
11242                1,
11243            ));
11244        }
11245        #[cfg(test)]
11246        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11247            hook();
11248        }
11249        self.target_preparation_checkpoint(options.deadline, 0)?;
11250        #[cfg(test)]
11251        self.target_preparation_fault(
11252            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11253        )?;
11254        let current_revisions = self.preparation_revisions();
11255        let staged_leases_match = prepared.iter().all(|formula| {
11256            self.staged_formula_index
11257                .lease_matches(&formula.sheet, formula.lease)
11258        }) && prepared_packages.iter().all(|package| {
11259            self.staged_formula_index
11260                .package_lease_matches(&package.sheet, package.lease)
11261        });
11262        let stale_reason = Self::preparation_revision_stale_reason(
11263            &assumptions,
11264            &current_revisions,
11265            &planning_requests,
11266            staged_leases_match,
11267        );
11268        if let Some(reason) = stale_reason {
11269            return Err(Self::preparation_stale(
11270                reason,
11271                "target graph preparation plan is stale",
11272            ));
11273        }
11274        #[cfg(test)]
11275        self.target_preparation_fault(
11276            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11277        )?;
11278        self.graph
11279            .validate_prepared_legacy_graph_plan(&legacy_graph)
11280            .map_err(|error| {
11281                Self::preparation_stale(
11282                    formualizer_common::PreparationStaleReason::Graph,
11283                    format!("target graph preparation plan is stale: {error}"),
11284                )
11285            })?;
11286        if let Some(append) = formula_plane.as_ref() {
11287            self.graph
11288                .formula_authority()
11289                .validate_prepared_formula_plane_append(
11290                    append,
11291                    self.graph.data_store(),
11292                    self.graph.sheet_reg(),
11293                )
11294                .map_err(|error| {
11295                    Self::preparation_stale(
11296                        formualizer_common::PreparationStaleReason::Authority,
11297                        format!("target FormulaPlane preparation is stale: {error}"),
11298                    )
11299                })?;
11300        }
11301        #[cfg(test)]
11302        self.target_preparation_fault(
11303            crate::engine::target_preparation::TargetPreparationFault::Reservation,
11304        )?;
11305        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
11306        self.formula_parse_diagnostics
11307            .try_reserve(pending_diagnostics.len())
11308            .map_err(|_| {
11309                resource(
11310                    formualizer_common::ResourceExhaustionReason::Admission,
11311                    pending_diagnostics.len() as u64,
11312                    pending_diagnostics.len() as u64,
11313                )
11314            })?;
11315        self.target_preparation_checkpoint(options.deadline, 0)?;
11316        #[cfg(test)]
11317        self.target_preparation_fault(
11318            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
11319        )?;
11320
11321        let commit_started = crate::instant::FzInstant::now();
11322        let committed = self
11323            .graph
11324            .apply_prevalidated_legacy_graph_plan(legacy_graph);
11325        let plane_report =
11326            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11327                formula_plane.map(|append| {
11328                    self.graph
11329                        .formula_authority_mut()
11330                        .apply_prevalidated_formula_plane_append(append)
11331                })
11332            } else {
11333                None
11334            };
11335        if let Some(report) = plane_report.as_ref() {
11336            self.graph.mark_formula_spans_dirty(
11337                report.spans.iter().copied(),
11338                WholeSpanDirtyReason::NewSpan,
11339            );
11340        }
11341        for formula in &prepared {
11342            let removed = self
11343                .staged_formulas
11344                .get_mut(&formula.sheet)
11345                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
11346            debug_assert!(removed.is_some());
11347            let index_removed = self.staged_formula_index.remove(
11348                &formula.sheet,
11349                formula.lease.row,
11350                formula.lease.col,
11351            );
11352            debug_assert!(index_removed);
11353        }
11354        for package in &prepared_packages {
11355            let removed = self
11356                .staged_formulas
11357                .get_mut(&package.sheet)
11358                .and_then(|staged| staged.deferred_package.take());
11359            debug_assert!(removed.is_some());
11360            self.staged_formula_index.set_package(&package.sheet, None);
11361        }
11362        let empty_sheets = self
11363            .staged_formulas
11364            .iter()
11365            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
11366            .collect::<Vec<_>>();
11367        for sheet in empty_sheets {
11368            self.staged_formulas.remove(&sheet);
11369        }
11370        if committed > 0 || plane_report.is_some() {
11371            self.mark_topology_edited();
11372        }
11373        self.formula_parse_diagnostics.extend(pending_diagnostics);
11374        if !prepared.is_empty() || !prepared_packages.is_empty() {
11375            let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
11376            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
11377                prepared_packages
11378                    .iter()
11379                    .map(|package| package.replay_records.len() as u64)
11380                    .sum::<u64>(),
11381            );
11382            ingest_delta.graph_formula_cells_materialized = committed as u64;
11383            ingest_delta.graph_vertices_created = new_vertices as u64;
11384            ingest_delta.graph_edges_created = new_edges as u64;
11385            for package in &prepared_packages {
11386                let source = &package.source_report;
11387                ingest_delta.source_formula_events = ingest_delta
11388                    .source_formula_events
11389                    .saturating_add(source.source_formula_events);
11390                ingest_delta.source_formula_records_spooled = ingest_delta
11391                    .source_formula_records_spooled
11392                    .saturating_add(source.source_formula_records_spooled);
11393                ingest_delta.source_spool_encoded_bytes = ingest_delta
11394                    .source_spool_encoded_bytes
11395                    .saturating_add(source.source_spool_encoded_bytes);
11396                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
11397                    .source_spool_peak_memory_bytes
11398                    .max(source.source_spool_peak_memory_bytes);
11399                ingest_delta.source_spool_spilled_bytes = ingest_delta
11400                    .source_spool_spilled_bytes
11401                    .saturating_add(source.source_spool_spilled_bytes);
11402                ingest_delta.source_spool_spill_files = ingest_delta
11403                    .source_spool_spill_files
11404                    .saturating_add(source.source_spool_spill_files);
11405                ingest_delta.source_spool_replays = ingest_delta
11406                    .source_spool_replays
11407                    .saturating_add(source.source_spool_replays)
11408                    .saturating_add(1);
11409                ingest_delta.source_families_seen = ingest_delta
11410                    .source_families_seen
11411                    .saturating_add(source.families_seen);
11412                ingest_delta.source_family_cells_seen = ingest_delta
11413                    .source_family_cells_seen
11414                    .saturating_add(source.family_cells_seen);
11415                ingest_delta.source_family_shadow_eligible = ingest_delta
11416                    .source_family_shadow_eligible
11417                    .saturating_add(source.source_clean_families);
11418                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
11419                    .source_family_shadow_eligible_cells
11420                    .saturating_add(source.source_clean_cells);
11421                ingest_delta.source_partitioned_families_seen = ingest_delta
11422                    .source_partitioned_families_seen
11423                    .saturating_add(source.source_fragmentable_families);
11424                ingest_delta.source_partition_holes = ingest_delta
11425                    .source_partition_holes
11426                    .saturating_add(source.source_hole_exclusions);
11427                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
11428                    .source_partition_ordinary_exceptions
11429                    .saturating_add(source.source_ordinary_exclusions);
11430                ingest_delta.source_partition_surviving_cells = ingest_delta
11431                    .source_partition_surviving_cells
11432                    .saturating_add(source.source_fragmentable_cells);
11433                for (reason, count) in &source.fallback_reasons {
11434                    let total = ingest_delta
11435                        .fallback_reasons
11436                        .entry(reason.clone())
11437                        .or_default();
11438                    *total = total.saturating_add(*count);
11439                }
11440                if self.config.formula_plane_mode == FormulaPlaneMode::Off {
11441                    ingest_delta.source_family_fallback = ingest_delta
11442                        .source_family_fallback
11443                        .saturating_add(source.families_seen);
11444                    ingest_delta.source_family_fallback_cells = ingest_delta
11445                        .source_family_fallback_cells
11446                        .saturating_add(source.family_cells_seen);
11447                } else {
11448                    ingest_delta.shadow_candidate_cells = ingest_delta
11449                        .shadow_candidate_cells
11450                        .saturating_add(source.family_cells_seen);
11451                    ingest_delta.shadow_accepted_span_cells = ingest_delta
11452                        .shadow_accepted_span_cells
11453                        .saturating_add(package.direct_cells);
11454                    ingest_delta.shadow_fallback_cells =
11455                        ingest_delta.shadow_fallback_cells.saturating_add(
11456                            source
11457                                .family_cells_seen
11458                                .saturating_sub(package.direct_cells),
11459                        );
11460                    ingest_delta.source_anchor_parses = ingest_delta
11461                        .source_anchor_parses
11462                        .saturating_add(package.anchor_parses);
11463                    ingest_delta.source_anchor_asts = ingest_delta
11464                        .source_anchor_asts
11465                        .saturating_add(package.anchor_asts);
11466                    ingest_delta.source_anchor_analyses = ingest_delta
11467                        .source_anchor_analyses
11468                        .saturating_add(package.anchor_analyses);
11469                    ingest_delta.source_compressed_families_prepared = ingest_delta
11470                        .source_compressed_families_prepared
11471                        .saturating_add(package.direct_complete_families);
11472                    ingest_delta.source_compressed_cells_prepared = ingest_delta
11473                        .source_compressed_cells_prepared
11474                        .saturating_add(package.direct_complete_cells);
11475                    ingest_delta.source_partitioned_families_prepared = ingest_delta
11476                        .source_partitioned_families_prepared
11477                        .saturating_add(package.direct_partition_families);
11478                    ingest_delta.source_partition_fragments_prepared = ingest_delta
11479                        .source_partition_fragments_prepared
11480                        .saturating_add(package.direct_fragments);
11481                    ingest_delta.source_partition_span_cells_prepared = ingest_delta
11482                        .source_partition_span_cells_prepared
11483                        .saturating_add(package.direct_partition_cells);
11484                    ingest_delta.source_partition_analyses_reused = ingest_delta
11485                        .source_partition_analyses_reused
11486                        .saturating_add(
11487                            package
11488                                .direct_fragments
11489                                .saturating_sub(package.direct_partition_families),
11490                        );
11491                    ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
11492                        .graph_formula_vertices_avoided_shadow
11493                        .saturating_add(package.direct_cells);
11494                    ingest_delta.ast_roots_avoided_shadow =
11495                        ingest_delta.ast_roots_avoided_shadow.saturating_add(
11496                            package
11497                                .direct_complete_cells
11498                                .saturating_sub(package.direct_complete_families)
11499                                .saturating_add(
11500                                    package
11501                                        .direct_partition_cells
11502                                        .saturating_sub(package.direct_fragments),
11503                                ),
11504                        );
11505                    ingest_delta.edge_rows_avoided_shadow = ingest_delta
11506                        .edge_rows_avoided_shadow
11507                        .saturating_add(package.direct_cells);
11508                    if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11509                    {
11510                        ingest_delta.source_family_promoted = ingest_delta
11511                            .source_family_promoted
11512                            .saturating_add(package.direct_families as u64);
11513                        ingest_delta.source_family_promoted_cells = ingest_delta
11514                            .source_family_promoted_cells
11515                            .saturating_add(package.direct_cells);
11516                        let descendants = package
11517                            .direct_complete_cells
11518                            .saturating_sub(package.direct_complete_families)
11519                            .saturating_add(
11520                                package
11521                                    .direct_partition_cells
11522                                    .saturating_sub(package.direct_partition_families),
11523                            );
11524                        ingest_delta.source_descendant_strings_avoided = ingest_delta
11525                            .source_descendant_strings_avoided
11526                            .saturating_add(descendants);
11527                        ingest_delta.source_descendant_events_avoided = ingest_delta
11528                            .source_descendant_events_avoided
11529                            .saturating_add(descendants);
11530                        ingest_delta.source_descendant_analyses_avoided = ingest_delta
11531                            .source_descendant_analyses_avoided
11532                            .saturating_add(descendants);
11533                    }
11534                    ingest_delta.source_family_fallback =
11535                        ingest_delta.source_family_fallback.saturating_add(
11536                            source
11537                                .families_seen
11538                                .saturating_sub(package.direct_families as u64),
11539                        );
11540                    ingest_delta.source_family_fallback_cells =
11541                        ingest_delta.source_family_fallback_cells.saturating_add(
11542                            source
11543                                .family_cells_seen
11544                                .saturating_sub(package.direct_cells),
11545                        );
11546                }
11547            }
11548            if let Some(report) = plane_report.as_ref() {
11549                ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
11550                ingest_delta.shadow_spans_created = report.spans.len() as u64;
11551            }
11552            self.record_formula_ingest_report(ingest_delta);
11553        }
11554        let commit_window = commit_started.elapsed();
11555        let retained_cells = self
11556            .staged_formula_index
11557            .all_leases()
11558            .into_iter()
11559            .filter_map(|(sheet, lease)| {
11560                formualizer_common::RangeAddress::new(
11561                    sheet, lease.row, lease.col, lease.row, lease.col,
11562                )
11563                .ok()
11564            })
11565            .collect::<Vec<_>>();
11566        let observed_scratch_bytes = self
11567            .active_resource_ledger
11568            .as_ref()
11569            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
11570            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11571        let committed_spans = plane_report
11572            .as_ref()
11573            .map_or(0, |report| report.spans.len() as u64);
11574        let selected_source_families = prepared_packages
11575            .iter()
11576            .map(|package| package.lease.family_count)
11577            .sum::<usize>();
11578        let selected_staged_cells = prepared
11579            .len()
11580            .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
11581        let actual_commit_work = (new_vertices as u64)
11582            .saturating_add(new_edges as u64)
11583            .saturating_add(committed as u64)
11584            .saturating_add(committed_spans)
11585            .saturating_add(prepared.len() as u64)
11586            .saturating_add(prepared_packages.len() as u64);
11587        let report = PreparedTargetGraphReport {
11588            request_id: request_id.unwrap_or_default(),
11589            requested_targets: targets.len(),
11590            normalized_regions: visited_regions.len(),
11591            normalized_target_list: normalized,
11592            selected_staged_cells,
11593            selected_source_families,
11594            retained_staged_cells: self.staged_formula_count(),
11595            selected_cells,
11596            retained_cells,
11597            widened_scope: scope,
11598            widening_reasons: reasons,
11599            revisions: assumptions,
11600            commit_window,
11601            estimated_scratch_bytes: scratch_bytes,
11602            observed_scratch_bytes,
11603            estimated_commit_work: (new_vertices as u64)
11604                .saturating_add(new_edges as u64)
11605                .saturating_add(prepared.len() as u64)
11606                .saturating_add(selected_package_records),
11607            actual_commit_work,
11608            outcome: PreparationOutcome::Prepared,
11609        };
11610        self.observe_target_preparation_report(&report);
11611        Ok(report)
11612    }
11613
11614    /// Build graph for all staged formulas.
11615    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11616        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
11617            engine.build_graph_all_unobserved()
11618        })
11619    }
11620
11621    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
11622        let selected = self.staged_formula_count();
11623        let started = crate::instant::FzInstant::now();
11624        self.resource_checkpoint(selected as u64)?;
11625        let scratch_bytes = (selected as u64).saturating_mul(256);
11626        let result = self.with_request_scratch(scratch_bytes, |engine| {
11627            let index_snapshot = engine.staged_formula_index.clone();
11628            let collected = std::mem::take(&mut engine.staged_formulas)
11629                .into_iter()
11630                .collect();
11631            engine.staged_formula_index.clear_all();
11632            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
11633        });
11634        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11635        result
11636    }
11637
11638    /// Build graph for specific sheets (consuming only those staged entries).
11639    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
11640        &mut self,
11641        sheets: I,
11642    ) -> Result<(), formualizer_parse::ExcelError> {
11643        let mut sheets = sheets.into_iter();
11644        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
11645            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
11646            engine.with_request_scratch(name_scratch, |engine| {
11647                // Allocation failure follows the baseline process-fatal policy; it is not a
11648                // recoverable resource error or a new staged-preparation route.
11649                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
11650                engine.charge_bounded_work(names.len() as u64)?;
11651                engine.build_graph_for_sheet_names_unobserved(names)
11652            })
11653        })
11654    }
11655
11656    fn build_graph_for_sheet_names_unobserved(
11657        &mut self,
11658        sheets: Vec<String>,
11659    ) -> Result<(), formualizer_parse::ExcelError> {
11660        let started = crate::instant::FzInstant::now();
11661        let selected = sheets
11662            .iter()
11663            .filter_map(|sheet| self.staged_formulas.get(sheet))
11664            .map(StagedSheet::len)
11665            .sum::<usize>();
11666        self.resource_checkpoint(selected as u64)?;
11667        let scratch_bytes = (selected as u64).saturating_mul(256);
11668        self.reserve_request_scratch(scratch_bytes)?;
11669        let index_snapshot = self.staged_formula_index.clone();
11670        let mut collected = Vec::new();
11671        for sheet in sheets {
11672            if let Some(staged) = self.staged_formulas.remove(&sheet) {
11673                self.index_removed_staged_sheet(&sheet, &staged);
11674                collected.push((sheet, staged));
11675            }
11676        }
11677        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
11678        self.release_request_scratch(scratch_bytes);
11679        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
11680        result
11681    }
11682
11683    fn build_graph_from_staged_batches(
11684        &mut self,
11685        collected: StagedFormulaBatches,
11686        share_parse_cache_across_sheets: bool,
11687        staged_index_snapshot: StagedFormulaIndex,
11688    ) -> Result<(), formualizer_parse::ExcelError> {
11689        if collected.is_empty() {
11690            return Ok(());
11691        }
11692        for (sheet, _) in &collected {
11693            let _ = self.add_sheet(sheet);
11694        }
11695
11696        let diagnostics_len = self.formula_parse_diagnostics.len();
11697        let mut collected = collected;
11698        let prepared = match self
11699            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
11700        {
11701            Ok(prepared) => prepared,
11702            Err(error) => {
11703                self.formula_parse_diagnostics.truncate(diagnostics_len);
11704                for (sheet, staged) in collected {
11705                    self.restore_staged_sheet(sheet, staged);
11706                }
11707                self.staged_formula_index = staged_index_snapshot;
11708                return Err(error);
11709            }
11710        };
11711        let (ordinary, compressed, direct) = prepared;
11712
11713        // Deferred packages are consumed exactly once once authority commit starts.
11714        // Pre-commit lock, replay, and parse failures restore them above for retry.
11715        if !ordinary.is_empty()
11716            && let Err(error) = self.ingest_formula_batches(ordinary)
11717        {
11718            self.formula_parse_diagnostics.truncate(diagnostics_len);
11719            for (sheet, staged) in collected {
11720                self.restore_staged_sheet(sheet, staged);
11721            }
11722            self.staged_formula_index = staged_index_snapshot;
11723            return Err(error);
11724        }
11725        if !compressed.is_empty() {
11726            let _ = self.ingest_compressed_formula_source_batches(compressed)?;
11727        }
11728        if !direct.is_empty() {
11729            let _ = self.finish_compressed_formula_sources(direct)?;
11730        }
11731        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
11732        Ok(())
11733    }
11734
11735    fn prepare_staged_formula_batches(
11736        &mut self,
11737        collected: &mut StagedFormulaBatches,
11738        share_parse_cache_across_sheets: bool,
11739    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
11740        let mut ordinary = Vec::new();
11741        let mut compressed = Vec::new();
11742        let mut direct = Vec::new();
11743        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
11744            rustc_hash::FxHashMap::default();
11745        cache.reserve(4096);
11746
11747        for (sheet, staged) in collected {
11748            if !share_parse_cache_across_sheets {
11749                cache.clear();
11750            }
11751            let mut entries: Vec<_> = staged
11752                .entries
11753                .iter()
11754                .cloned()
11755                .map(|(row, col, text)| (row, col, text, None))
11756                .collect();
11757            let mut deferred_source = None;
11758            let mut deferred_fallback = None;
11759            if let Some(package) = staged.deferred_package.as_mut() {
11760                if package.sheet_name != *sheet {
11761                    return Err(ExcelError::new(ExcelErrorKind::Value)
11762                        .with_message("deferred formula package sheet mismatch"));
11763                }
11764                let eligible: Vec<_> = package
11765                    .families
11766                    .iter()
11767                    .filter(|family| !package.invalidated.contains(&family.source_id))
11768                    .cloned()
11769                    .collect();
11770                let eligible_partitions: Vec<_> = package
11771                    .partitioned_families
11772                    .iter()
11773                    .filter(|family| !package.invalidated.contains(&family.source_id))
11774                    .cloned()
11775                    .collect();
11776                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
11777                    let mut preparation = self.prepare_source_formula_proposals(
11778                        sheet,
11779                        &eligible,
11780                        &eligible_partitions,
11781                        &package.partitioned_families,
11782                        package.report.source_formula_records_spooled,
11783                        Arc::clone(&package.replay),
11784                        &package.suppressed,
11785                    )?;
11786                    let replay_records = std::mem::take(&mut preparation.eager_replay);
11787                    let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
11788                        let accepted: BTreeMap<_, _> = preparation
11789                            .fragmented
11790                            .iter()
11791                            .map(|fragment| (fragment.source.source_id, &fragment.source))
11792                            .collect();
11793                        replay_records.into_iter().partition(|record| {
11794                            record.partition_owner.is_some_and(|owner| {
11795                                accepted.get(&owner).is_some_and(|source| {
11796                                    Self::exact_replay_record_is_prepared_partition_legacy(
11797                                        source, record,
11798                                    )
11799                                })
11800                            })
11801                        })
11802                    };
11803                    preparation.eager_replay = fragment_legacy;
11804                    entries.extend(ordered_fallback.into_iter().map(|record| {
11805                        (
11806                            record.row,
11807                            record.col,
11808                            record.text,
11809                            Some((record.source_order, record.family, record.partition_owner)),
11810                        )
11811                    }));
11812                    deferred_source = Some((package.report.clone(), preparation));
11813                } else {
11814                    let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
11815                    for partition in &eligible_partitions {
11816                        replay_disposition
11817                            .register_partition(partition, false)
11818                            .map_err(|reason| {
11819                                ExcelError::new(ExcelErrorKind::Value).with_message(reason)
11820                            })?;
11821                    }
11822                    replay_disposition
11823                        .extend_suppressed_excel_coords(package.suppressed.iter().copied());
11824                    let replayed = package
11825                        .replay
11826                        .lock()
11827                        .map_err(|_| {
11828                            ExcelError::new(ExcelErrorKind::Value)
11829                                .with_message("deferred formula spool lock poisoned")
11830                        })?
11831                        .replay(&replay_disposition)
11832                        .map_err(|message| {
11833                            ExcelError::new(ExcelErrorKind::Value).with_message(message)
11834                        })?;
11835                    entries.extend(replayed.into_iter().map(|record| {
11836                        (
11837                            record.row,
11838                            record.col,
11839                            record.text,
11840                            Some((record.source_order, record.family, record.partition_owner)),
11841                        )
11842                    }));
11843                    let mut report = package.report.clone();
11844                    report.source_spool_replays = report.source_spool_replays.saturating_add(1);
11845                    deferred_fallback =
11846                        Some((report, package.families.clone(), eligible_partitions));
11847                }
11848            }
11849
11850            let mut formulas = Vec::new();
11851            let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
11852            for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
11853                let key = if txt.starts_with('=') {
11854                    txt
11855                } else {
11856                    format!("={txt}")
11857                };
11858                let ast_id = if let Some(cached) = cache.get(&key) {
11859                    *cached
11860                } else {
11861                    let parsed = match formualizer_parse::parser::parse(&key) {
11862                        Ok(parsed) => Some(parsed),
11863                        Err(error) => self.handle_formula_parse_error(
11864                            sheet,
11865                            row,
11866                            col,
11867                            &key,
11868                            error.to_string(),
11869                        )?,
11870                    };
11871                    let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
11872                    cache.insert(key.clone(), ast_id);
11873                    ast_id
11874                };
11875
11876                if let Some(ast_id) = ast_id {
11877                    let mut formula = FormulaIngestRecord::new(
11878                        row,
11879                        col,
11880                        ast_id,
11881                        Some(Arc::<str>::from(key.clone())),
11882                    );
11883                    if let Some((order, family, owner)) = source_proof {
11884                        formula = formula.with_source_proof(order, family, owner);
11885                    } else if deferred_source.is_some() {
11886                        formula = formula.with_source_proof(
11887                            crate::engine::SourceFormulaOrder::new(
11888                                staged_order_base.saturating_add(entry_index as u64),
11889                            ),
11890                            None,
11891                            None,
11892                        );
11893                    }
11894                    formulas.push(formula);
11895                }
11896            }
11897
11898            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
11899            if let Some((report, preparation)) = deferred_source {
11900                direct.push((batch, report, preparation));
11901            } else if let Some((report, families, partitions)) = deferred_fallback {
11902                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
11903                    batch.sheet_name.clone(),
11904                    report,
11905                    families,
11906                    partitions,
11907                );
11908                compressed.push((batch, source_batch));
11909            } else if !batch.is_empty() {
11910                ordinary.push(batch);
11911            }
11912        }
11913        Ok((ordinary, compressed, direct))
11914    }
11915
11916    /// Begin bulk Arrow ingest for base values (Phase A)
11917    pub fn begin_bulk_ingest_arrow(
11918        &mut self,
11919    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
11920        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
11921    }
11922
11923    /// Begin bulk updates to Arrow store (Phase C)
11924    pub fn begin_bulk_update_arrow(
11925        &mut self,
11926    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
11927        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
11928    }
11929
11930    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
11931        self.graph.sheet_id(sheet).ok_or(
11932            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
11933                name: sheet.to_string(),
11934                reason: "Unknown sheet".to_string(),
11935            },
11936        )
11937    }
11938
11939    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
11940        if row_1based == 0 {
11941            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11942        }
11943        Ok(row_1based - 1)
11944    }
11945
11946    fn normalize_row_range_1based(
11947        start_row_1based: u32,
11948        end_row_1based: u32,
11949    ) -> Result<(u32, u32), crate::engine::EditorError> {
11950        if start_row_1based == 0 || end_row_1based == 0 {
11951            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
11952        }
11953        if start_row_1based > end_row_1based {
11954            return Err(crate::engine::EditorError::TransactionFailed {
11955                reason: "Row range start is greater than end".to_string(),
11956            });
11957        }
11958        Ok((start_row_1based - 1, end_row_1based - 1))
11959    }
11960
11961    fn invalidate_row_visibility_mask_cache(&self) {
11962        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
11963            cache.clear();
11964        }
11965    }
11966
11967    fn set_row_hidden_by_sheet_id(
11968        &mut self,
11969        sheet_id: SheetId,
11970        row0: u32,
11971        hidden: bool,
11972        source: RowVisibilitySource,
11973    ) -> bool {
11974        let changed = {
11975            let state = self.row_visibility.entry(sheet_id).or_default();
11976            state.set_row_hidden(row0, hidden, source)
11977        };
11978
11979        let remove_entry = self
11980            .row_visibility
11981            .get(&sheet_id)
11982            .map(|state| state.is_empty())
11983            .unwrap_or(false);
11984        if remove_entry {
11985            self.row_visibility.remove(&sheet_id);
11986        }
11987
11988        if changed {
11989            self.invalidate_row_visibility_mask_cache();
11990        }
11991
11992        changed
11993    }
11994
11995    fn set_rows_hidden_by_sheet_id(
11996        &mut self,
11997        sheet_id: SheetId,
11998        start_row0: u32,
11999        end_row0: u32,
12000        hidden: bool,
12001        source: RowVisibilitySource,
12002    ) -> bool {
12003        let changed = {
12004            let state = self.row_visibility.entry(sheet_id).or_default();
12005            state.set_rows_hidden(start_row0, end_row0, hidden, source)
12006        };
12007
12008        let remove_entry = self
12009            .row_visibility
12010            .get(&sheet_id)
12011            .map(|state| state.is_empty())
12012            .unwrap_or(false);
12013        if remove_entry {
12014            self.row_visibility.remove(&sheet_id);
12015        }
12016
12017        if changed {
12018            self.invalidate_row_visibility_mask_cache();
12019        }
12020
12021        changed
12022    }
12023
12024    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
12025        if count == 0 {
12026            return;
12027        }
12028        let mut changed = false;
12029        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12030            changed = state.insert_rows(before0, count);
12031            state.is_empty()
12032        } else {
12033            false
12034        };
12035        if remove_entry {
12036            self.row_visibility.remove(&sheet_id);
12037        }
12038        if changed {
12039            self.invalidate_row_visibility_mask_cache();
12040        }
12041    }
12042
12043    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
12044        if count == 0 {
12045            return;
12046        }
12047        let mut changed = false;
12048        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12049            changed = state.delete_rows(start0, count);
12050            state.is_empty()
12051        } else {
12052            false
12053        };
12054        if remove_entry {
12055            self.row_visibility.remove(&sheet_id);
12056        }
12057        if changed {
12058            self.invalidate_row_visibility_mask_cache();
12059        }
12060    }
12061
12062    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12063        if let crate::engine::ChangeEvent::SetRowVisibility {
12064            sheet_id,
12065            row0,
12066            source,
12067            old_hidden,
12068            ..
12069        } = event
12070        {
12071            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12072        }
12073    }
12074
12075    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12076        if let crate::engine::ChangeEvent::SetRowVisibility {
12077            sheet_id,
12078            row0,
12079            source,
12080            new_hidden,
12081            ..
12082        } = event
12083        {
12084            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12085        }
12086    }
12087
12088    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12089        for event in events.iter().rev() {
12090            self.apply_inverse_row_visibility_event(event);
12091        }
12092    }
12093
12094    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12095        for event in events {
12096            self.apply_forward_row_visibility_event(event);
12097        }
12098    }
12099
12100    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12101        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12102            sheet,
12103            row,
12104            col,
12105            old,
12106            ..
12107        } = event
12108        {
12109            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12110        }
12111    }
12112
12113    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12114        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12115            sheet,
12116            row,
12117            col,
12118            new,
12119            ..
12120        } = event
12121        {
12122            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12123        }
12124    }
12125
12126    /// Set a single cell's staged formula text to `target` (clearing it when
12127    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
12128    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12129        match target {
12130            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12131            None => {
12132                self.clear_staged_formula_text(sheet, row, col);
12133            }
12134        }
12135    }
12136
12137    pub fn set_row_hidden(
12138        &mut self,
12139        sheet: &str,
12140        row_1based: u32,
12141        hidden: bool,
12142        source: RowVisibilitySource,
12143    ) -> Result<(), crate::engine::EditorError> {
12144        self.observe_function_semantic_epoch()
12145            .map_err(crate::engine::EditorError::Excel)?;
12146        self.observe_function_semantic_epoch()
12147            .map_err(crate::engine::EditorError::Excel)?;
12148        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12149        let row0 = Self::normalize_row_1based(row_1based)?;
12150        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12151            self.record_formula_plane_structural_change(StructuralScope::Region(
12152                Region::whole_row(sheet_id, row0),
12153            ));
12154            self.mark_data_edited();
12155        }
12156        Ok(())
12157    }
12158
12159    pub fn set_rows_hidden(
12160        &mut self,
12161        sheet: &str,
12162        start_row_1based: u32,
12163        end_row_1based: u32,
12164        hidden: bool,
12165        source: RowVisibilitySource,
12166    ) -> Result<(), crate::engine::EditorError> {
12167        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12168        let (start_row0, end_row0) =
12169            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12170        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12171            if start_row0 == end_row0 {
12172                self.record_formula_plane_structural_change(StructuralScope::Region(
12173                    Region::whole_row(sheet_id, start_row0),
12174                ));
12175            } else {
12176                self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12177            }
12178            self.mark_data_edited();
12179        }
12180        Ok(())
12181    }
12182
12183    pub fn is_row_hidden(
12184        &self,
12185        sheet: &str,
12186        row_1based: u32,
12187        source: Option<RowVisibilitySource>,
12188    ) -> Option<bool> {
12189        let sheet_id = self.graph.sheet_id(sheet)?;
12190        let row0 = row_1based.checked_sub(1)?;
12191        Some(
12192            self.row_visibility
12193                .get(&sheet_id)
12194                .map(|state| state.is_row_hidden(row0, source))
12195                .unwrap_or(false),
12196        )
12197    }
12198
12199    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12200        let sheet_id = self.graph.sheet_id(sheet)?;
12201        Some(
12202            self.row_visibility
12203                .get(&sheet_id)
12204                .map(|state| state.version())
12205                .unwrap_or(0),
12206        )
12207    }
12208
12209    fn build_row_visibility_mask_for_view(
12210        &self,
12211        view: &RangeView<'_>,
12212        mode: VisibilityMaskMode,
12213    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12214        let sheet_rows = view.sheet().nrows as usize;
12215        if sheet_rows == 0 || view.start_row() >= sheet_rows {
12216            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12217        }
12218
12219        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12220        let start_row0 = view.start_row() as u32;
12221        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12222        let version = self
12223            .row_visibility
12224            .get(&sheet_id)
12225            .map(|state| state.version())
12226            .unwrap_or(0);
12227        let key = VisibilityMaskCacheKey {
12228            sheet_id,
12229            start_row0,
12230            end_row0,
12231            mode,
12232            version,
12233        };
12234
12235        if let Ok(cache) = self.row_visibility_mask_cache.read()
12236            && let Some(mask) = cache.get(&key)
12237        {
12238            #[cfg(test)]
12239            visibility_mask_test_hooks::inc_hit();
12240            return Some(mask.clone());
12241        }
12242
12243        #[cfg(test)]
12244        visibility_mask_test_hooks::inc_miss();
12245
12246        let state = self.row_visibility.get(&sheet_id);
12247        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12248        for row0 in start_row0..=end_row0 {
12249            let manual_hidden = state
12250                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12251                .unwrap_or(false);
12252            let filter_hidden = state
12253                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12254                .unwrap_or(false);
12255
12256            let include = match mode {
12257                VisibilityMaskMode::IncludeAll => true,
12258                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12259                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12260                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12261                    !(manual_hidden || filter_hidden)
12262                }
12263            };
12264            out.push(include);
12265        }
12266
12267        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12268        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12269            const MAX_CACHE_ENTRIES: usize = 4096;
12270            if cache.len() >= MAX_CACHE_ENTRIES {
12271                cache.clear();
12272                #[cfg(test)]
12273                visibility_mask_test_hooks::inc_eviction();
12274            }
12275            cache.insert(key, mask.clone());
12276        }
12277
12278        Some(mask)
12279    }
12280
12281    fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
12282        match error {
12283            crate::engine::EditorError::Excel(error) => error,
12284            other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
12285        }
12286    }
12287
12288    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
12289        let changes =
12290            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
12291        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
12292        let provider_revision = self.resolver.planning_semantic_revision();
12293        let provider_changed = provider_revision != self.function_provider_revision_seen;
12294        if !global_changed && !provider_changed {
12295            return Ok(false);
12296        }
12297
12298        let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
12299        let sheets: BTreeSet<_> = self
12300            .graph
12301            .formula_authority()
12302            .plane
12303            .spans
12304            .active_spans()
12305            .filter_map(|span| {
12306                let Some(template) = self
12307                    .graph
12308                    .formula_authority()
12309                    .plane
12310                    .templates
12311                    .get(span.template_id)
12312                else {
12313                    return (provider_changed || global_changed && !changes.complete)
12314                        .then_some(span.domain.sheet_id());
12315                };
12316                let Some(ast) = self
12317                    .graph
12318                    .data_store()
12319                    .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
12320                else {
12321                    return (provider_changed || global_changed && !changes.complete)
12322                        .then_some(span.domain.sheet_id());
12323                };
12324                let global_affected = global_changed
12325                    && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
12326                let provider_affected = provider_changed && Self::ast_contains_function(&ast);
12327                (global_affected || provider_affected).then_some(span.domain.sheet_id())
12328            })
12329            .collect();
12330        let invalidated = !sheets.is_empty();
12331        for sheet_id in sheets {
12332            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
12333                .map_err(Self::editor_error_to_excel)?;
12334        }
12335        if global_changed && !changed.is_empty() || provider_changed {
12336            self.cached_static_schedule = None;
12337        }
12338        self.function_semantic_epoch_seen = changes.epoch;
12339        self.function_provider_revision_seen = provider_revision;
12340        Ok(invalidated)
12341    }
12342
12343    pub(crate) fn ast_uses_changed_function(
12344        ast: &ASTNode,
12345        changed: &BTreeSet<(String, String)>,
12346    ) -> bool {
12347        match &ast.node_type {
12348            ASTNodeType::Function { name, args } => {
12349                let normalized = name.to_uppercase();
12350                let mut spellings = vec![(String::new(), normalized.clone())];
12351                let mut stripped = normalized.as_str();
12352                loop {
12353                    let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
12354                        .iter()
12355                        .find_map(|prefix| stripped.strip_prefix(prefix))
12356                    else {
12357                        break;
12358                    };
12359                    stripped = rest;
12360                    spellings.push((String::new(), stripped.to_string()));
12361                }
12362                let resolved = crate::function_registry::resolve("", name);
12363                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
12364                    || resolved.as_ref().is_some_and(|resolved| {
12365                        changed.contains(&(
12366                            resolved.namespace.clone(),
12367                            resolved.canonical_name.clone(),
12368                        ))
12369                    });
12370                directly_changed
12371                    || resolved.is_none()
12372                    || args
12373                        .iter()
12374                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
12375            }
12376            ASTNodeType::Call { callee, args } => {
12377                Self::ast_uses_changed_function(callee, changed)
12378                    || args
12379                        .iter()
12380                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
12381            }
12382            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
12383            ASTNodeType::BinaryOp { left, right, .. } => {
12384                Self::ast_uses_changed_function(left, changed)
12385                    || Self::ast_uses_changed_function(right, changed)
12386            }
12387            ASTNodeType::Array(rows) => rows
12388                .iter()
12389                .flatten()
12390                .any(|node| Self::ast_uses_changed_function(node, changed)),
12391            _ => false,
12392        }
12393    }
12394
12395    fn ast_contains_function(ast: &ASTNode) -> bool {
12396        match &ast.node_type {
12397            ASTNodeType::Function { .. } => true,
12398            ASTNodeType::Call { callee, args } => {
12399                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
12400            }
12401            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
12402            ASTNodeType::BinaryOp { left, right, .. } => {
12403                Self::ast_contains_function(left) || Self::ast_contains_function(right)
12404            }
12405            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
12406            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
12407        }
12408    }
12409
12410    fn demote_span_containing_cell_for_write(
12411        &mut self,
12412        sheet_id: SheetId,
12413        row0: u32,
12414        col0: u32,
12415    ) -> Result<(), crate::engine::EditorError> {
12416        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12417            return Ok(());
12418        }
12419        let placement = PlacementCoord::new(sheet_id, row0, col0);
12420        let inside_active_span = self
12421            .graph
12422            .formula_authority()
12423            .plane
12424            .spans
12425            .find_at(placement)
12426            .is_some();
12427        if inside_active_span {
12428            self.demote_spans_preserving_computed_overlays(
12429                sheet_id,
12430                Region::point(sheet_id, row0, col0),
12431            )?;
12432        }
12433        Ok(())
12434    }
12435
12436    fn demote_spans_preserving_computed_overlays(
12437        &mut self,
12438        _sheet_id: SheetId,
12439        affected_region: Region,
12440    ) -> Result<(), crate::engine::EditorError> {
12441        // Per-cell write inside a span (or whole-sheet demote via remove_sheet):
12442        // not a structural axis shift. Demote every span whose result or read
12443        // region intersects `affected_region`; leave disjoint spans untouched.
12444        self.demote_spans_for_structural_op_impl(None, affected_region, false)
12445    }
12446
12447    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
12448        Region::rows_from(sheet_id, start_row0)
12449    }
12450
12451    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
12452        Region::cols_from(sheet_id, start_col0)
12453    }
12454
12455    fn structural_dirty_region_for_domain(
12456        domain: &PlacementDomain,
12457        op: StructuralOp,
12458    ) -> Option<Region> {
12459        let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
12460            PlacementDomain::RowRun {
12461                sheet_id,
12462                row_start,
12463                row_end,
12464                col,
12465            } => (*sheet_id, *row_start, *row_end, *col, *col),
12466            PlacementDomain::ColRun {
12467                sheet_id,
12468                row,
12469                col_start,
12470                col_end,
12471            } => (*sheet_id, *row, *row, *col_start, *col_end),
12472            PlacementDomain::Rect {
12473                sheet_id,
12474                row_start,
12475                row_end,
12476                col_start,
12477                col_end,
12478            } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
12479        };
12480        match op {
12481            StructuralOp::InsertRows {
12482                sheet_id: edited,
12483                before,
12484                ..
12485            } if sheet_id == edited && row_end >= before => Some(Region::rect(
12486                sheet_id,
12487                row_start.max(before),
12488                row_end,
12489                col_start,
12490                col_end,
12491            )),
12492            StructuralOp::DeleteRows {
12493                sheet_id: edited,
12494                start,
12495                ..
12496            } if sheet_id == edited && row_end >= start => Some(Region::rect(
12497                sheet_id,
12498                row_start.max(start),
12499                row_end,
12500                col_start,
12501                col_end,
12502            )),
12503            StructuralOp::InsertColumns {
12504                sheet_id: edited,
12505                before,
12506                ..
12507            } if sheet_id == edited && col_end >= before => Some(Region::rect(
12508                sheet_id,
12509                row_start,
12510                row_end,
12511                col_start.max(before),
12512                col_end,
12513            )),
12514            StructuralOp::DeleteColumns {
12515                sheet_id: edited,
12516                start,
12517                ..
12518            } if sheet_id == edited && col_end >= start => Some(Region::rect(
12519                sheet_id,
12520                row_start,
12521                row_end,
12522                col_start.max(start),
12523                col_end,
12524            )),
12525            _ => None,
12526        }
12527    }
12528
12529    fn span_result_region_intersects_affected(
12530        span: &crate::formula_plane::runtime::FormulaSpan,
12531        affected_region: &Region,
12532    ) -> bool {
12533        Region::from_domain(span.result_region.domain()).intersects(affected_region)
12534    }
12535
12536    fn span_any_read_region_intersects_affected(
12537        plane: &FormulaPlane,
12538        span: &crate::formula_plane::runtime::FormulaSpan,
12539        affected_region: &Region,
12540    ) -> bool {
12541        span.read_summary_id
12542            .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
12543            .is_some_and(|summary| {
12544                summary
12545                    .dependencies
12546                    .iter()
12547                    .any(|dependency| dependency.read_region.intersects(affected_region))
12548            })
12549    }
12550
12551    fn insert_formula_plane_dirty_coords_for_span(
12552        &self,
12553        span_ref: FormulaSpanRef,
12554        dirty: ProducerDirtyDomain,
12555        out: &mut FxHashSet<(SheetId, u32, u32)>,
12556    ) -> Result<(), crate::engine::EditorError> {
12557        let authority = self.graph.formula_authority();
12558        let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
12559            ExcelError::new(ExcelErrorKind::NImpl)
12560                .with_message("FormulaPlane dirty transfer referenced a stale span")
12561        })?;
12562        match dirty {
12563            ProducerDirtyDomain::Whole => {
12564                out.extend(
12565                    span.domain
12566                        .iter()
12567                        .map(|coord| (coord.sheet_id, coord.row, coord.col)),
12568                );
12569            }
12570            ProducerDirtyDomain::Cells(cells) => {
12571                out.extend(cells.into_iter().filter_map(|key| {
12572                    let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
12573                    span.domain
12574                        .contains(coord)
12575                        .then_some((coord.sheet_id, coord.row, coord.col))
12576                }));
12577            }
12578            ProducerDirtyDomain::Regions(regions) => {
12579                out.extend(span.domain.iter().filter_map(|coord| {
12580                    let key = crate::formula_plane::region_index::RegionKey::from(coord);
12581                    regions
12582                        .iter()
12583                        .any(|region| region.contains_key(key))
12584                        .then_some((coord.sheet_id, coord.row, coord.col))
12585                }));
12586            }
12587        }
12588        Ok(())
12589    }
12590
12591    fn compute_current_formula_plane_dirty_result_coords(
12592        &self,
12593    ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
12594        use crate::formula_plane::producer::compute_dirty_closure;
12595
12596        let authority = self.graph.formula_authority();
12597        let span_refs = authority.active_span_refs();
12598        let span_refs_by_id = span_refs
12599            .iter()
12600            .copied()
12601            .map(|span_ref| (span_ref.id, span_ref))
12602            .collect::<BTreeMap<_, _>>();
12603        let mut dirty_coords = FxHashSet::default();
12604
12605        for span_ref in self.graph.pending_formula_dirty_whole_spans() {
12606            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12607                self.insert_formula_plane_dirty_coords_for_span(
12608                    span_ref,
12609                    ProducerDirtyDomain::Whole,
12610                    &mut dirty_coords,
12611                )?;
12612            }
12613        }
12614        for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
12615            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
12616                self.insert_formula_plane_dirty_coords_for_span(
12617                    span_ref,
12618                    ProducerDirtyDomain::Regions(vec![region]),
12619                    &mut dirty_coords,
12620                )?;
12621            }
12622        }
12623
12624        let pending_changed_regions = self
12625            .graph
12626            .pending_formula_dirty_regions()
12627            .collect::<Vec<_>>();
12628        if pending_changed_regions.is_empty() {
12629            return Ok(dirty_coords);
12630        }
12631
12632        let closure = compute_dirty_closure(
12633            &authority.consumer_reads,
12634            pending_changed_regions,
12635            |producer| authority.producer_results.producer_result_region(producer),
12636        );
12637        if closure.incomplete {
12638            for span_ref in span_refs {
12639                self.insert_formula_plane_dirty_coords_for_span(
12640                    span_ref,
12641                    ProducerDirtyDomain::Whole,
12642                    &mut dirty_coords,
12643                )?;
12644            }
12645            return Ok(dirty_coords);
12646        }
12647        for work in closure.work {
12648            let FormulaProducerId::Span(span_id) = work.producer else {
12649                continue;
12650            };
12651            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12652                continue;
12653            };
12654            self.insert_formula_plane_dirty_coords_for_span(
12655                span_ref,
12656                work.dirty,
12657                &mut dirty_coords,
12658            )?;
12659        }
12660        for fallback in closure.fallbacks {
12661            let FormulaProducerId::Span(span_id) = fallback.consumer else {
12662                continue;
12663            };
12664            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
12665                continue;
12666            };
12667            self.insert_formula_plane_dirty_coords_for_span(
12668                span_ref,
12669                ProducerDirtyDomain::Whole,
12670                &mut dirty_coords,
12671            )?;
12672        }
12673
12674        Ok(dirty_coords)
12675    }
12676
12677    /// Demote active FormulaPlane spans affected by a structural edit on `sheet_id`.
12678    ///
12679    /// This is the conservative Option-A correctness path for structural edits: rather than
12680    /// attempting to transform FormulaPlane span domains/templates/indexes, materialize each span
12681    /// placement as an ordinary legacy graph formula at its current coordinate, remove the span,
12682    /// and let the existing VertexEditor structural machinery shift/delete those vertices and
12683    /// adjust their ASTs.  Spans whose formula domain is on `sheet_id` are affected directly; spans
12684    /// on other sheets are also affected when one of their retained read regions targets
12685    /// `sheet_id`, because those read-region coordinates become stale after row/column shifts.
12686    fn demote_spans_for_structural_op(
12687        &mut self,
12688        op: StructuralOp,
12689        affected_region: Region,
12690    ) -> Result<(), crate::engine::EditorError> {
12691        if op.count() == 0 {
12692            return Ok(());
12693        }
12694        // #171: after an axis edit, retained read-summary relocation cannot be
12695        // used as a proof of disconnection. Fail closed for this engine.
12696        self.legacy_island_structural_summaries_trusted = false;
12697        self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
12698    }
12699
12700    fn indexed_structural_candidate_span_refs(
12701        &self,
12702        affected_region: Region,
12703    ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
12704        use crate::formula_plane::region_index::BoundedRegionQueryResult;
12705
12706        let authority = self.graph.formula_authority();
12707        if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
12708            return Err(ExcelError::new(ExcelErrorKind::NImpl)
12709                .with_message("FormulaPlane structural candidate indexes are stale")
12710                .into());
12711        }
12712        let indexed_region = match affected_region.axis_ranges() {
12713            (
12714                crate::formula_plane::region_index::AxisRange::From(start),
12715                crate::formula_plane::region_index::AxisRange::All,
12716            ) => Region {
12717                sheet_id: affected_region.sheet_id(),
12718                rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12719                cols: crate::formula_plane::region_index::AxisRange::All,
12720            },
12721            (
12722                crate::formula_plane::region_index::AxisRange::All,
12723                crate::formula_plane::region_index::AxisRange::From(start),
12724            ) => Region {
12725                sheet_id: affected_region.sheet_id(),
12726                rows: crate::formula_plane::region_index::AxisRange::All,
12727                cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
12728            },
12729            _ => affected_region,
12730        };
12731        let max_candidates = self.config.max_formula_plane_cache_candidates;
12732        let result_query = match authority
12733            .producer_results
12734            .query_bounded(indexed_region, max_candidates)
12735        {
12736            BoundedRegionQueryResult::Complete(result) => result,
12737            BoundedRegionQueryResult::Incomplete {
12738                observed_candidates,
12739            } => {
12740                return Err(ExcelError::new(ExcelErrorKind::NImpl)
12741                    .with_message(format!(
12742                        "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
12743                    ))
12744                    .into());
12745            }
12746        };
12747        let remaining = max_candidates.saturating_sub(result_query.matches.len());
12748        let read_query = match authority
12749            .consumer_reads
12750            .query_changed_region_bounded(indexed_region, remaining)
12751        {
12752            BoundedRegionQueryResult::Complete(result) => result,
12753            BoundedRegionQueryResult::Incomplete { .. } => {
12754                return Err(ExcelError::new(ExcelErrorKind::NImpl)
12755                    .with_message("FormulaPlane structural read candidate limit exceeded")
12756                    .into());
12757            }
12758        };
12759
12760        let mut ids = BTreeSet::new();
12761        for matched in result_query.matches {
12762            if let FormulaProducerId::Span(id) = matched.value.producer {
12763                ids.insert(id);
12764            }
12765        }
12766        for matched in read_query.matches {
12767            if let FormulaProducerId::Span(id) = matched.value.consumer {
12768                ids.insert(id);
12769            }
12770        }
12771
12772        ids.into_iter()
12773            .map(|id| {
12774                authority.plane.spans.current_ref(id).ok_or_else(|| {
12775                    crate::engine::EditorError::Excel(
12776                        ExcelError::new(ExcelErrorKind::NImpl)
12777                            .with_message("FormulaPlane structural index referenced a stale span"),
12778                    )
12779                })
12780            })
12781            .collect()
12782    }
12783
12784    fn demote_spans_for_structural_op_impl(
12785        &mut self,
12786        op: Option<StructuralOp>,
12787        affected_region: Region,
12788        clear_computed_overlays: bool,
12789    ) -> Result<(), crate::engine::EditorError> {
12790        struct SpanPlan {
12791            span_ref: FormulaSpanRef,
12792            sheet_id: SheetId,
12793            ast: ASTNode,
12794            origin_row: u32,
12795            origin_col: u32,
12796            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12797            placements: Vec<(u32, u32)>,
12798        }
12799
12800        fn substitute_literal_slots_for_template_placement(
12801            ast: &ASTNode,
12802            binding: &[LiteralValue],
12803        ) -> ASTNode {
12804            fn clone_with_slots(
12805                ast: &ASTNode,
12806                binding: &[LiteralValue],
12807                next: &mut usize,
12808                in_array: bool,
12809            ) -> ASTNode {
12810                let node_type = match &ast.node_type {
12811                    ASTNodeType::Literal(_) if !in_array => {
12812                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
12813                        *next = next.saturating_add(1);
12814                        ASTNodeType::Literal(value)
12815                    }
12816                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
12817                    ASTNodeType::Omitted => ASTNodeType::Omitted,
12818                    ASTNodeType::Reference {
12819                        original,
12820                        reference,
12821                    } => ASTNodeType::Reference {
12822                        original: original.clone(),
12823                        reference: reference.clone(),
12824                    },
12825                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
12826                        op: op.clone(),
12827                        expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
12828                    },
12829                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
12830                        op: op.clone(),
12831                        left: Box::new(clone_with_slots(left, binding, next, in_array)),
12832                        right: Box::new(clone_with_slots(right, binding, next, in_array)),
12833                    },
12834                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
12835                        name: name.clone(),
12836                        args: args
12837                            .iter()
12838                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
12839                            .collect(),
12840                    },
12841                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
12842                        callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
12843                        args: args
12844                            .iter()
12845                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
12846                            .collect(),
12847                    },
12848                    ASTNodeType::Array(rows) => ASTNodeType::Array(
12849                        rows.iter()
12850                            .map(|row| {
12851                                row.iter()
12852                                    .map(|cell| clone_with_slots(cell, binding, next, true))
12853                                    .collect()
12854                            })
12855                            .collect(),
12856                    ),
12857                };
12858                ASTNode::new(node_type, ast.source_token.clone())
12859            }
12860            let mut next = 0usize;
12861            clone_with_slots(ast, binding, &mut next, false)
12862        }
12863
12864        let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
12865        self.formula_plane_structural_span_candidates = self
12866            .formula_plane_structural_span_candidates
12867            .saturating_add(span_refs.len() as u64);
12868        if span_refs.is_empty() {
12869            return Ok(());
12870        }
12871        let dirty_span_coords = if clear_computed_overlays {
12872            FxHashSet::default()
12873        } else {
12874            self.compute_current_formula_plane_dirty_result_coords()?
12875        };
12876
12877        /// A template AST rewrite accompanying a span shift: the op displaced
12878        /// the target of an absolute read, which origin relocation cannot
12879        /// repoint (issue #168). The adjusted AST is re-interned as a fresh
12880        /// arena AST + template record at the apply site; keys are re-derived
12881        /// from the rewritten AST so the new record can never collide with
12882        /// the (immutable, possibly shared) pre-rewrite template.
12883        struct SpanTemplateRewrite {
12884            adjusted_ast: ASTNode,
12885            exact_canonical_key: Arc<str>,
12886            parameterized_canonical_key: Arc<str>,
12887            formula_text: Option<Arc<str>>,
12888            /// Canonical form of the rewritten AST at the new origin, used to
12889            /// rebuild the binding set's template-derived slot data (the
12890            /// literal slot map is keyed by arena node ids of the template
12891            /// AST; value-ref slot patterns carry canonical reference
12892            /// coordinates — both go stale across a rewrite).
12893            canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
12894            value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
12895        }
12896
12897        struct ShiftPlan {
12898            span_ref: FormulaSpanRef,
12899            template_id: crate::formula_plane::ids::FormulaTemplateId,
12900            new_origin_row: u32,
12901            new_origin_col: u32,
12902            new_domain: crate::formula_plane::runtime::PlacementDomain,
12903            new_read_summary: Option<SpanReadSummary>,
12904            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12905            force_binding_residual_axes: bool,
12906            rewrite: Option<SpanTemplateRewrite>,
12907            dirty_region: Option<Region>,
12908        }
12909
12910        /// StructuralOp and ShiftOperation are field-for-field identical
12911        /// (both 0-based); the adjuster type is the legacy shared one.
12912        fn shift_operation_for(
12913            op: StructuralOp,
12914        ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
12915            use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
12916            match op {
12917                StructuralOp::InsertRows {
12918                    sheet_id,
12919                    before,
12920                    count,
12921                } => ShiftOperation::InsertRows {
12922                    sheet_id,
12923                    before,
12924                    count,
12925                },
12926                StructuralOp::DeleteRows {
12927                    sheet_id,
12928                    start,
12929                    count,
12930                } => ShiftOperation::DeleteRows {
12931                    sheet_id,
12932                    start,
12933                    count,
12934                },
12935                StructuralOp::InsertColumns {
12936                    sheet_id,
12937                    before,
12938                    count,
12939                } => ShiftOperation::InsertColumns {
12940                    sheet_id,
12941                    before,
12942                    count,
12943                },
12944                StructuralOp::DeleteColumns {
12945                    sheet_id,
12946                    start,
12947                    count,
12948                } => ShiftOperation::DeleteColumns {
12949                    sheet_id,
12950                    start,
12951                    count,
12952                },
12953            }
12954        }
12955
12956        /// Split a straddled span into an unshifted upper half (which keeps the
12957        /// span id via `replace_span_geometry`) and a shifted lower half (a
12958        /// fresh span). Domains and origins here are POST-insert for the lower
12959        /// half, mirroring `ShiftPlan`.
12960        struct SplitPlan {
12961            span_ref: FormulaSpanRef,
12962            sheet_id: SheetId,
12963            template_id: crate::formula_plane::ids::FormulaTemplateId,
12964            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
12965            is_constant_result: bool,
12966            upper_domain: crate::formula_plane::runtime::PlacementDomain,
12967            upper_read_summary: Option<SpanReadSummary>,
12968            lower_new_origin_row: u32,
12969            lower_new_origin_col: u32,
12970            lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
12971            lower_new_read_summary: Option<SpanReadSummary>,
12972            lower_force_binding_residual_axes: bool,
12973            lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
12974            upper_dirty_region: Option<Region>,
12975            lower_dirty_region: Option<Region>,
12976        }
12977
12978        fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
12979            u32::try_from(i64::from(value).checked_add(delta)?).ok()
12980        }
12981
12982        /// Structural transform of a read summary for a NO-REWRITE shift.
12983        /// Regions move with the op; projection rules keep their absolute
12984        /// bounds (the AST is untouched) but must shift their RELATIVE
12985        /// bounds by `-origin_delta`: rules store placement-relative
12986        /// offsets, which equal `authored_coordinate - template_origin`,
12987        /// and a moved origin over an unchanged AST changes that difference
12988        /// (otherwise incremental dirty projection silently goes stale —
12989        /// issue #168 family). Rewrite plans do NOT use this transform;
12990        /// their summaries are re-derived from the rewritten canonical
12991        /// template instead.
12992        fn shifted_read_summary(
12993            read_summary: &SpanReadSummary,
12994            new_result_region: Region,
12995            op: StructuralOp,
12996            row_delta: i64,
12997            col_delta: i64,
12998            origin_row_delta: i64,
12999            origin_col_delta: i64,
13000        ) -> Option<SpanReadSummary> {
13001            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13002            for dependency in &read_summary.dependencies {
13003                let read_region = match op.classify_region(dependency.read_region) {
13004                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13005                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13006                        dependency.read_region
13007                    }
13008                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13009                        ..
13010                    } => dependency
13011                        .read_region
13012                        .project_through_axis_shift(row_delta, col_delta)?,
13013                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles
13014                    | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13015                        return None;
13016                    }
13017                };
13018                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13019                    read_region,
13020                    projection: dependency
13021                        .projection
13022                        .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
13023                });
13024            }
13025            Some(SpanReadSummary {
13026                result_region: new_result_region,
13027                dependencies,
13028            })
13029        }
13030
13031        fn compact_axis_through_delete(
13032            min: u32,
13033            max: u32,
13034            start: u32,
13035            count: u32,
13036        ) -> Option<(u32, u32)> {
13037            let end = start.saturating_add(count);
13038            if max < start || min >= end {
13039                return Some((min.saturating_sub(count), max.saturating_sub(count)));
13040            }
13041            let keeps_left = min < start;
13042            let keeps_right = max >= end;
13043            match (keeps_left, keeps_right) {
13044                (false, false) => None,
13045                (true, false) => Some((min, start.checked_sub(1)?)),
13046                (false, true) => Some((start, max.checked_sub(count)?)),
13047                (true, true) => Some((min, max.checked_sub(count)?)),
13048            }
13049        }
13050
13051        fn compact_domain_through_delete(
13052            domain: &PlacementDomain,
13053            op: StructuralOp,
13054        ) -> Option<PlacementDomain> {
13055            match (domain, op) {
13056                (
13057                    PlacementDomain::RowRun {
13058                        sheet_id,
13059                        row_start,
13060                        row_end,
13061                        col,
13062                    },
13063                    StructuralOp::DeleteRows { start, count, .. },
13064                ) => {
13065                    let (row_start, row_end) =
13066                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13067                    Some(PlacementDomain::row_run(
13068                        *sheet_id, row_start, row_end, *col,
13069                    ))
13070                }
13071                (
13072                    PlacementDomain::Rect {
13073                        sheet_id,
13074                        row_start,
13075                        row_end,
13076                        col_start,
13077                        col_end,
13078                    },
13079                    StructuralOp::DeleteRows { start, count, .. },
13080                ) => {
13081                    let (row_start, row_end) =
13082                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13083                    Some(PlacementDomain::rect(
13084                        *sheet_id, row_start, row_end, *col_start, *col_end,
13085                    ))
13086                }
13087                (
13088                    PlacementDomain::ColRun {
13089                        sheet_id,
13090                        row,
13091                        col_start,
13092                        col_end,
13093                    },
13094                    StructuralOp::DeleteColumns { start, count, .. },
13095                ) => {
13096                    let (col_start, col_end) =
13097                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13098                    Some(PlacementDomain::col_run(
13099                        *sheet_id, *row, col_start, col_end,
13100                    ))
13101                }
13102                (
13103                    PlacementDomain::Rect {
13104                        sheet_id,
13105                        row_start,
13106                        row_end,
13107                        col_start,
13108                        col_end,
13109                    },
13110                    StructuralOp::DeleteColumns { start, count, .. },
13111                ) => {
13112                    let (col_start, col_end) =
13113                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13114                    Some(PlacementDomain::rect(
13115                        *sheet_id, *row_start, *row_end, col_start, col_end,
13116                    ))
13117                }
13118                _ => None,
13119            }
13120        }
13121
13122        fn compact_axis_range_through_delete(
13123            axis: crate::formula_plane::region_index::AxisRange,
13124            start: u32,
13125            count: u32,
13126        ) -> Option<crate::formula_plane::region_index::AxisRange> {
13127            use crate::formula_plane::region_index::AxisRange;
13128            match axis {
13129                AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13130                    .map(|(point, _)| AxisRange::Point(point)),
13131                AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13132                    .map(|(min, max)| AxisRange::Span(min, max)),
13133                AxisRange::All => Some(AxisRange::All),
13134                AxisRange::From(_) | AxisRange::To(_) => None,
13135            }
13136        }
13137
13138        fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13139            let (rows, cols) = region.axis_ranges();
13140            match op {
13141                StructuralOp::DeleteRows {
13142                    sheet_id,
13143                    start,
13144                    count,
13145                } if region.sheet_id() == sheet_id => Some(Region {
13146                    sheet_id,
13147                    rows: compact_axis_range_through_delete(rows, start, count)?,
13148                    cols,
13149                }),
13150                StructuralOp::DeleteColumns {
13151                    sheet_id,
13152                    start,
13153                    count,
13154                } if region.sheet_id() == sheet_id => Some(Region {
13155                    sheet_id,
13156                    rows,
13157                    cols: compact_axis_range_through_delete(cols, start, count)?,
13158                }),
13159                _ => Some(region),
13160            }
13161        }
13162
13163        fn compact_read_summary_through_delete(
13164            read_summary: &SpanReadSummary,
13165            new_result_region: Region,
13166            op: StructuralOp,
13167        ) -> Option<SpanReadSummary> {
13168            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13169            for dependency in &read_summary.dependencies {
13170                let read_region = match op.classify_region(dependency.read_region) {
13171                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13172                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13173                        dependency.read_region
13174                    }
13175                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13176                        ..
13177                    } => {
13178                        let (row_delta, col_delta) = op.axis_shift_delta();
13179                        dependency
13180                            .read_region
13181                            .project_through_axis_shift(row_delta, col_delta)?
13182                    }
13183                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13184                        compact_region_through_delete(dependency.read_region, op)?
13185                    }
13186                    crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13187                        return None;
13188                    }
13189                };
13190                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13191                    read_region,
13192                    projection: dependency.projection,
13193                });
13194            }
13195            Some(SpanReadSummary {
13196                result_region: new_result_region,
13197                dependencies,
13198            })
13199        }
13200
13201        /// Re-derive a span read summary for a sub-domain of the original
13202        /// result region by replaying each retained projection rule against
13203        /// the half's result region — the same derivation ingest performs in
13204        /// `SpanReadSummary::from_formula_summary`, minus the dependency
13205        /// re-analysis. Returns None (caller demotes) when any rule cannot
13206        /// produce explicit read regions.
13207        fn rederive_read_summary_for_result(
13208            read_summary: &SpanReadSummary,
13209            result_region: Region,
13210        ) -> Option<SpanReadSummary> {
13211            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13212            for dependency in &read_summary.dependencies {
13213                let sheet_id = dependency.read_region.sheet_id();
13214                let read_regions = dependency
13215                    .projection
13216                    .read_regions_for_result(sheet_id, result_region)
13217                    .ok()?;
13218                for read_region in read_regions {
13219                    let dependency = crate::formula_plane::producer::SpanReadDependency {
13220                        read_region,
13221                        projection: dependency.projection,
13222                    };
13223                    if !dependencies.contains(&dependency) {
13224                        dependencies.push(dependency);
13225                    }
13226                }
13227            }
13228            Some(SpanReadSummary {
13229                result_region,
13230                dependencies,
13231            })
13232        }
13233
13234        /// Plan a conservative span split for a mid-domain insert. Returns
13235        /// None whenever the split is not provably clean — missing template,
13236        /// masked span, non-trivial literal bindings, an unsplittable domain,
13237        /// a half that does not re-classify to NoOp (upper) / Shift (lower),
13238        /// or an overlay punch-out that straddles the boundary — in which case
13239        /// the caller falls back to demoting the whole span.
13240        fn plan_span_split(
13241            authority: &crate::formula_plane::authority::FormulaAuthority,
13242            span: &crate::formula_plane::runtime::FormulaSpan,
13243            span_ref: FormulaSpanRef,
13244            read_summary: Option<&SpanReadSummary>,
13245            op: StructuralOp,
13246        ) -> Option<SplitPlan> {
13247            use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13248
13249            if span.intrinsic_mask_id.is_some() {
13250                // v1 does not slice intrinsic masks.
13251                return None;
13252            }
13253            let binding_set = span
13254                .binding_set_id
13255                .and_then(|id| authority.plane.binding_sets.get(id));
13256            if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13257                // Dictionary/affine per-placement literal bindings are keyed by
13258                // domain ordinal; splitting re-bases the lower half's ordinals.
13259                // Only trivially-uniform bindings are safe to carry across.
13260                return None;
13261            }
13262
13263            let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13264            let upper_result_region = Region::from_domain(&upper_domain);
13265            let lower_result_region = Region::from_domain(&lower_domain);
13266            let (upper_read_summary, lower_read_summary) = match read_summary {
13267                Some(summary) => (
13268                    Some(rederive_read_summary_for_result(
13269                        summary,
13270                        upper_result_region,
13271                    )?),
13272                    Some(rederive_read_summary_for_result(
13273                        summary,
13274                        lower_result_region,
13275                    )?),
13276                ),
13277                None => (None, None),
13278            };
13279
13280            // Re-classify each half in the pre-insert frame. The upper half
13281            // must be a strict no-op (unshifted result, unshifted reads); the
13282            // lower half must be a clean whole-span shift.
13283            let upper_span = crate::formula_plane::runtime::FormulaSpan {
13284                domain: upper_domain.clone(),
13285                result_region: ResultRegion::scalar_cells(upper_domain.clone()),
13286                ..span.clone()
13287            };
13288            if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
13289                != SpanShiftPlan::NoOp
13290            {
13291                return None;
13292            }
13293            let lower_span = crate::formula_plane::runtime::FormulaSpan {
13294                domain: lower_domain.clone(),
13295                result_region: ResultRegion::scalar_cells(lower_domain.clone()),
13296                ..span.clone()
13297            };
13298            let SpanShiftPlan::Shift {
13299                row_delta,
13300                col_delta,
13301                origin_row_delta,
13302                origin_col_delta,
13303                rewrite_absolute_reads,
13304            } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
13305            else {
13306                return None;
13307            };
13308            if rewrite_absolute_reads {
13309                // A displaced absolute read in the lower half implies the
13310                // upper half reads it too and cannot be a NoOp, so this is
13311                // unreachable when the upper check above passed; keep the
13312                // guard so a future classifier change cannot silently skip
13313                // the template rewrite.
13314                return None;
13315            }
13316
13317            // Span relocation is the ownership boundary for demotion and
13318            // formula lookup. Canonically shared templates may have a different
13319            // authoring origin, so never derive split provenance from them.
13320            let lower_new_origin_row =
13321                checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
13322            let lower_new_origin_col =
13323                checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
13324            let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
13325            let lower_new_result_region = Region::from_domain(&lower_new_domain);
13326            let lower_new_read_summary = match lower_read_summary.as_ref() {
13327                Some(summary) => Some(shifted_read_summary(
13328                    summary,
13329                    lower_new_result_region,
13330                    op,
13331                    row_delta,
13332                    col_delta,
13333                    origin_row_delta,
13334                    origin_col_delta,
13335                )?),
13336                None => None,
13337            };
13338
13339            // Overlay punch-outs sourced from this span: entries entirely
13340            // before the boundary stay with the (id-retaining) upper half;
13341            // entries at/after the boundary transfer to the new lower span.
13342            // A straddling entry means the split is not clean.
13343            let mut lower_overlay_refs = Vec::new();
13344            for overlay_ref in authority
13345                .plane
13346                .formula_overlay
13347                .refs_for_source_span(span_ref)
13348            {
13349                let entry = authority.plane.formula_overlay.get(overlay_ref)?;
13350                match op.classify_region(Region::from_domain(&entry.domain)) {
13351                    AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
13352                    AxisShiftCase::EntirelyAboveShift { .. } => {
13353                        lower_overlay_refs.push(overlay_ref);
13354                    }
13355                    AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
13356                        return None;
13357                    }
13358                }
13359            }
13360
13361            let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
13362                !binding_set.value_ref_slots.is_empty()
13363                    && (origin_row_delta != 0 || origin_col_delta != 0)
13364            });
13365
13366            Some(SplitPlan {
13367                span_ref,
13368                sheet_id: span.sheet_id,
13369                template_id: span.template_id,
13370                binding_set_id: span.binding_set_id,
13371                is_constant_result: span.is_constant_result,
13372                upper_domain,
13373                upper_read_summary,
13374                lower_new_origin_row,
13375                lower_new_origin_col,
13376                lower_new_domain,
13377                lower_new_read_summary,
13378                lower_force_binding_residual_axes,
13379                lower_overlay_refs,
13380                upper_dirty_region: None,
13381                lower_dirty_region: None,
13382            })
13383        }
13384
13385        /// Carry a 1-based template origin coordinate through a delete of
13386        /// `count` rows/columns starting at 0-based `start`. Returns None when
13387        /// the delete removes the origin coordinate itself: the template's
13388        /// relative reference deltas are anchored there, so the caller must
13389        /// demote rather than guess a new anchor.
13390        fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
13391            let origin0 = origin.checked_sub(1)?;
13392            let end = start.saturating_add(count);
13393            if origin0 < start {
13394                Some(origin)
13395            } else if origin0 >= end {
13396                Some(origin - count)
13397            } else {
13398                None
13399            }
13400        }
13401
13402        /// Carry the template origin through a delete instead of rebasing it
13403        /// to the compacted domain's start. Rebasing is only correct when the
13404        /// origin coincides with the domain start; split/shifted spans keep
13405        /// their original origin (Shift{origin_delta: 0} convention), and a
13406        /// head-overlapping delete moves the domain start away from the
13407        /// origin, so rebasing silently re-anchors every relative read.
13408        fn origin_through_delete(
13409            origin_row: u32,
13410            origin_col: u32,
13411            op: StructuralOp,
13412        ) -> Option<(u32, u32)> {
13413            match op {
13414                StructuralOp::DeleteRows { start, count, .. } => Some((
13415                    origin_coord_through_delete(origin_row, start, count)?,
13416                    origin_col,
13417                )),
13418                StructuralOp::DeleteColumns { start, count, .. } => Some((
13419                    origin_row,
13420                    origin_coord_through_delete(origin_col, start, count)?,
13421                )),
13422                StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
13423            }
13424        }
13425
13426        let mut shift_plans = Vec::new();
13427        let mut split_plans = Vec::new();
13428        let mut remove_refs = Vec::new();
13429        let mut demote_refs = Vec::new();
13430        for span_ref in span_refs {
13431            let authority = self.graph.formula_authority();
13432            let Some(span) = authority.plane.spans.get(span_ref) else {
13433                continue;
13434            };
13435            let read_summary = span
13436                .read_summary_id
13437                .and_then(|id| authority.plane.span_read_summaries.get(id));
13438            let Some(op) = op else {
13439                // Non-structural demote path (per-cell write into span, or
13440                // remove_sheet's whole-sheet sweep). Only demote spans whose
13441                // result or read region intersects affected_region; leave
13442                // disjoint spans untouched.
13443                let result_region_affected =
13444                    Self::span_result_region_intersects_affected(span, &affected_region);
13445                let read_region_affected = Self::span_any_read_region_intersects_affected(
13446                    &authority.plane,
13447                    span,
13448                    &affected_region,
13449                );
13450                if result_region_affected || read_region_affected {
13451                    demote_refs.push(span_ref);
13452                }
13453                continue;
13454            };
13455            match classify_span_for_op(span, read_summary, op) {
13456                SpanShiftPlan::NoOp => {}
13457                SpanShiftPlan::Remove => {
13458                    remove_refs.push(span_ref);
13459                }
13460                SpanShiftPlan::Demote {
13461                    reason:
13462                        crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
13463                } => {
13464                    let binding_compaction_safe = span
13465                        .binding_set_id
13466                        .and_then(|id| authority.plane.binding_sets.get(id))
13467                        .is_none_or(|binding_set| binding_set.is_single_literal_binding());
13468                    // Compaction projects read regions through the delete but
13469                    // keeps the template AST: an absolute read whose target
13470                    // the delete displaces would keep its stale coordinate
13471                    // (issue #168), so demote to the per-cell path instead.
13472                    let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
13473                        !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
13474                            summary, op,
13475                        )
13476                    });
13477                    if binding_compaction_safe
13478                        && absolute_reads_compaction_safe
13479                        && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
13480                    {
13481                        let new_result_region = Region::from_domain(&new_domain);
13482                        let new_read_summary = if let Some(summary) = read_summary {
13483                            compact_read_summary_through_delete(summary, new_result_region, op)
13484                        } else {
13485                            None
13486                        };
13487                        let Some(template) = authority.plane.templates.get(span.template_id) else {
13488                            return Err(ExcelError::new(ExcelErrorKind::Ref)
13489                                .with_message(
13490                                    "FormulaPlane delete compaction found a span with a missing template",
13491                                )
13492                                .into());
13493                        };
13494                        let carried_origin =
13495                            origin_through_delete(template.origin_row, template.origin_col, op);
13496                        if (read_summary.is_none() || new_read_summary.is_some())
13497                            && let Some((new_origin_row, new_origin_col)) = carried_origin
13498                        {
13499                            let force_binding_residual_axes = span
13500                                .binding_set_id
13501                                .and_then(|id| authority.plane.binding_sets.get(id))
13502                                .is_some_and(|binding_set| {
13503                                    !binding_set.value_ref_slots.is_empty()
13504                                        && (new_origin_row != template.origin_row
13505                                            || new_origin_col != template.origin_col)
13506                                });
13507                            let dirty_region =
13508                                Self::structural_dirty_region_for_domain(&new_domain, op);
13509                            shift_plans.push(ShiftPlan {
13510                                span_ref,
13511                                template_id: span.template_id,
13512                                new_origin_row,
13513                                new_origin_col,
13514                                new_domain,
13515                                new_read_summary,
13516                                binding_set_id: span.binding_set_id,
13517                                force_binding_residual_axes,
13518                                rewrite: None,
13519                                dirty_region,
13520                            });
13521                        } else {
13522                            demote_refs.push(span_ref);
13523                        }
13524                    } else {
13525                        demote_refs.push(span_ref);
13526                    }
13527                }
13528                SpanShiftPlan::Demote { .. } => {
13529                    demote_refs.push(span_ref);
13530                }
13531                SpanShiftPlan::Split => {
13532                    match plan_span_split(authority, span, span_ref, read_summary, op) {
13533                        Some(mut plan) => {
13534                            plan.upper_dirty_region =
13535                                Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
13536                            plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
13537                                &plan.lower_new_domain,
13538                                op,
13539                            );
13540                            split_plans.push(plan);
13541                        }
13542                        // Not provably clean: demote the whole span (the
13543                        // pre-split conservative path).
13544                        None => demote_refs.push(span_ref),
13545                    }
13546                }
13547                SpanShiftPlan::Shift {
13548                    row_delta,
13549                    col_delta,
13550                    origin_row_delta,
13551                    origin_col_delta,
13552                    rewrite_absolute_reads,
13553                } => {
13554                    let Some(template) = authority.plane.templates.get(span.template_id) else {
13555                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13556                            .with_message("FormulaPlane shift found a span with a missing template")
13557                            .into());
13558                    };
13559                    let Some(new_origin_row) =
13560                        checked_shift_u32(template.origin_row, origin_row_delta)
13561                    else {
13562                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13563                            .with_message("FormulaPlane shift overflowed template origin row")
13564                            .into());
13565                    };
13566                    let Some(new_origin_col) =
13567                        checked_shift_u32(template.origin_col, origin_col_delta)
13568                    else {
13569                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13570                            .with_message("FormulaPlane shift overflowed template origin column")
13571                            .into());
13572                    };
13573                    let Some(new_domain) =
13574                        span.domain.project_through_axis_shift(row_delta, col_delta)
13575                    else {
13576                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13577                            .with_message("FormulaPlane shift overflowed span domain")
13578                            .into());
13579                    };
13580                    let new_result_region = Region::from_domain(&new_domain);
13581                    let force_binding_residual_axes = span
13582                        .binding_set_id
13583                        .and_then(|id| authority.plane.binding_sets.get(id))
13584                        .is_some_and(|binding_set| {
13585                            !binding_set.value_ref_slots.is_empty()
13586                                && (origin_row_delta != 0 || origin_col_delta != 0)
13587                        });
13588                    let rewrite_and_summary = if rewrite_absolute_reads {
13589                        // Reify the (immutable, possibly shared) template
13590                        // AST and repoint its displaced references through
13591                        // the shared adjuster — the same relocation the
13592                        // per-cell path applies, so span-ON stays equal to
13593                        // span-OFF (issue #168). Canonical keys are
13594                        // re-derived from the rewritten AST at the new
13595                        // origin, so the fresh record cannot collide with
13596                        // the pre-rewrite template.
13597                        //
13598                        // FRAME GUARD: the adjuster relocates by AUTHORED
13599                        // coordinate, but a template's authoring frame may
13600                        // diverge from its domain (split lower halves keep
13601                        // the original anchor; pinned-origin shifts move the
13602                        // domain without the origin). The rewrite is only
13603                        // equivalent to per-cell adjustment when every
13604                        // relative read's authored coordinate classifies
13605                        // against the op boundary the same way its read
13606                        // region does — otherwise the rewrite shears
13607                        // relative reads by the op count. Demote instead;
13608                        // see rewrite_frame_is_sound for the full invariant.
13609                        let frame_sound = read_summary.is_some_and(|summary| {
13610                            crate::formula_plane::structural_shift::rewrite_frame_is_sound(
13611                                summary,
13612                                template.origin_row,
13613                                template.origin_col,
13614                                op,
13615                            )
13616                        });
13617                        if !frame_sound {
13618                            demote_refs.push(span_ref);
13619                            continue;
13620                        }
13621                        let Some(ast) = self
13622                            .graph
13623                            .data_store()
13624                            .retrieve_ast(template.ast_id, self.graph.sheet_reg())
13625                        else {
13626                            demote_refs.push(span_ref);
13627                            continue;
13628                        };
13629                        let shift_op = shift_operation_for(op);
13630                        let adjuster =
13631                            crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
13632                        let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
13633                            span.sheet_id,
13634                            self.graph.sheet_reg(),
13635                        );
13636                        let Some(adjusted_ast) =
13637                            adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
13638                        else {
13639                            // The classifier saw a displaced absolute read,
13640                            // so an unchanged AST is a contract violation;
13641                            // demote conservatively rather than shift with a
13642                            // stale template.
13643                            demote_refs.push(span_ref);
13644                            continue;
13645                        };
13646                        let canonical = crate::formula_plane::template_canonical::canonicalize_template(
13647                            &adjusted_ast,
13648                            new_origin_row,
13649                            new_origin_col,
13650                        );
13651                        if !canonical.labels.is_authority_supported() {
13652                            demote_refs.push(span_ref);
13653                            continue;
13654                        }
13655                        // The rewrite touches only references; the literal
13656                        // slot layout must survive unchanged or the binding
13657                        // set's per-placement literal bindings no longer
13658                        // line up. Demote defensively if it differs.
13659                        let literal_slots_compatible = span
13660                            .binding_set_id
13661                            .and_then(|id| authority.plane.binding_sets.get(id))
13662                            .is_none_or(|binding_set| {
13663                                binding_set.literal_slots.as_ref()
13664                                    == canonical.literal_slot_descriptors.as_ref()
13665                            });
13666                        if !literal_slots_compatible {
13667                            demote_refs.push(span_ref);
13668                            continue;
13669                        }
13670                        // Value-ref slot patterns are re-derived from the
13671                        // rewritten canonical template so memoized reads
13672                        // resolve the repointed coordinates (issue #168).
13673                        // Placement-relative memo KEYS are additionally
13674                        // protected by force_binding_residual_axes below
13675                        // (the rewrite always moves the origin, so the
13676                        // existing origin-delta condition fires).
13677                        let value_ref_slots = Arc::from(
13678                            crate::formula_plane::placement::value_ref_slot_descriptors(
13679                                &canonical.expr,
13680                            )
13681                            .into_boxed_slice(),
13682                        );
13683                        // The read summary is RE-DERIVED from the rewritten
13684                        // canonical template rather than structurally
13685                        // transformed: a structural transform would keep the
13686                        // old projection rules, whose absolute indices (and
13687                        // relative offsets, via the moved origin) no longer
13688                        // match the rewritten AST — incremental writes to
13689                        // the repointed targets would then silently fail to
13690                        // dirty the span. Re-derivation rebuilds rules in
13691                        // the (rewritten AST, new origin) frame, keeping
13692                        // `result_region == domain region` by construction.
13693                        let dependency_summary =
13694                            crate::formula_plane::dependency_summary::summarize_canonical_template(
13695                                &canonical,
13696                            );
13697                        let rewritten_result_region =
13698                            ResultRegion::scalar_cells(new_domain.clone());
13699                        let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
13700                            span.sheet_id,
13701                            &rewritten_result_region,
13702                            &dependency_summary,
13703                            self.graph.sheet_reg(),
13704                        ) else {
13705                            demote_refs.push(span_ref);
13706                            continue;
13707                        };
13708                        let formula_text = Some(Arc::<str>::from(
13709                            formualizer_parse::pretty::canonical_formula(&adjusted_ast),
13710                        ));
13711                        Some((
13712                            SpanTemplateRewrite {
13713                                exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
13714                                parameterized_canonical_key: Arc::<str>::from(
13715                                    canonical.parameterized_key.payload(),
13716                                ),
13717                                formula_text,
13718                                canonical_expr: canonical.expr,
13719                                value_ref_slots,
13720                                adjusted_ast,
13721                            },
13722                            rederived_summary,
13723                        ))
13724                    } else {
13725                        None
13726                    };
13727                    let (rewrite, new_read_summary) = match rewrite_and_summary {
13728                        Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
13729                        None => {
13730                            let new_read_summary = if let Some(summary) = read_summary {
13731                                Some(
13732                                    shifted_read_summary(
13733                                        summary,
13734                                        new_result_region,
13735                                        op,
13736                                        row_delta,
13737                                        col_delta,
13738                                        origin_row_delta,
13739                                        origin_col_delta,
13740                                    )
13741                                    .ok_or_else(|| {
13742                                        ExcelError::new(ExcelErrorKind::Ref).with_message(
13743                                            "FormulaPlane shift could not project read summary",
13744                                        )
13745                                    })?,
13746                                )
13747                            } else {
13748                                None
13749                            };
13750                            (None, new_read_summary)
13751                        }
13752                    };
13753                    let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
13754                    shift_plans.push(ShiftPlan {
13755                        span_ref,
13756                        template_id: span.template_id,
13757                        new_origin_row,
13758                        new_origin_col,
13759                        new_domain,
13760                        new_read_summary,
13761                        binding_set_id: span.binding_set_id,
13762                        force_binding_residual_axes,
13763                        rewrite,
13764                        dirty_region,
13765                    });
13766                }
13767            }
13768        }
13769        let span_geometry_changed =
13770            !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
13771        let mut span_dirty_deltas = Vec::new();
13772        if span_geometry_changed {
13773            // Rewritten template ASTs must be interned into the graph's AST
13774            // arena before the authority is borrowed mutably; the literal
13775            // slot map is keyed by the freshly interned arena node ids.
13776            let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
13777            for plan in shift_plans {
13778                let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
13779                    let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
13780                    let template_slot_map =
13781                        crate::formula_plane::placement::build_template_slot_map(
13782                            ast_id,
13783                            self.graph.data_store(),
13784                            &rewrite.canonical_expr,
13785                        );
13786                    (ast_id, template_slot_map)
13787                });
13788                prepared_shift_plans.push((plan, rewrite_prepared));
13789            }
13790            let authority = self.graph.formula_authority_mut();
13791            for span_ref in remove_refs {
13792                authority.plane.remove_overlays_for_source_span(span_ref);
13793                authority.plane.remove_span(span_ref);
13794            }
13795            for (plan, rewrite_prepared) in prepared_shift_plans {
13796                let dirty_region = plan.dirty_region;
13797                let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
13798                    let Some((ast_id, template_slot_map)) = rewrite_prepared else {
13799                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13800                            .with_message("FormulaPlane shift lost a rewritten template AST")
13801                            .into());
13802                    };
13803                    let Some(template_id) = authority.plane.intern_rewritten_template(
13804                        plan.template_id,
13805                        ast_id,
13806                        rewrite.exact_canonical_key,
13807                        rewrite.parameterized_canonical_key,
13808                        rewrite.formula_text,
13809                        plan.new_origin_row,
13810                        plan.new_origin_col,
13811                    ) else {
13812                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13813                            .with_message("FormulaPlane shift could not intern rewritten template")
13814                            .into());
13815                    };
13816                    (
13817                        template_id,
13818                        Some((template_slot_map, rewrite.value_ref_slots)),
13819                    )
13820                } else {
13821                    let Some(template_id) = authority.plane.intern_shifted_template_origin(
13822                        plan.template_id,
13823                        plan.new_origin_row,
13824                        plan.new_origin_col,
13825                    ) else {
13826                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13827                            .with_message("FormulaPlane shift could not clone template origin")
13828                            .into());
13829                    };
13830                    (template_id, None)
13831                };
13832                if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
13833                    && let Some(binding_set_id) = plan.binding_set_id
13834                {
13835                    authority.plane.set_binding_template_slots(
13836                        binding_set_id,
13837                        template_slot_map,
13838                        value_ref_slots,
13839                    );
13840                }
13841                if let Some(binding_set_id) = plan.binding_set_id {
13842                    let Some(template) = authority.plane.templates.get(template_id) else {
13843                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13844                            .with_message("FormulaPlane shift could not find shifted template")
13845                            .into());
13846                    };
13847                    let (ast_id, origin_row, origin_col) =
13848                        (template.ast_id, template.origin_row, template.origin_col);
13849                    authority.plane.set_binding_template_anchor(
13850                        binding_set_id,
13851                        ast_id,
13852                        origin_row,
13853                        origin_col,
13854                    );
13855                }
13856                let read_summary_id = plan
13857                    .new_read_summary
13858                    .map(|summary| authority.plane.insert_span_read_summary(summary));
13859                let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
13860                if !authority.plane.replace_span_geometry(
13861                    plan.span_ref,
13862                    template_id,
13863                    plan.new_domain,
13864                    result_region,
13865                    read_summary_id,
13866                ) {
13867                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13868                        .with_message("FormulaPlane shift could not update span geometry")
13869                        .into());
13870                }
13871                if let Some(region) = dirty_region {
13872                    span_dirty_deltas.push((plan.span_ref, region));
13873                }
13874                if plan.force_binding_residual_axes
13875                    && let Some(binding_set_id) = plan.binding_set_id
13876                {
13877                    // Value-ref memoization keys are placement-relative. When a
13878                    // structural op moves the formula origin while keeping some
13879                    // precedents fixed (e.g. insert a column before a formula
13880                    // family that reads column A), those keys no longer name
13881                    // the same producer cells. Keep correctness by forcing
13882                    // placement offsets into the key so memoization falls back
13883                    // to per-placement work rather than broadcasting stale
13884                    // representative values.
13885                    authority.plane.force_binding_residual_axes(binding_set_id);
13886                }
13887            }
13888            for plan in split_plans {
13889                let upper_dirty_region = plan.upper_dirty_region;
13890                let lower_dirty_region = plan.lower_dirty_region;
13891                let Some(split_relocation) = authority
13892                    .plane
13893                    .spans
13894                    .get(plan.span_ref)
13895                    .map(|span| span.ast_relocation)
13896                else {
13897                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13898                        .with_message("FormulaPlane split lost its anchor AST state")
13899                        .into());
13900                };
13901                // Lower half: a fresh span at the POST-insert coordinates,
13902                // mirroring the Shift arm's template/binding handling.
13903                let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
13904                    plan.template_id,
13905                    plan.lower_new_origin_row,
13906                    plan.lower_new_origin_col,
13907                ) else {
13908                    return Err(ExcelError::new(ExcelErrorKind::Ref)
13909                        .with_message("FormulaPlane split could not clone template origin")
13910                        .into());
13911                };
13912                let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
13913                    let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
13914                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13915                            .with_message(
13916                                "FormulaPlane split found a span with a missing binding set",
13917                            )
13918                            .into());
13919                    };
13920                    // Safe to clone wholesale: `plan_span_split` only accepts
13921                    // single-literal-binding sets, whose per-placement lookups
13922                    // are ordinal-independent.
13923                    let clone = source.clone();
13924                    let new_binding_set_id = authority.plane.insert_binding_set(clone);
13925                    let Some(template) = authority.plane.templates.get(lower_template_id) else {
13926                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13927                            .with_message("FormulaPlane split could not find shifted template")
13928                            .into());
13929                    };
13930                    let (ast_id, origin_row, origin_col) =
13931                        (template.ast_id, template.origin_row, template.origin_col);
13932                    authority.plane.set_binding_template_anchor(
13933                        new_binding_set_id,
13934                        ast_id,
13935                        origin_row,
13936                        origin_col,
13937                    );
13938                    Some(new_binding_set_id)
13939                } else {
13940                    None
13941                };
13942                let lower_read_summary_id = plan
13943                    .lower_new_read_summary
13944                    .map(|summary| authority.plane.insert_span_read_summary(summary));
13945                let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
13946                let lower_ref = authority.plane.insert_span_with_ast_relocation(
13947                    crate::formula_plane::runtime::NewFormulaSpan {
13948                        sheet_id: plan.sheet_id,
13949                        template_id: lower_template_id,
13950                        domain: plan.lower_new_domain,
13951                        result_region: lower_result_region,
13952                        intrinsic_mask_id: None,
13953                        read_summary_id: lower_read_summary_id,
13954                        binding_set_id: lower_binding_set_id,
13955                        is_constant_result: plan.is_constant_result,
13956                    },
13957                    crate::formula_plane::runtime::SpanAstRelocation {
13958                        ast_id: split_relocation.ast_id,
13959                        anchor_row: plan.lower_new_origin_row,
13960                        anchor_col: plan.lower_new_origin_col,
13961                    },
13962                );
13963                if let Some(region) = lower_dirty_region {
13964                    span_dirty_deltas.push((lower_ref, region));
13965                }
13966                if let Some(binding_set_id) = lower_binding_set_id {
13967                    authority
13968                        .plane
13969                        .set_binding_span_ref(binding_set_id, lower_ref);
13970                    if plan.lower_force_binding_residual_axes {
13971                        // Same contract as the Shift arm: a moved origin with
13972                        // stationary value-ref precedents invalidates
13973                        // placement-relative memoization keys.
13974                        authority.plane.force_binding_residual_axes(binding_set_id);
13975                    }
13976                }
13977                // The lower half inherits the parent's defined-name
13978                // invalidation registrations; the upper half keeps them by
13979                // retaining the span id.
13980                let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
13981                if !name_keys.is_empty() {
13982                    authority
13983                        .plane
13984                        .register_span_name_dependents(lower_ref, &name_keys);
13985                }
13986                for overlay_ref in plan.lower_overlay_refs {
13987                    if !authority
13988                        .plane
13989                        .set_overlay_source_span(overlay_ref, Some(lower_ref))
13990                    {
13991                        return Err(ExcelError::new(ExcelErrorKind::Ref)
13992                            .with_message(
13993                                "FormulaPlane split could not re-point overlay provenance",
13994                            )
13995                            .into());
13996                    }
13997                }
13998                // Upper half: keep the span id, truncate the domain, and swap
13999                // in a summary re-derived for the truncated result region.
14000                let upper_read_summary_id = plan
14001                    .upper_read_summary
14002                    .map(|summary| authority.plane.insert_span_read_summary(summary));
14003                let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
14004                if !authority.plane.replace_span_geometry_with_ast_relocation(
14005                    plan.span_ref,
14006                    plan.template_id,
14007                    split_relocation,
14008                    plan.upper_domain,
14009                    upper_result_region,
14010                    upper_read_summary_id,
14011                ) {
14012                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14013                        .with_message("FormulaPlane split could not update span geometry")
14014                        .into());
14015                }
14016                if let Some(region) = upper_dirty_region {
14017                    span_dirty_deltas.push((plan.span_ref, region));
14018                }
14019            }
14020            authority.rebuild_indexes();
14021        }
14022        for (span_ref, region) in span_dirty_deltas {
14023            self.graph.mark_formula_span_region_dirty(span_ref, region);
14024        }
14025
14026        let mut span_plans = Vec::new();
14027        for span_ref in demote_refs {
14028            let authority = self.graph.formula_authority();
14029            let Some(span) = authority.plane.spans.get(span_ref) else {
14030                continue;
14031            };
14032            let relocation = span.ast_relocation;
14033            let ast = self
14034                .graph
14035                .data_store()
14036                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14037                .ok_or_else(|| {
14038                    ExcelError::new(ExcelErrorKind::Ref).with_message(
14039                        "FormulaPlane demotion could not retrieve the span anchor AST",
14040                    )
14041                })?;
14042            let placements = span
14043                .domain
14044                .iter()
14045                .map(|placement| (placement.row + 1, placement.col + 1))
14046                .collect();
14047            span_plans.push(SpanPlan {
14048                span_ref,
14049                sheet_id: span.sheet_id,
14050                ast,
14051                origin_row: relocation.anchor_row,
14052                origin_col: relocation.anchor_col,
14053                binding_set_id: span.binding_set_id,
14054                placements,
14055            });
14056        }
14057        if span_plans.is_empty() {
14058            return Ok(());
14059        }
14060
14061        let mut relocated = Vec::new();
14062        let mut placement_cells = Vec::new();
14063        for plan in &span_plans {
14064            for &(row, col) in &plan.placements {
14065                let row_delta = i64::from(row) - i64::from(plan.origin_row);
14066                let col_delta = i64::from(col) - i64::from(plan.origin_col);
14067                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14068                    let authority = self.graph.formula_authority();
14069                    if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14070                        if binding_set.is_single_literal_binding() {
14071                            plan.ast.clone()
14072                        } else {
14073                            let placement = crate::formula_plane::runtime::PlacementCoord::new(
14074                                plan.sheet_id,
14075                                row.saturating_sub(1),
14076                                col.saturating_sub(1),
14077                            );
14078                            let binding =
14079                                authority.plane.spans.get(plan.span_ref).and_then(|span| {
14080                                    binding_set
14081                                        .literal_bindings_for_placement(&span.domain, placement)
14082                                });
14083                            if let Some(binding) = binding {
14084                                substitute_literal_slots_for_template_placement(
14085                                    &plan.ast,
14086                                    binding.as_ref(),
14087                                )
14088                            } else {
14089                                plan.ast.clone()
14090                            }
14091                        }
14092                    } else {
14093                        plan.ast.clone()
14094                    }
14095                } else {
14096                    plan.ast.clone()
14097                };
14098                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14099                relocated.push((plan.sheet_id, row, col, ast));
14100                placement_cells.push((plan.sheet_id, row, col));
14101            }
14102        }
14103        let planned_by_sheet = {
14104            let mut pipeline = self.ingest_pipeline();
14105            let mut planned_by_sheet: BTreeMap<
14106                SheetId,
14107                Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14108            > = BTreeMap::new();
14109            for (formula_sheet_id, row, col, ast) in relocated {
14110                let placement =
14111                    CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14112                let ingested =
14113                    pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14114                planned_by_sheet.entry(formula_sheet_id).or_default().push((
14115                    row,
14116                    col,
14117                    ingested.ast_id,
14118                    ingested.dep_plan,
14119                ));
14120            }
14121            planned_by_sheet
14122        };
14123        {
14124            let authority = self.graph.formula_authority_mut();
14125            for plan in &span_plans {
14126                authority
14127                    .plane
14128                    .remove_overlays_for_source_span(plan.span_ref);
14129                authority.plane.remove_span(plan.span_ref);
14130            }
14131            authority.rebuild_indexes();
14132        }
14133        if clear_computed_overlays {
14134            // Only clear placement cells whose coordinate intersects the affected
14135            // structural region. The structural-op contract preserves cells
14136            // BEFORE the structural boundary; the legacy `clear_computed_overlay_after_*`
14137            // call honors that. Demoting a span whose footprint straddles the
14138            // boundary still must not clear cells before the boundary, even
14139            // though the span as a whole is demoted.
14140            self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14141        }
14142        for (formula_sheet_id, planned) in planned_by_sheet {
14143            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14144            self.graph
14145                .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14146        }
14147        if !clear_computed_overlays {
14148            for (formula_sheet_id, row, col) in &placement_cells {
14149                let row0 = row.saturating_sub(1);
14150                let col0 = col.saturating_sub(1);
14151                if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14152                    continue;
14153                }
14154                let cell =
14155                    CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14156                if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14157                    self.graph.set_dirty(vertex_id, false);
14158                }
14159            }
14160        }
14161        Ok(())
14162    }
14163
14164    /// Prepare an exact-ref, additions-only refinement of FormulaPlane spans
14165    /// into legacy graph formulas. All fallible relocation, dependency analysis,
14166    /// identifier reservation, and conflict checks happen before commit.
14167    pub(crate) fn prepare_formula_span_demotion(
14168        &mut self,
14169        span_refs: &[FormulaSpanRef],
14170    ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14171        #[cfg(test)]
14172        let fault = self
14173            .formula_span_demotion_fault_for_test
14174            .take()
14175            .unwrap_or_default();
14176        #[cfg(not(test))]
14177        let fault = FormulaSpanDemotionFault::None;
14178
14179        if fault == FormulaSpanDemotionFault::AstPreparation {
14180            return Err(FormulaSpanDemotionError::Injected(fault));
14181        }
14182
14183        fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14184            fn visit(
14185                ast: &ASTNode,
14186                binding: &[LiteralValue],
14187                next: &mut usize,
14188                in_array: bool,
14189            ) -> ASTNode {
14190                let node_type = match &ast.node_type {
14191                    ASTNodeType::Literal(_) if !in_array => {
14192                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14193                        *next = next.saturating_add(1);
14194                        ASTNodeType::Literal(value)
14195                    }
14196                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14197                    ASTNodeType::Omitted => ASTNodeType::Omitted,
14198                    ASTNodeType::Reference {
14199                        original,
14200                        reference,
14201                    } => ASTNodeType::Reference {
14202                        original: original.clone(),
14203                        reference: reference.clone(),
14204                    },
14205                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14206                        op: op.clone(),
14207                        expr: Box::new(visit(expr, binding, next, in_array)),
14208                    },
14209                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14210                        op: op.clone(),
14211                        left: Box::new(visit(left, binding, next, in_array)),
14212                        right: Box::new(visit(right, binding, next, in_array)),
14213                    },
14214                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
14215                        name: name.clone(),
14216                        args: args
14217                            .iter()
14218                            .map(|arg| visit(arg, binding, next, in_array))
14219                            .collect(),
14220                    },
14221                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14222                        callee: Box::new(visit(callee, binding, next, in_array)),
14223                        args: args
14224                            .iter()
14225                            .map(|arg| visit(arg, binding, next, in_array))
14226                            .collect(),
14227                    },
14228                    ASTNodeType::Array(rows) => ASTNodeType::Array(
14229                        rows.iter()
14230                            .map(|row| {
14231                                row.iter()
14232                                    .map(|cell| visit(cell, binding, next, true))
14233                                    .collect()
14234                            })
14235                            .collect(),
14236                    ),
14237                };
14238                ASTNode::new(node_type, ast.source_token.clone())
14239            }
14240            let mut next = 0;
14241            visit(ast, binding, &mut next, false)
14242        }
14243
14244        struct SpanMaterialization {
14245            span_ref: FormulaSpanRef,
14246            sheet_id: SheetId,
14247            ast: ASTNode,
14248            origin_row: u32,
14249            origin_col: u32,
14250            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
14251            domain: PlacementDomain,
14252        }
14253
14254        let request_id = self
14255            .active_evaluation_resource_request
14256            .as_ref()
14257            .map(|request| request.request_id);
14258        let resource_error =
14259            |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
14260                FormulaSpanDemotionError::Resource(
14261                    crate::engine::ResourceLedgerError::Exhausted(
14262                        formualizer_common::ResourceExhaustionDetail {
14263                            reason,
14264                            limit,
14265                            observed,
14266                            request_id,
14267                        },
14268                    )
14269                    .into_excel_error(),
14270                )
14271            };
14272        let authority = self.graph.formula_authority();
14273        let expected_plane_epoch = authority.plane.epoch().0;
14274        let expected_indexes_epoch = authority.indexes_epoch();
14275        let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
14276        let mut unique = Vec::new();
14277        unique.try_reserve_exact(span_refs.len()).map_err(|_| {
14278            resource_error(
14279                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14280                u64::MAX,
14281                span_refs.len() as u64,
14282            )
14283        })?;
14284        let mut plans = Vec::new();
14285        plans.try_reserve_exact(span_refs.len()).map_err(|_| {
14286            resource_error(
14287                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14288                u64::MAX,
14289                span_refs.len() as u64,
14290            )
14291        })?;
14292        let mut placement_count = 0usize;
14293        let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
14294        for &span_ref in span_refs {
14295            if unique.contains(&span_ref) {
14296                continue;
14297            }
14298            unique.push(span_ref);
14299            let span = authority
14300                .plane
14301                .spans
14302                .get(span_ref)
14303                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14304            let relocation = span.ast_relocation;
14305            let ast = self
14306                .graph
14307                .data_store()
14308                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14309                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14310            let count = usize::try_from(span.domain.cell_count())
14311                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
14312            placement_count = placement_count
14313                .checked_add(count)
14314                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14315            if u64::try_from(placement_count)
14316                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
14317                > self.workbook_load_limits.max_formula_plane_fallback_cells
14318            {
14319                return Err(resource_error(
14320                    formualizer_common::ResourceExhaustionReason::MaterializationCells,
14321                    self.workbook_load_limits.max_formula_plane_fallback_cells,
14322                    placement_count as u64,
14323                ));
14324            }
14325            let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
14326            *sheet_count = sheet_count
14327                .checked_add(span.domain.cell_count())
14328                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
14329            if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
14330                return Err(resource_error(
14331                    formualizer_common::ResourceExhaustionReason::Admission,
14332                    self.workbook_load_limits.max_sheet_logical_cells,
14333                    *sheet_count,
14334                ));
14335            }
14336            let domain_in_bounds = match &span.domain {
14337                PlacementDomain::RowRun { row_end, col, .. } => {
14338                    *row_end < self.workbook_load_limits.max_sheet_rows
14339                        && *col < self.workbook_load_limits.max_sheet_cols
14340                }
14341                PlacementDomain::ColRun { row, col_end, .. } => {
14342                    *row < self.workbook_load_limits.max_sheet_rows
14343                        && *col_end < self.workbook_load_limits.max_sheet_cols
14344                }
14345                PlacementDomain::Rect {
14346                    row_end, col_end, ..
14347                } => {
14348                    *row_end < self.workbook_load_limits.max_sheet_rows
14349                        && *col_end < self.workbook_load_limits.max_sheet_cols
14350                }
14351            };
14352            if !domain_in_bounds {
14353                return Err(resource_error(
14354                    formualizer_common::ResourceExhaustionReason::Admission,
14355                    u64::from(self.workbook_load_limits.max_sheet_rows)
14356                        .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
14357                    span.domain.cell_count(),
14358                ));
14359            }
14360            plans.push(SpanMaterialization {
14361                span_ref,
14362                sheet_id: span.sheet_id,
14363                ast,
14364                origin_row: relocation.anchor_row,
14365                origin_col: relocation.anchor_col,
14366                binding_set_id: span.binding_set_id,
14367                domain: span.domain.clone(),
14368            });
14369        }
14370        let mut relocated = Vec::new();
14371        relocated.try_reserve_exact(placement_count).map_err(|_| {
14372            resource_error(
14373                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14374                u64::MAX,
14375                placement_count as u64,
14376            )
14377        })?;
14378        for plan in &plans {
14379            for coord in plan.domain.iter() {
14380                let row = coord.row.saturating_add(1);
14381                let col = coord.col.saturating_add(1);
14382                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14383                    let authority = self.graph.formula_authority();
14384                    let binding_set = authority
14385                        .plane
14386                        .binding_sets
14387                        .get(binding_set_id)
14388                        .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14389                    if binding_set.is_single_literal_binding() {
14390                        plan.ast.clone()
14391                    } else {
14392                        let placement = PlacementCoord::new(
14393                            plan.sheet_id,
14394                            row.saturating_sub(1),
14395                            col.saturating_sub(1),
14396                        );
14397                        let span = authority
14398                            .plane
14399                            .spans
14400                            .get(plan.span_ref)
14401                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14402                        let binding = binding_set
14403                            .literal_bindings_for_placement(&span.domain, placement)
14404                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
14405                        substitute_literal_slots(&plan.ast, binding.as_ref())
14406                    }
14407                } else {
14408                    plan.ast.clone()
14409                };
14410                let row_delta = i64::from(row) - i64::from(plan.origin_row);
14411                let col_delta = i64::from(col) - i64::from(plan.origin_col);
14412                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
14413                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
14414                relocated.push((plan.sheet_id, row, col, ast));
14415            }
14416        }
14417
14418        let mut analyzed = Vec::new();
14419        analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
14420            resource_error(
14421                formualizer_common::ResourceExhaustionReason::ScratchMemory,
14422                u64::MAX,
14423                relocated.len() as u64,
14424            )
14425        })?;
14426        {
14427            let mut pipeline = self.ingest_pipeline();
14428            for (sheet_id, row, col, ast) in relocated {
14429                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
14430                let ingested = pipeline
14431                    .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
14432                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
14433                analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
14434            }
14435        }
14436
14437        if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
14438            return Err(FormulaSpanDemotionError::Injected(fault));
14439        }
14440        let legacy_graph = self
14441            .graph
14442            .prepare_legacy_graph_plan_multi_sheet(analyzed)
14443            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14444        self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
14445            .map_err(FormulaSpanDemotionError::Resource)?;
14446
14447        Ok(PreparedFormulaSpanDemotion {
14448            span_refs: unique,
14449            expected_plane_epoch,
14450            expected_indexes_epoch,
14451            expected_indexed_plane_epoch,
14452            placement_count,
14453            legacy_graph,
14454            fault,
14455        })
14456    }
14457
14458    pub(crate) fn validate_prepared_formula_span_demotion(
14459        &self,
14460        prepared: &PreparedFormulaSpanDemotion,
14461    ) -> Result<(), FormulaSpanDemotionError> {
14462        let authority = self.graph.formula_authority();
14463        if authority.plane.epoch().0 != prepared.expected_plane_epoch
14464            || authority.indexes_epoch() != prepared.expected_indexes_epoch
14465            || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14466            || prepared
14467                .span_refs
14468                .iter()
14469                .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14470        {
14471            return Err(FormulaSpanDemotionError::StaleAuthority);
14472        }
14473        self.graph
14474            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14475            .map_err(FormulaSpanDemotionError::LegacyGraph)
14476    }
14477
14478    /// Validate every recoverable assumption and fault before the first
14479    /// mutation, then compose the existing prevalidated graph append with exact
14480    /// authority removal. Allocator OOM/panic remains process-fatal, matching
14481    /// the E2/E3 prepared-transaction contract; there is no recoverable branch
14482    /// after graph application begins.
14483    pub(crate) fn commit_prepared_formula_span_demotion(
14484        &mut self,
14485        prepared: PreparedFormulaSpanDemotion,
14486    ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
14487        self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
14488            .map_err(FormulaSpanDemotionError::Resource)?;
14489        if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
14490            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14491        }
14492        self.graph
14493            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
14494            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
14495        if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
14496            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14497        }
14498        {
14499            let authority = self.graph.formula_authority();
14500            if authority.plane.epoch().0 != prepared.expected_plane_epoch
14501                || authority.indexes_epoch() != prepared.expected_indexes_epoch
14502                || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
14503                || prepared
14504                    .span_refs
14505                    .iter()
14506                    .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
14507            {
14508                return Err(FormulaSpanDemotionError::StaleAuthority);
14509            }
14510        }
14511        if matches!(
14512            prepared.fault,
14513            FormulaSpanDemotionFault::AllocationReservation
14514                | FormulaSpanDemotionFault::BeforeFirstMutation
14515        ) {
14516            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
14517        }
14518
14519        let PreparedFormulaSpanDemotion {
14520            span_refs,
14521            placement_count,
14522            legacy_graph,
14523            ..
14524        } = prepared;
14525        let _ = self
14526            .graph
14527            .apply_prevalidated_legacy_graph_plan(legacy_graph);
14528        let authority = self.graph.formula_authority_mut();
14529        for span_ref in &span_refs {
14530            authority.plane.remove_overlays_for_source_span(*span_ref);
14531            let _ = authority.plane.remove_span(*span_ref);
14532        }
14533        let _ = authority.rebuild_indexes();
14534        self.mark_topology_edited();
14535        Ok(FormulaSpanDemotionReport {
14536            spans_demoted: span_refs.len(),
14537            placements_materialized: placement_count,
14538        })
14539    }
14540
14541    #[cfg(test)]
14542    pub(crate) fn set_formula_span_demotion_fault_for_test(
14543        &mut self,
14544        fault: FormulaSpanDemotionFault,
14545    ) {
14546        self.formula_span_demotion_fault_for_test = Some(fault);
14547    }
14548
14549    #[cfg(test)]
14550    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
14551        self.force_non_cycle_schedule_fallback_for_test = true;
14552    }
14553
14554    #[cfg(test)]
14555    pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
14556        self.mixed_topology_index_builds_for_test
14557    }
14558
14559    /// Collect the [`FormulaSpanRef`]s for span producers the mixed scheduler
14560    /// reported as cycle members (gotcha G8, refs #112). These spans must be
14561    /// demoted to legacy so the cycle members are resolved on the legacy SCC
14562    /// path; see [`Self::demote_cyclic_spans`].
14563    fn collect_cyclic_span_refs(
14564        &self,
14565        schedule: &MixedSchedule,
14566        span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
14567    ) -> Vec<FormulaSpanRef> {
14568        let mut refs = Vec::new();
14569        for fallback in &schedule.fallbacks {
14570            if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
14571                continue;
14572            }
14573            if let FormulaProducerId::Span(span_id) = fallback.producer
14574                && let Some(span_ref) = span_refs_by_id.get(&span_id)
14575                && !refs.contains(span_ref)
14576            {
14577                refs.push(*span_ref);
14578            }
14579        }
14580        refs
14581    }
14582
14583    /// Demote the given cyclic spans to legacy graph vertices so their member
14584    /// cells participate in the legacy Tarjan SCC pass (gotcha G8, refs #112).
14585    ///
14586    /// Uses one exact-ref prepared transaction for every cyclic span reported
14587    /// by this schedule. All recoverable validation and injected faults precede
14588    /// the first graph/authority mutation, so a multi-span or multi-sheet cycle
14589    /// cannot publish only an earlier subset.
14590    fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
14591        let materialization_started = crate::instant::FzInstant::now();
14592        let prepared = self
14593            .prepare_formula_span_demotion(span_refs)
14594            .map_err(|error| {
14595                if let FormulaSpanDemotionError::Resource(error) = error {
14596                    error
14597                } else {
14598                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14599                        "FormulaPlane cycle-member span demotion preparation failed: {error}"
14600                    ))
14601                }
14602            })?;
14603        let report = self
14604            .commit_prepared_formula_span_demotion(prepared)
14605            .map_err(|error| {
14606                ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
14607                    "FormulaPlane cycle-member span demotion commit failed: {error}"
14608                ))
14609            })?;
14610        self.observe_materialization(
14611            report.placements_materialized,
14612            true,
14613            materialization_started.elapsed(),
14614        );
14615        self.formula_plane_cycle_member_span_demotions = self
14616            .formula_plane_cycle_member_span_demotions
14617            .saturating_add(report.spans_demoted as u64);
14618        Self::record_shadow_fallback_reason(
14619            &mut self.formula_ingest_report_total,
14620            PlacementFallbackReason::CycleMember,
14621            report.spans_demoted as u64,
14622        );
14623        Ok(())
14624    }
14625
14626    /// Evaluate residual *legacy-only* cyclic SCCs before the FormulaPlane
14627    /// mixed schedule runs (gotcha G8, refs #112).
14628    ///
14629    /// After cyclic spans are demoted to legacy ([`Self::demote_cyclic_spans`]),
14630    /// every cycle member is a graph vertex, so the cycle is now visible to the
14631    /// legacy Tarjan pass and lives entirely among legacy producers. The mixed
14632    /// schedule treats any cycle as not authoritative-safe; rather than abandon
14633    /// the surviving spans by falling through to a pure-legacy `evaluate_all`,
14634    /// stamp/evaluate just the cyclic SCC units here (`handle_cycle_unit` honors
14635    /// `CycleDetection::Static` vs `Runtime`), clear their dirty flags, and let
14636    /// the mixed schedule proceed cycle-free over the surviving spans plus the
14637    /// acyclic legacy work.
14638    ///
14639    /// Returns the number of cyclic SCC units that stamped at least one cell.
14640    fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
14641        let dirty = self.graph.get_evaluation_vertices();
14642        if dirty.is_empty() {
14643            return Ok(0);
14644        }
14645        let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
14646        let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
14647        let mut cycle_errors = 0usize;
14648        let mut stamped_vertices: Vec<VertexId> = Vec::new();
14649        for &unit in &schedule.units {
14650            let ScheduleUnit::Cycle(i) = unit else {
14651                continue;
14652            };
14653            let members = schedule.unit_cycle(i);
14654            let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
14655            if stamped > 0 {
14656                cycle_errors += 1;
14657            }
14658            stamped_vertices.extend(members.iter().copied());
14659        }
14660        // Clear dirty only on the cyclic members so the subsequent mixed
14661        // schedule no longer sees them as dirty legacy producers (which is what
14662        // surfaced the cycle). Acyclic legacy work stays dirty and is scheduled
14663        // normally alongside the surviving spans.
14664        if !stamped_vertices.is_empty() {
14665            self.resource_checkpoint(0)?;
14666            self.graph.clear_dirty_flags(&stamped_vertices);
14667        }
14668        Ok(cycle_errors)
14669    }
14670
14671    fn materialize_deferred_sheet_before_structural_edit(
14672        &mut self,
14673        sheet: &str,
14674    ) -> Result<(), crate::engine::EditorError> {
14675        if self.staged_formulas.contains_key(sheet) {
14676            self.build_graph_for_sheets([sheet])?;
14677        }
14678        Ok(())
14679    }
14680
14681    /// Insert rows (1-based) and mirror into Arrow store when enabled
14682    pub fn insert_rows(
14683        &mut self,
14684        sheet: &str,
14685        before: u32,
14686        count: u32,
14687    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14688    {
14689        if count == 0 {
14690            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14691        }
14692        self.observe_function_semantic_epoch()
14693            .map_err(crate::engine::EditorError::Excel)?;
14694        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14695        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14696        let sheet_id = self.ensure_known_sheet_id(sheet)?;
14697        let before0 = before.saturating_sub(1);
14698        let affected_region = Self::structural_row_region(sheet_id, before0);
14699        let op = StructuralOp::InsertRows {
14700            sheet_id,
14701            before: before0,
14702            count,
14703        };
14704        self.demote_spans_for_structural_op(op, affected_region)?;
14705        let summary = {
14706            let mut editor = VertexEditor::new(&mut self.graph);
14707            editor.insert_rows(sheet_id, before0, count)?
14708        };
14709        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14710            let before0 = before0 as usize;
14711            asheet.insert_rows(before0, count as usize);
14712        }
14713        self.mark_moved_formula_vertices_dirty(&summary);
14714        self.clear_computed_overlay_after_row(sheet, before0 as usize);
14715        self.shift_row_visibility_insert(sheet_id, before0, count);
14716        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14717        self.mark_topology_edited();
14718        Ok(summary)
14719    }
14720
14721    /// Delete rows (1-based) and mirror into Arrow store when enabled
14722    pub fn delete_rows(
14723        &mut self,
14724        sheet: &str,
14725        start: u32,
14726        count: u32,
14727    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14728    {
14729        if count == 0 {
14730            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14731        }
14732        self.observe_function_semantic_epoch()
14733            .map_err(crate::engine::EditorError::Excel)?;
14734        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14735        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14736        let sheet_id = self.ensure_known_sheet_id(sheet)?;
14737        let start0 = start.saturating_sub(1);
14738        let affected_region = Self::structural_row_region(sheet_id, start0);
14739        let op = StructuralOp::DeleteRows {
14740            sheet_id,
14741            start: start0,
14742            count,
14743        };
14744        self.demote_spans_for_structural_op(op, affected_region)?;
14745        let summary = {
14746            let mut editor = VertexEditor::new(&mut self.graph);
14747            editor.delete_rows(sheet_id, start0, count)?
14748        };
14749        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14750            let start0 = start0 as usize;
14751            asheet.delete_rows(start0, count as usize);
14752        }
14753        self.mark_moved_formula_vertices_dirty(&summary);
14754        self.clear_computed_overlay_after_row(sheet, start0 as usize);
14755        self.shift_row_visibility_delete(sheet_id, start0, count);
14756        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14757        self.mark_topology_edited();
14758        Ok(summary)
14759    }
14760
14761    /// Insert columns (1-based) and mirror into Arrow store when enabled
14762    pub fn insert_columns(
14763        &mut self,
14764        sheet: &str,
14765        before: u32,
14766        count: u32,
14767    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14768    {
14769        if count == 0 {
14770            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14771        }
14772        self.observe_function_semantic_epoch()
14773            .map_err(crate::engine::EditorError::Excel)?;
14774        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14775        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14776        let sheet_id = self.graph.sheet_id(sheet).ok_or(
14777            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14778                name: sheet.to_string(),
14779                reason: "Unknown sheet".to_string(),
14780            },
14781        )?;
14782        let before0 = before.saturating_sub(1);
14783        let affected_region = Self::structural_col_region(sheet_id, before0);
14784        let op = StructuralOp::InsertColumns {
14785            sheet_id,
14786            before: before0,
14787            count,
14788        };
14789        self.demote_spans_for_structural_op(op, affected_region)?;
14790        let summary = {
14791            let mut editor = VertexEditor::new(&mut self.graph);
14792            editor.insert_columns(sheet_id, before0, count)?
14793        };
14794        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14795            let before0 = before0 as usize;
14796            asheet.insert_columns(before0, count as usize);
14797        }
14798        self.mark_moved_formula_vertices_dirty(&summary);
14799        self.clear_computed_overlay_after_col(sheet, before0 as usize);
14800        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14801        self.mark_topology_edited();
14802        Ok(summary)
14803    }
14804
14805    /// Delete columns (1-based) and mirror into Arrow store when enabled
14806    pub fn delete_columns(
14807        &mut self,
14808        sheet: &str,
14809        start: u32,
14810        count: u32,
14811    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
14812    {
14813        if count == 0 {
14814            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
14815        }
14816        self.observe_function_semantic_epoch()
14817            .map_err(crate::engine::EditorError::Excel)?;
14818        use crate::engine::graph::editor::vertex_editor::VertexEditor;
14819        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
14820        let sheet_id = self.graph.sheet_id(sheet).ok_or(
14821            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
14822                name: sheet.to_string(),
14823                reason: "Unknown sheet".to_string(),
14824            },
14825        )?;
14826        let start0 = start.saturating_sub(1);
14827        let affected_region = Self::structural_col_region(sheet_id, start0);
14828        let op = StructuralOp::DeleteColumns {
14829            sheet_id,
14830            start: start0,
14831            count,
14832        };
14833        self.demote_spans_for_structural_op(op, affected_region)?;
14834        let summary = {
14835            let mut editor = VertexEditor::new(&mut self.graph);
14836            editor.delete_columns(sheet_id, start0, count)?
14837        };
14838        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
14839            let start0 = start0 as usize;
14840            asheet.delete_columns(start0, count as usize);
14841        }
14842        self.mark_moved_formula_vertices_dirty(&summary);
14843        self.clear_computed_overlay_after_col(sheet, start0 as usize);
14844        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
14845        self.mark_topology_edited();
14846        Ok(summary)
14847    }
14848    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
14849    fn arrow_used_row_bounds(
14850        &self,
14851        sheet: &str,
14852        start_col: u32,
14853        end_col: u32,
14854    ) -> Option<(u32, u32)> {
14855        let a = self.sheet_store().sheet(sheet)?;
14856        if a.columns.is_empty() {
14857            return None;
14858        }
14859        let sc0 = start_col.saturating_sub(1) as usize;
14860        let ec0 = end_col.saturating_sub(1) as usize;
14861        let col_hi = a.columns.len().saturating_sub(1);
14862        if sc0 > col_hi {
14863            return None;
14864        }
14865        let ec0 = ec0.min(col_hi);
14866        // Pass-scoped cache with snapshot guard
14867        let snap = self.data_snapshot_id();
14868        let mut min_r0: Option<usize> = None;
14869        for ci in sc0..=ec0 {
14870            let sheet_id = self.graph.sheet_id(sheet)?;
14871            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
14872                g.as_ref()
14873                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14874            }) {
14875                let mv = mv as usize;
14876                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
14877                continue;
14878            }
14879            // Compute and store
14880            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14881            if let Ok(mut g) = self.row_bounds_cache.write() {
14882                g.get_or_insert_with(|| RowBoundsCache::new(snap))
14883                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
14884            }
14885            if let Some(m) = min_c {
14886                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
14887            }
14888        }
14889        min_r0?;
14890        let mut max_r0: Option<usize> = None;
14891        for ci in sc0..=ec0 {
14892            let sheet_id = self.graph.sheet_id(sheet)?;
14893            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
14894                g.as_ref()
14895                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
14896            }) {
14897                let mv = mv as usize;
14898                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
14899                continue;
14900            }
14901            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
14902            if let Ok(mut g) = self.row_bounds_cache.write() {
14903                g.get_or_insert_with(|| RowBoundsCache::new(snap))
14904                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
14905            }
14906            if let Some(m) = max_c {
14907                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
14908            }
14909        }
14910        match (min_r0, max_r0) {
14911            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
14912            _ => None,
14913        }
14914    }
14915
14916    fn scan_column_used_bounds(
14917        a: &crate::arrow_store::ArrowSheet,
14918        ci: usize,
14919    ) -> (Option<u32>, Option<u32>) {
14920        let col = &a.columns[ci];
14921
14922        // Min: scan dense chunks first, then sparse chunks in ascending index order.
14923        let mut min_r0: Option<u32> = None;
14924        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
14925            let tags = chunk.type_tag.values();
14926            for (off, &t) in tags.iter().enumerate() {
14927                let overlay_non_empty = chunk
14928                    .overlay
14929                    .get(off)
14930                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14931                    .unwrap_or(false)
14932                    || chunk
14933                        .computed_overlay
14934                        .get(off)
14935                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14936                        .unwrap_or(false);
14937                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14938                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14939                        break;
14940                    };
14941                    let row0 = chunk_start + off;
14942                    min_r0 = Some(row0 as u32);
14943                    break;
14944                }
14945            }
14946            if min_r0.is_some() {
14947                break;
14948            }
14949        }
14950        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
14951            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14952            sparse_idxs.sort_unstable();
14953            for chunk_idx in sparse_idxs {
14954                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14955                    continue;
14956                };
14957                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14958                    continue;
14959                };
14960                let tags = chunk.type_tag.values();
14961                for (off, &t) in tags.iter().enumerate() {
14962                    let overlay_non_empty = chunk
14963                        .overlay
14964                        .get(off)
14965                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14966                        .unwrap_or(false)
14967                        || chunk
14968                            .computed_overlay
14969                            .get(off)
14970                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
14971                            .unwrap_or(false);
14972                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
14973                        let row0 = chunk_start + off;
14974                        min_r0 = Some(row0 as u32);
14975                        break;
14976                    }
14977                }
14978                if min_r0.is_some() {
14979                    break;
14980                }
14981            }
14982        }
14983
14984        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
14985        let mut max_r0: Option<u32> = None;
14986        if !col.sparse_chunks.is_empty() {
14987            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
14988            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
14989            for chunk_idx in sparse_idxs {
14990                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
14991                    continue;
14992                };
14993                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
14994                    continue;
14995                };
14996                let tags = chunk.type_tag.values();
14997                for (rev_idx, &t) in tags.iter().enumerate().rev() {
14998                    let overlay_non_empty = chunk
14999                        .overlay
15000                        .get(rev_idx)
15001                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15002                        .unwrap_or(false)
15003                        || chunk
15004                            .computed_overlay
15005                            .get(rev_idx)
15006                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15007                            .unwrap_or(false);
15008                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15009                        let row0 = chunk_start + rev_idx;
15010                        max_r0 = Some(row0 as u32);
15011                        break;
15012                    }
15013                }
15014                if max_r0.is_some() {
15015                    break;
15016                }
15017            }
15018        }
15019        if max_r0.is_none() {
15020            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
15021                let tags = chunk.type_tag.values();
15022                for (rev_idx, &t) in tags.iter().enumerate().rev() {
15023                    let overlay_non_empty = chunk
15024                        .overlay
15025                        .get(rev_idx)
15026                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15027                        .unwrap_or(false)
15028                        || chunk
15029                            .computed_overlay
15030                            .get(rev_idx)
15031                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15032                            .unwrap_or(false);
15033                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15034                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15035                            break;
15036                        };
15037                        let row0 = chunk_start + rev_idx;
15038                        max_r0 = Some(row0 as u32);
15039                        break;
15040                    }
15041                }
15042                if max_r0.is_some() {
15043                    break;
15044                }
15045            }
15046        }
15047
15048        (min_r0, max_r0)
15049    }
15050
15051    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
15052    fn arrow_used_col_bounds(
15053        &self,
15054        sheet: &str,
15055        start_row: u32,
15056        end_row: u32,
15057    ) -> Option<(u32, u32)> {
15058        let a = self.sheet_store().sheet(sheet)?;
15059        if a.columns.is_empty() {
15060            return None;
15061        }
15062        let sr0 = start_row.saturating_sub(1) as usize;
15063        let er0 = end_row.saturating_sub(1) as usize;
15064        if sr0 > er0 {
15065            return None;
15066        }
15067        // Map start/end rows into chunk ranges
15068        // We will scan each column for any non-empty within [sr0..=er0]
15069        let mut min_c0: Option<usize> = None;
15070        let mut max_c0: Option<usize> = None;
15071        // Precompute chunk bounds for row range
15072        for (ci, col) in a.columns.iter().enumerate() {
15073            let mut any_in_range = false;
15074
15075            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15076                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15077                    return false;
15078                };
15079                let chunk_len = chunk.type_tag.len();
15080                if chunk_len == 0 {
15081                    return false;
15082                }
15083                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15084                // check intersection
15085                if sr0 > chunk_end || er0 < chunk_start {
15086                    return false;
15087                }
15088                let start_off = sr0.max(chunk_start) - chunk_start;
15089                let end_off = er0.min(chunk_end) - chunk_start;
15090                let tags = chunk.type_tag.values();
15091                for off in start_off..=end_off {
15092                    let overlay_non_empty = chunk
15093                        .overlay
15094                        .get(off)
15095                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15096                        .unwrap_or(false)
15097                        || chunk
15098                            .computed_overlay
15099                            .get(off)
15100                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15101                            .unwrap_or(false);
15102                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15103                        return true;
15104                    }
15105                }
15106                false
15107            };
15108
15109            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15110                if scan_chunk(chunk_idx, chunk) {
15111                    any_in_range = true;
15112                    break;
15113                }
15114            }
15115
15116            if !any_in_range && !col.sparse_chunks.is_empty() {
15117                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15118                    if scan_chunk(chunk_idx, chunk) {
15119                        any_in_range = true;
15120                        break;
15121                    }
15122                }
15123            }
15124
15125            if any_in_range {
15126                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
15127                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
15128            }
15129        }
15130        match (min_c0, max_c0) {
15131            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15132            _ => None,
15133        }
15134    }
15135
15136    fn formula_row_bounds_for_columns(
15137        &self,
15138        sheet: &str,
15139        start_col: u32,
15140        end_col: u32,
15141    ) -> Option<(u32, u32)> {
15142        let sheet_id = self.graph.sheet_id(sheet)?;
15143        let sc0 = start_col.saturating_sub(1);
15144        let ec0 = end_col.saturating_sub(1);
15145        let mut min_r0: Option<u32> = None;
15146        let mut max_r0: Option<u32> = None;
15147
15148        if let Some(index) = self.graph.sheet_index(sheet_id) {
15149            for vid in index.vertices_in_col_range(sc0, ec0) {
15150                if !matches!(
15151                    self.graph.get_vertex_kind(vid),
15152                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15153                ) {
15154                    continue;
15155                }
15156                let row0 = self.graph.vertex_coord(vid).row();
15157                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15158                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15159            }
15160        } else {
15161            for vid in self.graph.vertices_in_sheet(sheet_id) {
15162                if !matches!(
15163                    self.graph.get_vertex_kind(vid),
15164                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15165                ) {
15166                    continue;
15167                }
15168                let coord = self.graph.vertex_coord(vid);
15169                let col0 = coord.col();
15170                if col0 < sc0 || col0 > ec0 {
15171                    continue;
15172                }
15173                let row0 = coord.row();
15174                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
15175                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
15176            }
15177        }
15178
15179        match (min_r0, max_r0) {
15180            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15181            _ => None,
15182        }
15183    }
15184
15185    fn formula_col_bounds_for_rows(
15186        &self,
15187        sheet: &str,
15188        start_row: u32,
15189        end_row: u32,
15190    ) -> Option<(u32, u32)> {
15191        let sheet_id = self.graph.sheet_id(sheet)?;
15192        let sr0 = start_row.saturating_sub(1);
15193        let er0 = end_row.saturating_sub(1);
15194        let mut min_c0: Option<u32> = None;
15195        let mut max_c0: Option<u32> = None;
15196
15197        if let Some(index) = self.graph.sheet_index(sheet_id) {
15198            for vid in index.vertices_in_row_range(sr0, er0) {
15199                if !matches!(
15200                    self.graph.get_vertex_kind(vid),
15201                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15202                ) {
15203                    continue;
15204                }
15205                let col0 = self.graph.vertex_coord(vid).col();
15206                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15207                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15208            }
15209        } else {
15210            for vid in self.graph.vertices_in_sheet(sheet_id) {
15211                if !matches!(
15212                    self.graph.get_vertex_kind(vid),
15213                    VertexKind::FormulaScalar | VertexKind::FormulaArray
15214                ) {
15215                    continue;
15216                }
15217                let coord = self.graph.vertex_coord(vid);
15218                let row0 = coord.row();
15219                if row0 < sr0 || row0 > er0 {
15220                    continue;
15221                }
15222                let col0 = coord.col();
15223                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
15224                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
15225            }
15226        }
15227
15228        match (min_c0, max_c0) {
15229            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
15230            _ => None,
15231        }
15232    }
15233
15234    fn union_used_bounds(
15235        first: Option<(u32, u32)>,
15236        second: Option<(u32, u32)>,
15237    ) -> Option<(u32, u32)> {
15238        match (first, second) {
15239            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
15240            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
15241            (None, None) => None,
15242        }
15243    }
15244
15245    /// Mirror a single cell value into the Arrow overlay if enabled.
15246    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
15247    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
15248        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15249            return;
15250        }
15251        if self.arrow_sheets.sheet(sheet).is_none() {
15252            self.arrow_sheets
15253                .sheets
15254                .push(crate::arrow_store::ArrowSheet {
15255                    name: std::sync::Arc::<str>::from(sheet),
15256                    date_system: self.config.date_system,
15257                    columns: Vec::new(),
15258                    nrows: 0,
15259                    chunk_starts: Vec::new(),
15260                    chunk_rows: 32 * 1024,
15261                });
15262        }
15263
15264        let row0 = row.saturating_sub(1) as usize;
15265        let col0 = col.saturating_sub(1) as usize;
15266
15267        let asheet = self
15268            .arrow_sheets
15269            .sheet_mut(sheet)
15270            .expect("ArrowSheet must exist");
15271
15272        let cur_cols = asheet.columns.len();
15273        if col0 >= cur_cols {
15274            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15275        }
15276
15277        if row0 >= asheet.nrows as usize {
15278            if asheet.columns.is_empty() {
15279                asheet.insert_columns(0, 1);
15280            }
15281            asheet.ensure_row_capacity(row0 + 1);
15282        }
15283        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
15284            let ov =
15285                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
15286            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
15287                let _ = ch.overlay.set(in_off, ov);
15288                // A user edit must invalidate any computed (formula/spill) overlay entry at
15289                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
15290                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
15291                // the edited base value under the read cascade.
15292                ch.computed_overlay.remove(in_off)
15293            } else {
15294                return;
15295            };
15296            // Heuristic compaction: > len/50 or > 1024
15297            let abs_threshold = 1024usize;
15298            let frac_den = 50usize;
15299            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
15300            if freed > 0 {
15301                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
15302            }
15303            self.adjust_computed_overlay_bytes(computed_delta);
15304        }
15305    }
15306
15307    /// Remove a delta-overlay entry for a single cell (if present).
15308    ///
15309    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
15310    /// does not continue to mask computed overlay outputs.
15311    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
15312        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15313            return;
15314        }
15315        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15316            return;
15317        };
15318        let row0 = row.saturating_sub(1) as usize;
15319        let col0 = col.saturating_sub(1) as usize;
15320        if row0 >= asheet.nrows as usize {
15321            return;
15322        }
15323        if col0 >= asheet.columns.len() {
15324            return;
15325        }
15326        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15327            return;
15328        };
15329        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
15330            let _ = ch.overlay.remove(in_off);
15331        }
15332    }
15333
15334    fn clear_computed_overlay_col_row_range(
15335        &mut self,
15336        sheet: &str,
15337        col0: usize,
15338        start_row0: usize,
15339        end_row0_exclusive: usize,
15340    ) {
15341        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15342            return;
15343        }
15344        if start_row0 >= end_row0_exclusive {
15345            return;
15346        }
15347
15348        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15349            return;
15350        };
15351        if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
15352            return;
15353        }
15354        let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
15355        if start_row0 >= end_row0_exclusive {
15356            return;
15357        }
15358
15359        let starts = asheet.chunk_starts.clone();
15360        let nrows = asheet.nrows as usize;
15361        let mut delta = 0isize;
15362        let Some(col) = asheet.columns.get_mut(col0) else {
15363            return;
15364        };
15365        for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15366            let Some(&chunk_start) = starts.get(chunk_idx) else {
15367                continue;
15368            };
15369            let chunk_end = starts
15370                .get(chunk_idx + 1)
15371                .copied()
15372                .unwrap_or(nrows)
15373                .min(chunk_start.saturating_add(ch.len()));
15374            let clear_start = start_row0.max(chunk_start);
15375            let clear_end = end_row0_exclusive.min(chunk_end);
15376            if clear_start >= clear_end {
15377                continue;
15378            }
15379            if clear_start == chunk_start && clear_end == chunk_end {
15380                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15381            } else {
15382                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15383                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15384                delta = delta.saturating_add(
15385                    ch.computed_overlay
15386                        .remove_range(start_in_chunk..end_in_chunk),
15387                );
15388            }
15389        }
15390        for (chunk_idx, ch) in &mut col.sparse_chunks {
15391            let Some(&chunk_start) = starts.get(*chunk_idx) else {
15392                continue;
15393            };
15394            let chunk_end = starts
15395                .get(*chunk_idx + 1)
15396                .copied()
15397                .unwrap_or(nrows)
15398                .min(chunk_start.saturating_add(ch.len()));
15399            let clear_start = start_row0.max(chunk_start);
15400            let clear_end = end_row0_exclusive.min(chunk_end);
15401            if clear_start >= clear_end {
15402                continue;
15403            }
15404            if clear_start == chunk_start && clear_end == chunk_end {
15405                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15406            } else {
15407                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
15408                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
15409                delta = delta.saturating_add(
15410                    ch.computed_overlay
15411                        .remove_range(start_in_chunk..end_in_chunk),
15412                );
15413            }
15414        }
15415        self.adjust_computed_overlay_bytes(delta);
15416    }
15417
15418    fn clear_computed_overlay_cells_in_region(
15419        &mut self,
15420        cells: &[(SheetId, u32, u32)],
15421        affected_region: &Region,
15422    ) {
15423        let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
15424        for (formula_sheet_id, row, col) in cells {
15425            let row0 = row.saturating_sub(1);
15426            let col0 = col.saturating_sub(1);
15427            let placement_region = Region::point(*formula_sheet_id, row0, col0);
15428            if placement_region.intersects(affected_region) {
15429                by_col
15430                    .entry((*formula_sheet_id, col0))
15431                    .or_default()
15432                    .push(row0);
15433            }
15434        }
15435
15436        for ((formula_sheet_id, col0), mut rows) in by_col {
15437            rows.sort_unstable();
15438            rows.dedup();
15439            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
15440            let mut start = rows[0];
15441            let mut prev = rows[0];
15442            for row in rows.into_iter().skip(1) {
15443                if row == prev.saturating_add(1) {
15444                    prev = row;
15445                    continue;
15446                }
15447                self.clear_computed_overlay_col_row_range(
15448                    &sheet_name,
15449                    col0 as usize,
15450                    start as usize,
15451                    prev.saturating_add(1) as usize,
15452                );
15453                start = row;
15454                prev = row;
15455            }
15456            self.clear_computed_overlay_col_row_range(
15457                &sheet_name,
15458                col0 as usize,
15459                start as usize,
15460                prev.saturating_add(1) as usize,
15461            );
15462        }
15463    }
15464
15465    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
15466        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15467            return;
15468        }
15469
15470        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15471            return;
15472        };
15473        if start_row0 >= asheet.nrows as usize {
15474            return;
15475        }
15476
15477        let starts = asheet.chunk_starts.clone();
15478        let nrows = asheet.nrows as usize;
15479        let mut delta = 0isize;
15480        for col in &mut asheet.columns {
15481            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
15482                let Some(&chunk_start) = starts.get(chunk_idx) else {
15483                    continue;
15484                };
15485                let chunk_end = starts
15486                    .get(chunk_idx + 1)
15487                    .copied()
15488                    .unwrap_or(nrows)
15489                    .min(chunk_start.saturating_add(ch.len()));
15490                if chunk_end <= start_row0 {
15491                    continue;
15492                }
15493                if chunk_start >= start_row0 {
15494                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15495                } else {
15496                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15497                    delta = delta
15498                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15499                }
15500            }
15501
15502            for (chunk_idx, ch) in &mut col.sparse_chunks {
15503                let Some(&chunk_start) = starts.get(*chunk_idx) else {
15504                    continue;
15505                };
15506                let chunk_end = starts
15507                    .get(*chunk_idx + 1)
15508                    .copied()
15509                    .unwrap_or(nrows)
15510                    .min(chunk_start.saturating_add(ch.len()));
15511                if chunk_end <= start_row0 {
15512                    continue;
15513                }
15514                if chunk_start >= start_row0 {
15515                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15516                } else {
15517                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
15518                    delta = delta
15519                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
15520                }
15521            }
15522        }
15523        self.adjust_computed_overlay_bytes(delta);
15524    }
15525
15526    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
15527        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
15528            return;
15529        }
15530
15531        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
15532            return;
15533        };
15534        if start_col0 >= asheet.columns.len() {
15535            return;
15536        }
15537
15538        let mut delta = 0isize;
15539        for col in asheet.columns.iter_mut().skip(start_col0) {
15540            for ch in &mut col.chunks {
15541                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15542            }
15543            for ch in col.sparse_chunks.values_mut() {
15544                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
15545            }
15546        }
15547        self.adjust_computed_overlay_bytes(delta);
15548    }
15549
15550    #[inline]
15551    fn literal_to_overlay_value(
15552        value: &LiteralValue,
15553        date_system: crate::engine::DateSystem,
15554    ) -> crate::arrow_store::OverlayValue {
15555        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
15556    }
15557
15558    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
15559        self.arrow_sheets
15560            .sheet(sheet)
15561            .map(|sheet| sheet.date_system)
15562            .unwrap_or(self.config.date_system)
15563    }
15564
15565    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
15566    /// absent and explicit `Empty`.
15567    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15568        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15569            return None;
15570        }
15571        let asheet = self.arrow_sheets.sheet(sheet)?;
15572        let row0 = row.saturating_sub(1) as usize;
15573        let col0 = col.saturating_sub(1) as usize;
15574        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15575            return None;
15576        }
15577        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15578        let ch = asheet.columns[col0].chunk(ch_idx)?;
15579        ch.overlay
15580            .get_scalar(in_off)
15581            .map(|ov| ov.to_literal_for(asheet.date_system))
15582    }
15583
15584    /// Read a single cell's computed overlay entry (if present), preserving the distinction
15585    /// between absent and explicit `Empty`.
15586    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
15587        if !(self.config.arrow_storage_enabled
15588            && self.config.delta_overlay_enabled
15589            && self.config.write_formula_overlay_enabled)
15590        {
15591            return None;
15592        }
15593        let asheet = self.arrow_sheets.sheet(sheet)?;
15594        let row0 = row.saturating_sub(1) as usize;
15595        let col0 = col.saturating_sub(1) as usize;
15596        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
15597            return None;
15598        }
15599        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
15600        let ch = asheet.columns[col0].chunk(ch_idx)?;
15601        ch.computed_overlay
15602            .get_scalar(in_off)
15603            .map(|ov| ov.to_literal_for(asheet.date_system))
15604    }
15605
15606    fn set_delta_overlay_cell_raw(
15607        &mut self,
15608        sheet: &str,
15609        row: u32,
15610        col: u32,
15611        value: Option<LiteralValue>,
15612    ) {
15613        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
15614            return;
15615        }
15616
15617        self.ensure_arrow_sheet(sheet);
15618        let date_system = self.arrow_sheet_date_system(sheet);
15619        let ov_opt = value
15620            .as_ref()
15621            .map(|value| Self::literal_to_overlay_value(value, date_system));
15622        let row0 = row.saturating_sub(1) as usize;
15623        let col0 = col.saturating_sub(1) as usize;
15624        let asheet = self
15625            .arrow_sheets
15626            .sheet_mut(sheet)
15627            .expect("ArrowSheet must exist");
15628
15629        let cur_cols = asheet.columns.len();
15630        if col0 >= cur_cols {
15631            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15632        }
15633        if row0 >= asheet.nrows as usize {
15634            if asheet.columns.is_empty() {
15635                asheet.insert_columns(0, 1);
15636            }
15637            asheet.ensure_row_capacity(row0 + 1);
15638        }
15639
15640        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15641            return;
15642        };
15643        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15644            return;
15645        };
15646
15647        if let Some(ov) = ov_opt {
15648            let _ = ch.overlay.set(in_off, ov);
15649        } else {
15650            let _ = ch.overlay.remove(in_off);
15651        }
15652    }
15653
15654    fn set_computed_overlay_cell_raw(
15655        &mut self,
15656        sheet: &str,
15657        row: u32,
15658        col: u32,
15659        value: Option<LiteralValue>,
15660    ) {
15661        if !(self.config.arrow_storage_enabled
15662            && self.config.delta_overlay_enabled
15663            && self.config.write_formula_overlay_enabled)
15664        {
15665            return;
15666        }
15667
15668        self.ensure_arrow_sheet(sheet);
15669        let date_system = self.arrow_sheet_date_system(sheet);
15670        let ov_opt = value
15671            .as_ref()
15672            .map(|value| Self::literal_to_overlay_value(value, date_system));
15673        let row0 = row.saturating_sub(1) as usize;
15674        let col0 = col.saturating_sub(1) as usize;
15675        let asheet = self
15676            .arrow_sheets
15677            .sheet_mut(sheet)
15678            .expect("ArrowSheet must exist");
15679
15680        let cur_cols = asheet.columns.len();
15681        if col0 >= cur_cols {
15682            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15683        }
15684        if row0 >= asheet.nrows as usize {
15685            if asheet.columns.is_empty() {
15686                asheet.insert_columns(0, 1);
15687            }
15688            asheet.ensure_row_capacity(row0 + 1);
15689        }
15690
15691        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15692            return;
15693        };
15694        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15695            return;
15696        };
15697
15698        let delta = if let Some(ov) = ov_opt {
15699            ch.computed_overlay.set(in_off, ov)
15700        } else {
15701            ch.computed_overlay.remove(in_off)
15702        };
15703        self.adjust_computed_overlay_bytes(delta);
15704    }
15705
15706    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
15707        use crate::engine::ArrowOp;
15708
15709        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
15710            Box::new(batch.ops.iter().rev())
15711        } else {
15712            Box::new(batch.ops.iter())
15713        };
15714
15715        for op in iter {
15716            match op {
15717                ArrowOp::SetDeltaCell {
15718                    sheet_id,
15719                    row0,
15720                    col0,
15721                    old,
15722                    new,
15723                } => {
15724                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15725                    let v = if undo { old.clone() } else { new.clone() };
15726                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15727                }
15728                ArrowOp::SetComputedCell {
15729                    sheet_id,
15730                    row0,
15731                    col0,
15732                    old,
15733                    new,
15734                } => {
15735                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15736                    let v = if undo { old.clone() } else { new.clone() };
15737                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
15738                }
15739                ArrowOp::RestoreComputedRect {
15740                    sheet_id,
15741                    sr0,
15742                    sc0,
15743                    er0,
15744                    ec0,
15745                    old,
15746                    new,
15747                } => {
15748                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15749                    let vals = if undo { old } else { new };
15750                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
15751                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
15752                    for r in 0..height {
15753                        for c in 0..width {
15754                            let v = vals
15755                                .get(r)
15756                                .and_then(|row| row.get(c))
15757                                .cloned()
15758                                .unwrap_or(LiteralValue::Empty);
15759                            self.set_computed_overlay_cell_raw(
15760                                &sheet,
15761                                *sr0 + 1 + r as u32,
15762                                *sc0 + 1 + c as u32,
15763                                Some(v),
15764                            );
15765                        }
15766                    }
15767                }
15768                ArrowOp::InsertRows {
15769                    sheet_id,
15770                    before0,
15771                    count,
15772                } => {
15773                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15774                    self.ensure_arrow_sheet(&sheet);
15775                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15776                        if undo {
15777                            asheet.delete_rows(*before0 as usize, *count as usize);
15778                        } else {
15779                            asheet.insert_rows(*before0 as usize, *count as usize);
15780                        }
15781                    }
15782                }
15783                ArrowOp::InsertCols {
15784                    sheet_id,
15785                    before0,
15786                    count,
15787                } => {
15788                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
15789                    self.ensure_arrow_sheet(&sheet);
15790                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
15791                        if undo {
15792                            asheet.delete_columns(*before0 as usize, *count as usize);
15793                        } else {
15794                            asheet.insert_columns(*before0 as usize, *count as usize);
15795                        }
15796                    }
15797                }
15798            }
15799        }
15800    }
15801
15802    fn record_spill_ops_into_arrow_undo(
15803        &mut self,
15804        undo: &mut crate::engine::ArrowUndoBatch,
15805        events: &[crate::engine::ChangeEvent],
15806    ) {
15807        use crate::engine::ChangeEvent;
15808        use formualizer_common::LiteralValue;
15809
15810        #[allow(clippy::type_complexity)]
15811        let rect_from_snapshot =
15812            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
15813             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
15814                if snap.target_cells.is_empty() {
15815                    return None;
15816                }
15817                let sheet_id = snap.target_cells[0].sheet_id;
15818                let sr0 = snap.target_cells[0].coord.row();
15819                let sc0 = snap.target_cells[0].coord.col();
15820                if snap.values.is_empty() || snap.values[0].is_empty() {
15821                    return None;
15822                }
15823                let h = snap.values.len() as u32;
15824                let w = snap.values[0].len() as u32;
15825                let er0 = sr0.saturating_add(h.saturating_sub(1));
15826                let ec0 = sc0.saturating_add(w.saturating_sub(1));
15827                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
15828            };
15829
15830        for ev in events {
15831            match ev {
15832                ChangeEvent::SpillCommitted { old, new, .. } => {
15833                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
15834                        let old_vals = if let Some(old_snap) = old {
15835                            rect_from_snapshot(old_snap)
15836                                .map(|(_, _, _, _, _, v)| v)
15837                                .unwrap_or_else(|| {
15838                                    vec![
15839                                        vec![LiteralValue::Empty; new_vals[0].len()];
15840                                        new_vals.len()
15841                                    ]
15842                                })
15843                        } else {
15844                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
15845                        };
15846                        undo.record_restore_computed_rect(
15847                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15848                        );
15849                    }
15850                }
15851                ChangeEvent::SpillCleared { old, .. } => {
15852                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
15853                        let new_vals =
15854                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
15855                        undo.record_restore_computed_rect(
15856                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
15857                        );
15858                    }
15859                }
15860                _ => {}
15861            }
15862        }
15863    }
15864
15865    /// Mirror a value into the computed overlay (formula/spill outputs).
15866    ///
15867    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
15868    /// If the cap is exceeded, computed overlays are compacted into base lanes.
15869    fn mirror_value_to_computed_overlay(
15870        &mut self,
15871        sheet: &str,
15872        row: u32,
15873        col: u32,
15874        value: &LiteralValue,
15875    ) {
15876        if !(self.config.arrow_storage_enabled
15877            && self.config.delta_overlay_enabled
15878            && self.config.write_formula_overlay_enabled)
15879        {
15880            return;
15881        }
15882        if self.computed_overlay_mirroring_disabled {
15883            return;
15884        }
15885
15886        let date_system = self.arrow_sheet_date_system(sheet);
15887        let ov = Self::literal_to_overlay_value(value, date_system);
15888        self.write_computed_overlay_value_0based(
15889            sheet,
15890            row.saturating_sub(1),
15891            col.saturating_sub(1),
15892            ov,
15893        );
15894    }
15895
15896    fn write_computed_overlay_value_0based(
15897        &mut self,
15898        sheet: &str,
15899        row0: u32,
15900        col0: u32,
15901        value: OverlayValue,
15902    ) {
15903        if !(self.config.arrow_storage_enabled
15904            && self.config.delta_overlay_enabled
15905            && self.config.write_formula_overlay_enabled)
15906        {
15907            return;
15908        }
15909        if self.computed_overlay_mirroring_disabled {
15910            return;
15911        }
15912
15913        self.ensure_arrow_sheet(sheet);
15914
15915        let row0 = row0 as usize;
15916        let col0 = col0 as usize;
15917        let asheet = self
15918            .arrow_sheets
15919            .sheet_mut(sheet)
15920            .expect("ArrowSheet must exist");
15921
15922        let cur_cols = asheet.columns.len();
15923        if col0 >= cur_cols {
15924            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
15925        }
15926
15927        if row0 >= asheet.nrows as usize {
15928            if asheet.columns.is_empty() {
15929                asheet.insert_columns(0, 1);
15930            }
15931            asheet.ensure_row_capacity(row0 + 1);
15932        }
15933
15934        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
15935            return;
15936        };
15937        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
15938            return;
15939        };
15940
15941        let delta = ch.computed_overlay.set_scalar(in_off, value);
15942        self.adjust_computed_overlay_bytes(delta);
15943
15944        if let Some(cap) = self.config.max_overlay_memory_bytes
15945            && self.computed_overlay_bytes_estimate > cap
15946        {
15947            self.disable_computed_overlay_mirroring_due_to_budget(cap);
15948        }
15949    }
15950
15951    pub(crate) fn plan_computed_write_coalescing(
15952        &self,
15953        buffer: &ComputedWriteBuffer,
15954    ) -> ComputedWriteCoalescingPlan {
15955        self.plan_computed_write_coalescing_from_writes(buffer.writes().iter().cloned())
15956    }
15957
15958    fn plan_owned_computed_write_coalescing(
15959        &self,
15960        writes: Vec<ComputedWrite>,
15961    ) -> ComputedWriteCoalescingPlan {
15962        self.plan_computed_write_coalescing_from_writes(writes)
15963    }
15964
15965    fn plan_computed_write_coalescing_from_writes(
15966        &self,
15967        writes: impl IntoIterator<Item = ComputedWrite>,
15968    ) -> ComputedWriteCoalescingPlan {
15969        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
15970            BTreeMap::new();
15971        let mut input_cells = 0usize;
15972
15973        for write in writes {
15974            match write {
15975                ComputedWrite::Cell {
15976                    seq,
15977                    sheet_id,
15978                    row0,
15979                    col0,
15980                    value,
15981                } => {
15982                    input_cells = input_cells.saturating_add(1);
15983                    self.push_computed_write_plan_entry(
15984                        &mut groups,
15985                        seq,
15986                        sheet_id,
15987                        row0,
15988                        col0,
15989                        value,
15990                    );
15991                }
15992                ComputedWrite::Rect {
15993                    seq,
15994                    sheet_id,
15995                    sr0,
15996                    sc0,
15997                    values,
15998                } => {
15999                    for (r_off, row) in values.into_iter().enumerate() {
16000                        for (c_off, value) in row.into_iter().enumerate() {
16001                            input_cells = input_cells.saturating_add(1);
16002                            self.push_computed_write_plan_entry(
16003                                &mut groups,
16004                                seq,
16005                                sheet_id,
16006                                sr0.saturating_add(r_off as u32),
16007                                sc0.saturating_add(c_off as u32),
16008                                value,
16009                            );
16010                        }
16011                    }
16012                }
16013            }
16014        }
16015
16016        let mut plan = ComputedWriteCoalescingPlan {
16017            chunks: Vec::with_capacity(groups.len()),
16018            input_cells,
16019            coalesced_cells: 0,
16020            overwritten_cells: 0,
16021        };
16022        for (key, entries) in groups {
16023            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(key, entries);
16024            plan.coalesced_cells = plan
16025                .coalesced_cells
16026                .saturating_add(chunk_plan.entries.len());
16027            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
16028            plan.chunks.push(chunk_plan);
16029        }
16030        debug_assert_eq!(
16031            plan.input_cells,
16032            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
16033        );
16034        plan
16035    }
16036
16037    fn push_computed_write_plan_entry(
16038        &self,
16039        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
16040        seq: u64,
16041        sheet_id: SheetId,
16042        row0: u32,
16043        col0: u32,
16044        value: OverlayValue,
16045    ) {
16046        let (chunk_idx, chunk_start_row0, row_in_chunk) =
16047            self.locate_computed_write_chunk(sheet_id, row0);
16048        let key = ComputedWriteChunkKey {
16049            sheet_id,
16050            col0,
16051            chunk_idx,
16052            chunk_start_row0,
16053        };
16054        groups
16055            .entry(key)
16056            .or_default()
16057            .push(ComputedWriteChunkEntryPlan {
16058                row_in_chunk,
16059                seq,
16060                value,
16061            });
16062    }
16063
16064    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
16065        let sheet_name = self.graph.sheet_name(sheet_id);
16066        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
16067            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
16068        }
16069        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
16070    }
16071
16072    fn locate_row_in_sheet_for_computed_write_plan(
16073        sheet: &crate::arrow_store::ArrowSheet,
16074        row0: usize,
16075    ) -> (usize, u32, usize) {
16076        if row0 < sheet.nrows as usize
16077            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
16078        {
16079            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
16080            return (chunk_idx, chunk_start as u32, row_in_chunk);
16081        }
16082
16083        let chunk_rows = sheet.chunk_rows.max(1);
16084        if sheet.chunk_starts.is_empty() {
16085            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
16086        }
16087
16088        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
16089        let mut chunk_start = sheet.chunk_starts[chunk_idx];
16090        while chunk_start.saturating_add(chunk_rows) <= row0 {
16091            chunk_idx = chunk_idx.saturating_add(1);
16092            chunk_start = chunk_start.saturating_add(chunk_rows);
16093        }
16094        (
16095            chunk_idx,
16096            chunk_start as u32,
16097            row0.saturating_sub(chunk_start),
16098        )
16099    }
16100
16101    fn locate_row_in_empty_sheet_for_computed_write_plan(
16102        row0: usize,
16103        chunk_rows: usize,
16104    ) -> (usize, u32, usize) {
16105        let chunk_rows = chunk_rows.max(1);
16106        let chunk_idx = row0 / chunk_rows;
16107        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
16108        (
16109            chunk_idx,
16110            chunk_start as u32,
16111            row0.saturating_sub(chunk_start),
16112        )
16113    }
16114
16115    #[cfg(test)]
16116    pub(crate) fn debug_plan_computed_write_coalescing(
16117        &self,
16118        buffer: &ComputedWriteBuffer,
16119    ) -> ComputedWriteCoalescingPlan {
16120        self.plan_computed_write_coalescing(buffer)
16121    }
16122
16123    fn record_computed_write_buffer_delta(
16124        &self,
16125        buffer: &ComputedWriteBuffer,
16126        delta: &mut DeltaCollector,
16127    ) {
16128        if delta.mode == DeltaMode::Off {
16129            return;
16130        }
16131        let mut final_values = BTreeMap::new();
16132        for write in buffer.writes() {
16133            match write {
16134                ComputedWrite::Cell {
16135                    sheet_id,
16136                    row0,
16137                    col0,
16138                    value,
16139                    ..
16140                } => {
16141                    final_values.insert((*sheet_id, *row0, *col0), value.clone());
16142                }
16143                ComputedWrite::Rect {
16144                    sheet_id,
16145                    sr0,
16146                    sc0,
16147                    values,
16148                    ..
16149                } => {
16150                    for (row_offset, row) in values.iter().enumerate() {
16151                        for (col_offset, value) in row.iter().enumerate() {
16152                            final_values.insert(
16153                                (
16154                                    *sheet_id,
16155                                    sr0.saturating_add(row_offset as u32),
16156                                    sc0.saturating_add(col_offset as u32),
16157                                ),
16158                                value.clone(),
16159                            );
16160                        }
16161                    }
16162                }
16163            }
16164        }
16165        for ((sheet_id, row0, col0), value) in final_values {
16166            let old = self
16167                .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
16168                .unwrap_or(LiteralValue::Empty);
16169            if old != value.to_literal_for(self.config.date_system) {
16170                delta.record_cell(sheet_id, row0, col0);
16171            }
16172        }
16173    }
16174
16175    pub(crate) fn flush_computed_write_buffer(
16176        &mut self,
16177        buffer: &mut ComputedWriteBuffer,
16178    ) -> Result<(), ExcelError> {
16179        if buffer.is_empty() {
16180            return Ok(());
16181        }
16182
16183        // Keep ownership of all pending writes until the final request
16184        // checkpoint and bounded commit-window preflight succeed so failures
16185        // remain retry safe. The plan and flush that follow are prevalidated
16186        // and contain no cancellation point.
16187        self.resource_checkpoint(0)?;
16188        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
16189        let plan = self.plan_owned_computed_write_coalescing(buffer.take_writes());
16190        self.flush_computed_write_plan(plan);
16191        self.observe_evaluation_commit_window(commit_started);
16192
16193        Ok(())
16194    }
16195
16196    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
16197        for chunk in plan.chunks {
16198            self.flush_computed_write_chunk_plan(chunk);
16199        }
16200    }
16201
16202    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
16203        match &chunk.shape {
16204            ComputedWriteChunkPlanShape::Point => {
16205                self.flush_computed_write_chunk_plan_as_points(chunk);
16206            }
16207            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
16208                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
16209            }
16210            ComputedWriteChunkPlanShape::DenseRange { .. } => {
16211                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16212            }
16213            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
16214                if Self::should_emit_computed_run_fragment(*len, *runs) {
16215                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
16216                } else {
16217                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
16218                }
16219            }
16220        }
16221    }
16222
16223    #[inline]
16224    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
16225        runs <= len / 2
16226    }
16227
16228    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
16229        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
16230        for entry in chunk.entries {
16231            let row0 = chunk
16232                .chunk_start_row0
16233                .saturating_add(entry.row_in_chunk as u32);
16234            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
16235        }
16236    }
16237
16238    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
16239        &mut self,
16240        chunk: ComputedWriteChunkPlan,
16241    ) {
16242        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
16243        let sheet_id = chunk.sheet_id;
16244        let col0 = chunk.col0;
16245        let chunk_idx = chunk.chunk_idx;
16246        let chunk_start_row0 = chunk.chunk_start_row0;
16247        let items: Vec<(usize, OverlayValue)> = chunk
16248            .entries
16249            .into_iter()
16250            .map(|entry| (entry.row_in_chunk, entry.value))
16251            .collect();
16252        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
16253            items,
16254            point_estimate,
16255        ) {
16256            Some(Ok(fragment)) => {
16257                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
16258            }
16259            Some(Err(cells)) => {
16260                self.flush_computed_overlay_cells_as_points(
16261                    sheet_id,
16262                    col0,
16263                    chunk_start_row0,
16264                    cells,
16265                );
16266            }
16267            None => {}
16268        }
16269    }
16270
16271    #[inline]
16272    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
16273        chunk
16274            .entries
16275            .iter()
16276            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
16277            .fold(0usize, usize::saturating_add)
16278    }
16279
16280    fn flush_computed_overlay_cells_as_points(
16281        &mut self,
16282        sheet_id: SheetId,
16283        col0: u32,
16284        chunk_start_row0: u32,
16285        cells: Vec<(usize, OverlayValue)>,
16286    ) {
16287        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16288        for (row_in_chunk, value) in cells {
16289            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
16290            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
16291        }
16292    }
16293
16294    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16295        if chunk.entries.is_empty() {
16296            return;
16297        }
16298        let start = chunk.entries[0].row_in_chunk;
16299        let values: Vec<OverlayValue> =
16300            chunk.entries.into_iter().map(|entry| entry.value).collect();
16301        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
16302            self.apply_computed_overlay_fragment(
16303                chunk.sheet_id,
16304                chunk.col0,
16305                chunk.chunk_idx,
16306                fragment,
16307            );
16308        }
16309    }
16310
16311    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
16312        if chunk.entries.is_empty() {
16313            return;
16314        }
16315        let start = chunk.entries[0].row_in_chunk;
16316        let values: Vec<OverlayValue> =
16317            chunk.entries.into_iter().map(|entry| entry.value).collect();
16318        if let Some(fragment) = OverlayFragment::run_range(start, values) {
16319            self.apply_computed_overlay_fragment(
16320                chunk.sheet_id,
16321                chunk.col0,
16322                chunk.chunk_idx,
16323                fragment,
16324            );
16325        }
16326    }
16327
16328    fn apply_computed_overlay_fragment(
16329        &mut self,
16330        sheet_id: SheetId,
16331        col0: u32,
16332        chunk_idx: usize,
16333        fragment: OverlayFragment,
16334    ) {
16335        if !(self.config.arrow_storage_enabled
16336            && self.config.delta_overlay_enabled
16337            && self.config.write_formula_overlay_enabled)
16338        {
16339            return;
16340        }
16341        if self.computed_overlay_mirroring_disabled {
16342            return;
16343        }
16344
16345        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16346        self.ensure_arrow_sheet(&sheet_name);
16347
16348        let col0 = col0 as usize;
16349        let asheet = self
16350            .arrow_sheets
16351            .sheet_mut(&sheet_name)
16352            .expect("ArrowSheet must exist");
16353
16354        let cur_cols = asheet.columns.len();
16355        if col0 >= cur_cols {
16356            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16357        }
16358
16359        let start_row0 = asheet
16360            .chunk_starts
16361            .get(chunk_idx)
16362            .copied()
16363            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
16364        let required_rows =
16365            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
16366        if required_rows > asheet.nrows as usize {
16367            if asheet.columns.is_empty() {
16368                asheet.insert_columns(0, 1);
16369            }
16370            asheet.ensure_row_capacity(required_rows);
16371        }
16372
16373        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
16374            return;
16375        };
16376        let delta = ch.computed_overlay.apply_fragment(fragment);
16377        self.adjust_computed_overlay_bytes(delta);
16378
16379        if let Some(cap) = self.config.max_overlay_memory_bytes
16380            && self.computed_overlay_bytes_estimate > cap
16381        {
16382            self.disable_computed_overlay_mirroring_due_to_budget(cap);
16383        }
16384    }
16385
16386    #[inline]
16387    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
16388        if delta >= 0 {
16389            self.computed_overlay_bytes_estimate = self
16390                .computed_overlay_bytes_estimate
16391                .saturating_add(delta as usize);
16392        } else {
16393            self.computed_overlay_bytes_estimate = self
16394                .computed_overlay_bytes_estimate
16395                .saturating_sub((-delta) as usize);
16396        }
16397    }
16398
16399    fn clear_all_computed_overlays(&mut self) {
16400        let mut freed_total = 0usize;
16401        for sh in self.arrow_sheets.sheets.iter_mut() {
16402            for col in sh.columns.iter_mut() {
16403                for ch in col.chunks.iter_mut() {
16404                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16405                }
16406                for ch in col.sparse_chunks.values_mut() {
16407                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
16408                }
16409            }
16410        }
16411        self.computed_overlay_bytes_estimate = self
16412            .computed_overlay_bytes_estimate
16413            .saturating_sub(freed_total);
16414    }
16415
16416    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
16417        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
16418        // Handle budget pressure by compacting computed overlays into base lanes.
16419        self.compact_all_computed_overlays();
16420    }
16421
16422    /// Fold all computed overlay entries across all sheets into their base arrays.
16423    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
16424    fn compact_all_computed_overlays(&mut self) {
16425        let mut freed_total = 0usize;
16426        for sheet in self.arrow_sheets.sheets.iter_mut() {
16427            for col_idx in 0..sheet.columns.len() {
16428                // Dense chunks
16429                let num_dense = sheet.columns[col_idx].chunks.len();
16430                for ch_idx in 0..num_dense {
16431                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
16432                }
16433                // Sparse chunks
16434                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
16435                    .sparse_chunks
16436                    .keys()
16437                    .copied()
16438                    .collect();
16439                for ch_idx in sparse_keys {
16440                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
16441                }
16442            }
16443        }
16444        self.computed_overlay_bytes_estimate = self
16445            .computed_overlay_bytes_estimate
16446            .saturating_sub(freed_total);
16447        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16448    }
16449
16450    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
16451        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
16452    }
16453
16454    fn record_vertex_value_to_overlay(
16455        &mut self,
16456        vertex_id: VertexId,
16457        value: &LiteralValue,
16458        computed_writes: Option<&mut ComputedWriteBuffer>,
16459    ) -> Result<(), ExcelError> {
16460        if !(self.config.arrow_storage_enabled
16461            && self.config.delta_overlay_enabled
16462            && self.config.write_formula_overlay_enabled)
16463        {
16464            return Ok(());
16465        }
16466        if self.computed_overlay_mirroring_disabled {
16467            return Ok(());
16468        }
16469        if !matches!(
16470            self.graph.get_vertex_kind(vertex_id),
16471            VertexKind::FormulaScalar | VertexKind::FormulaArray
16472        ) {
16473            return Ok(());
16474        }
16475        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
16476            return Ok(());
16477        };
16478        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
16479        let date_system = self.arrow_sheet_date_system(&sheet_name);
16480        let ov = Self::literal_to_overlay_value(value, date_system);
16481        if let Some(buffer) = computed_writes {
16482            buffer.push_cell(cell.sheet_id, cell.coord.row(), cell.coord.col(), ov);
16483            if self.should_flush_computed_write_buffer(buffer) {
16484                self.flush_computed_write_buffer(buffer)?;
16485            }
16486        } else {
16487            self.write_computed_overlay_value_0based(
16488                &sheet_name,
16489                cell.coord.row(),
16490                cell.coord.col(),
16491                ov,
16492            );
16493        }
16494        Ok(())
16495    }
16496
16497    #[inline]
16498    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
16499        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
16500            if cap == 0 {
16501                return false;
16502            }
16503            self.computed_overlay_bytes_estimate
16504                .saturating_add(buffer.estimated_bytes())
16505                > cap
16506        })
16507    }
16508
16509    /// Estimated memory usage for computed overlays (formula/spill mirroring).
16510    pub fn overlay_memory_usage(&self) -> usize {
16511        self.computed_overlay_bytes_estimate
16512    }
16513
16514    #[cfg(test)]
16515    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
16516        self.overlay_compactions
16517    }
16518
16519    #[cfg(test)]
16520    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
16521        let mut total = 0usize;
16522        for sheet in &self.arrow_sheets.sheets {
16523            for column in &sheet.columns {
16524                for chunk in &column.chunks {
16525                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16526                }
16527                for chunk in column.sparse_chunks.values() {
16528                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
16529                }
16530            }
16531        }
16532        self.computed_overlay_bytes_estimate = total;
16533        total
16534    }
16535
16536    fn resolve_sheet_locator_for_write(
16537        &mut self,
16538        loc: formualizer_common::SheetLocator<'_>,
16539        current_sheet: &str,
16540    ) -> Result<SheetId, ExcelError> {
16541        Ok(match loc {
16542            formualizer_common::SheetLocator::Id(id) => id,
16543            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
16544            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
16545        })
16546    }
16547
16548    fn resolve_sheet_locator_for_read(
16549        &self,
16550        loc: formualizer_common::SheetLocator<'_>,
16551        current_sheet: &str,
16552    ) -> Result<SheetId, ExcelError> {
16553        match loc {
16554            formualizer_common::SheetLocator::Id(id) => Ok(id),
16555            formualizer_common::SheetLocator::Name(name) => self
16556                .graph
16557                .sheet_id(name.as_ref())
16558                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16559            formualizer_common::SheetLocator::Current => self
16560                .graph
16561                .sheet_id(current_sheet)
16562                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
16563        }
16564    }
16565
16566    /// Set a cell value
16567    pub fn set_cell_value(
16568        &mut self,
16569        sheet: &str,
16570        row: u32,
16571        col: u32,
16572        value: LiteralValue,
16573    ) -> Result<(), ExcelError> {
16574        self.observe_function_semantic_epoch()?;
16575        let sheet_existed = self.graph.sheet_id(sheet).is_some();
16576        let sheet_id = self.graph.sheet_id_mut(sheet);
16577        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16578        let replaced_formula =
16579            self.graph
16580                .get_vertex_id_for_address(&cell_ref)
16581                .is_some_and(|vertex| {
16582                    matches!(
16583                        self.graph.get_vertex_kind(*vertex),
16584                        VertexKind::FormulaScalar | VertexKind::FormulaArray
16585                    )
16586                });
16587        self.demote_span_containing_cell_for_write(
16588            sheet_id,
16589            row.saturating_sub(1),
16590            col.saturating_sub(1),
16591        )
16592        .map_err(Self::editor_error_to_excel)?;
16593        self.graph.set_cell_value(sheet, row, col, value.clone())?;
16594        self.record_formula_plane_changed_cell(sheet, row, col);
16595        if !sheet_existed || replaced_formula {
16596            self.mark_topology_edited();
16597        }
16598        // Mirror into Arrow overlay when enabled
16599        self.mirror_value_to_overlay(sheet, row, col, &value);
16600        // Advance snapshot to reflect external mutation
16601        self.snapshot_id
16602            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16603        self.has_edited = true;
16604        Ok(())
16605    }
16606
16607    /// Record a single-cell change in FormulaPlane authority so the next
16608    /// `evaluate_all` under `AuthoritativeExperimental` can derive bounded
16609    /// span work from `FormulaConsumerReadIndex` instead of recomputing every
16610    /// active span.
16611    fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
16612        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16613            return;
16614        }
16615        let sheet_id = self.graph.sheet_id_mut(sheet);
16616        self.record_formula_plane_structural_change(StructuralScope::Cell {
16617            sheet: sheet_id,
16618            row: row.saturating_sub(1),
16619            col: col.saturating_sub(1),
16620        });
16621    }
16622
16623    fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
16624        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16625            return;
16626        }
16627
16628        match event {
16629            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16630                self.record_formula_plane_structural_change(StructuralScope::Cell {
16631                    sheet: addr.sheet_id,
16632                    row: addr.coord.row(),
16633                    col: addr.coord.col(),
16634                });
16635            }
16636            ChangeEvent::SpillCommitted { new, .. } => {
16637                if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
16638                    self.record_formula_plane_structural_change(scope);
16639                }
16640            }
16641            ChangeEvent::SpillCleared { old, .. } => {
16642                if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
16643                    self.record_formula_plane_structural_change(scope);
16644                }
16645            }
16646            ChangeEvent::DefineName { .. }
16647            | ChangeEvent::UpdateName { .. }
16648            | ChangeEvent::DeleteName { .. }
16649            | ChangeEvent::NamedRangeAdjusted { .. } => {
16650                // Direct name events are preflighted by the logged-name APIs.
16651                // Structural entry points preflight spans before emitting a
16652                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
16653                self.record_formula_plane_structural_change(StructuralScope::AllSheets);
16654            }
16655            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
16656                // Structural entry points publish their axis delta once after
16657                // graph, Arrow, and span geometry commits.
16658            }
16659            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
16660                self.record_formula_plane_structural_change(StructuralScope::Region(
16661                    Region::whole_row(*sheet_id, *row0),
16662                ));
16663            }
16664            ChangeEvent::AddVertex { .. }
16665            | ChangeEvent::RemoveVertex { .. }
16666            | ChangeEvent::EdgeAdded { .. }
16667            | ChangeEvent::EdgeRemoved { .. }
16668            | ChangeEvent::CompoundStart { .. }
16669            | ChangeEvent::CompoundEnd { .. }
16670            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
16671        }
16672    }
16673
16674    fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
16675        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
16676            return;
16677        }
16678
16679        match scope {
16680            StructuralScope::Cell { sheet, row, col } => {
16681                self.graph
16682                    .mark_formula_region_dirty(Region::point(sheet, row, col));
16683            }
16684            StructuralScope::Region(region) => {
16685                self.graph.mark_formula_region_dirty(region);
16686            }
16687            StructuralScope::Sheet(sheet_id) => {
16688                self.graph
16689                    .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
16690            }
16691            StructuralScope::RemovedSheet(sheet_id) => {
16692                let removed_refs = {
16693                    let authority = self.graph.formula_authority();
16694                    authority
16695                        .active_span_refs()
16696                        .into_iter()
16697                        .filter(|span_ref| {
16698                            authority
16699                                .plane
16700                                .spans
16701                                .get(*span_ref)
16702                                .map(|span| span.sheet_id == sheet_id)
16703                                .unwrap_or(false)
16704                        })
16705                        .collect::<Vec<_>>()
16706                };
16707
16708                {
16709                    let authority = self.graph.formula_authority_mut();
16710                    for span_ref in removed_refs {
16711                        authority.plane.remove_span(span_ref);
16712                    }
16713                    let _ = authority.rebuild_indexes();
16714                }
16715                self.graph
16716                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16717            }
16718            StructuralScope::OpaqueGlobal => {
16719                let _ = self.graph.formula_authority_mut().rebuild_indexes();
16720                self.graph
16721                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
16722            }
16723            StructuralScope::AllSheets => {
16724                // Name/table mutations dirty their exact graph dependents,
16725                // and name-dependent spans are demoted by the name invalidation
16726                // registry. A topology rebuild is cache invalidation only.
16727                let _ = self.graph.formula_authority_mut().rebuild_indexes();
16728            }
16729        }
16730    }
16731
16732    fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
16733        let first = cells.first()?;
16734        let sheet_id = first.sheet_id;
16735        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
16736            return Some(StructuralScope::OpaqueGlobal);
16737        }
16738        let mut row_start = first.coord.row();
16739        let mut row_end = row_start;
16740        let mut col_start = first.coord.col();
16741        let mut col_end = col_start;
16742        for cell in cells.iter().skip(1) {
16743            row_start = row_start.min(cell.coord.row());
16744            row_end = row_end.max(cell.coord.row());
16745            col_start = col_start.min(cell.coord.col());
16746            col_end = col_end.max(cell.coord.col());
16747        }
16748        Some(StructuralScope::Region(Region::rect(
16749            sheet_id, row_start, row_end, col_start, col_end,
16750        )))
16751    }
16752
16753    pub fn set_cell_value_ref(
16754        &mut self,
16755        cell: formualizer_common::SheetCellRef<'_>,
16756        current_sheet: &str,
16757        value: LiteralValue,
16758    ) -> Result<(), ExcelError> {
16759        let owned = cell.into_owned();
16760        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16761        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16762        self.set_cell_value(
16763            &sheet_name,
16764            owned.coord.row() + 1,
16765            owned.coord.col() + 1,
16766            value,
16767        )
16768    }
16769
16770    pub fn set_cell_formula_ref(
16771        &mut self,
16772        cell: formualizer_common::SheetCellRef<'_>,
16773        current_sheet: &str,
16774        ast: ASTNode,
16775    ) -> Result<(), ExcelError> {
16776        let owned = cell.into_owned();
16777        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
16778        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
16779        self.set_cell_formula(
16780            &sheet_name,
16781            owned.coord.row() + 1,
16782            owned.coord.col() + 1,
16783            ast,
16784        )
16785    }
16786
16787    pub fn get_cell_value_ref(
16788        &self,
16789        cell: formualizer_common::SheetCellRef<'_>,
16790        current_sheet: &str,
16791    ) -> Result<Option<LiteralValue>, ExcelError> {
16792        let owned = cell.into_owned();
16793        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
16794        let sheet_name = self.graph.sheet_name(sheet_id);
16795        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
16796    }
16797
16798    pub fn resolve_range_view_sheet_ref<'c>(
16799        &'c self,
16800        r: &formualizer_common::SheetRef<'_>,
16801        current_sheet: &str,
16802    ) -> Result<RangeView<'c>, ExcelError> {
16803        use formualizer_common::SheetLocator;
16804
16805        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
16806            match loc {
16807                SheetLocator::Current => Ok(None),
16808                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
16809                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
16810            }
16811        };
16812
16813        let rt = match r {
16814            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
16815                sheet: sheet_to_opt_name(cell.sheet.clone())?,
16816                row: cell.coord.row() + 1,
16817                col: cell.coord.col() + 1,
16818                row_abs: cell.coord.row_abs(),
16819                col_abs: cell.coord.col_abs(),
16820            },
16821            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
16822                sheet: sheet_to_opt_name(range.sheet.clone())?,
16823                start_row: range.start_row.map(|b| b.index + 1),
16824                start_col: range.start_col.map(|b| b.index + 1),
16825                end_row: range.end_row.map(|b| b.index + 1),
16826                end_col: range.end_col.map(|b| b.index + 1),
16827                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
16828                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
16829                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
16830                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
16831            },
16832        };
16833
16834        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
16835    }
16836
16837    /// Set a cell formula
16838    pub fn set_cell_formula(
16839        &mut self,
16840        sheet: &str,
16841        row: u32,
16842        col: u32,
16843        ast: ASTNode,
16844    ) -> Result<(), ExcelError> {
16845        self.observe_function_semantic_epoch()?;
16846        let sheet_id = self.graph.sheet_id_mut(sheet);
16847        self.demote_span_containing_cell_for_write(
16848            sheet_id,
16849            row.saturating_sub(1),
16850            col.saturating_sub(1),
16851        )
16852        .map_err(Self::editor_error_to_excel)?;
16853        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16854        let ingested = {
16855            let mut pipeline = self.ingest_pipeline();
16856            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
16857        };
16858        self.graph.set_cell_formula_with_plan(
16859            sheet,
16860            row,
16861            col,
16862            ingested.ast_id,
16863            &ingested.dep_plan,
16864            ingested.dep_plan.volatile,
16865            ingested.dep_plan.dynamic,
16866        )?;
16867        self.record_formula_plane_changed_cell(sheet, row, col);
16868
16869        // If the cell previously held a user value in the delta overlay, it must not continue
16870        // to mask the formula result under Arrow-canonical reads (overlay precedence is
16871        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
16872        // because an explicit `Empty` overlay would still take precedence over computed values.
16873        self.clear_delta_overlay_cell(sheet, row, col);
16874
16875        // Advance snapshot to reflect external mutation
16876        self.mark_topology_edited();
16877        Ok(())
16878    }
16879
16880    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
16881    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
16882    where
16883        I: IntoIterator<Item = (u32, u32, ASTNode)>,
16884    {
16885        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
16886        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
16887        let sheet_id = self.graph.sheet_id_mut(sheet);
16888        let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
16889            let placement =
16890                PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
16891            self.graph
16892                .formula_authority()
16893                .plane
16894                .spans
16895                .find_at(placement)
16896                .is_some()
16897        });
16898        if writes_inside_active_span {
16899            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
16900                .map_err(Self::editor_error_to_excel)?;
16901        }
16902        let ingested = {
16903            let mut pipeline = self.ingest_pipeline();
16904            let inputs = collected.into_iter().map(|(row, col, ast)| {
16905                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
16906                (FormulaAstInput::Tree(ast), placement, None)
16907            });
16908            pipeline.ingest_batch(inputs)?
16909        };
16910        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
16911            .into_iter()
16912            .map(|formula| {
16913                (
16914                    formula.placement.coord.row() + 1,
16915                    formula.placement.coord.col() + 1,
16916                    formula.ast_id,
16917                    formula.dep_plan,
16918                )
16919            })
16920            .collect();
16921        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
16922        for (row, col) in edited_cells {
16923            self.record_formula_plane_changed_cell(sheet, row, col);
16924        }
16925        // Single topology bump after batch
16926        if n > 0 {
16927            self.mark_topology_edited();
16928        }
16929        Ok(n)
16930    }
16931
16932    #[inline]
16933    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
16934        match v {
16935            LiteralValue::Empty => None,
16936            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
16937            other => Some(other),
16938        }
16939    }
16940
16941    /// Get a cell value
16942    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16943        self.read_cell_value(sheet, row, col)
16944            .and_then(Self::normalize_public_cell_read)
16945    }
16946
16947    /// Unified internal read API for a single cell value (Arrow-truth).
16948    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16949        let asheet = self.sheet_store().sheet(sheet)?;
16950        let r0 = row.saturating_sub(1) as usize;
16951        let c0 = col.saturating_sub(1) as usize;
16952        let v = asheet.get_cell_value(r0, c0);
16953        if matches!(v, LiteralValue::Empty) {
16954            None
16955        } else {
16956            Some(v)
16957        }
16958    }
16959
16960    /// Unified internal read API for a range of cell values (Arrow-truth).
16961    pub(crate) fn read_range_values(
16962        &self,
16963        sheet: &str,
16964        sr: u32,
16965        sc: u32,
16966        er: u32,
16967        ec: u32,
16968    ) -> RangeView<'_> {
16969        let Some(asheet) = self.sheet_store().sheet(sheet) else {
16970            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
16971        };
16972        if er < sr || ec < sc {
16973            return asheet.range_view(1, 1, 0, 0);
16974        }
16975        let sr0 = sr.saturating_sub(1) as usize;
16976        let sc0 = sc.saturating_sub(1) as usize;
16977        let er0 = er.saturating_sub(1) as usize;
16978        let ec0 = ec.saturating_sub(1) as usize;
16979        asheet.range_view(sr0, sc0, er0, ec0)
16980    }
16981
16982    pub(crate) fn formula_plane_span_ast_at(
16983        &self,
16984        span_ref: FormulaSpanRef,
16985        placement: PlacementCoord,
16986    ) -> Option<ASTNode> {
16987        let authority = self.graph.formula_authority();
16988        let span = authority.plane.spans.get(span_ref)?;
16989        let relocation = span.ast_relocation;
16990        let mut ast = self
16991            .graph
16992            .data_store()
16993            .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
16994        if let Some(binding_set_id) = span.binding_set_id {
16995            let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
16996            if !binding_set.is_single_literal_binding() {
16997                let binding =
16998                    binding_set.literal_bindings_for_placement(&span.domain, placement)?;
16999                ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
17000            }
17001        }
17002        let row = placement.row.checked_add(1)?;
17003        let col = placement.col.checked_add(1)?;
17004        let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
17005        let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
17006        relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
17007    }
17008
17009    /// Get formula AST (if any) and current stored value for a cell
17010    pub fn get_cell(
17011        &self,
17012        sheet: &str,
17013        row: u32,
17014        col: u32,
17015    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
17016        let v = self.get_cell_value(sheet, row, col);
17017        let sheet_id = self.graph.sheet_id(sheet)?;
17018        let coord = Coord::from_excel(row, col, true, true);
17019        let cell = CellRef::new(sheet_id, coord);
17020        let placement =
17021            crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
17022        let handle = self
17023            .graph
17024            .formula_authority()
17025            .plane
17026            .resolve_formula_at(placement, None);
17027        match handle.resolution {
17028            crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
17029                // Span authority wins before graph lookup. Missing or invalid
17030                // relocation state is fail-closed: never expose a stale legacy
17031                // vertex for a coordinate owned by a span.
17032                let ast = self.formula_plane_span_ast_at(span, placement);
17033                return Some((ast, v));
17034            }
17035            crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
17036                let ast = self
17037                    .graph
17038                    .formula_authority()
17039                    .plane
17040                    .formula_overlay
17041                    .get(overlay_ref)
17042                    .and_then(|overlay| match overlay.kind {
17043                        crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
17044                            template_id,
17045                        ) => self
17046                            .graph
17047                            .formula_authority()
17048                            .plane
17049                            .templates
17050                            .get(template_id)
17051                            .and_then(|template| {
17052                                self.graph
17053                                    .data_store()
17054                                    .retrieve_ast(template.ast_id, self.graph.sheet_reg())
17055                            }),
17056                        _ => None,
17057                    });
17058                return Some((ast, v));
17059            }
17060            _ => {}
17061        }
17062
17063        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
17064            let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
17065                self.graph
17066                    .data_store()
17067                    .retrieve_ast(ast_id, self.graph.sheet_reg())
17068            });
17069            Some((ast, v))
17070        } else if v.is_some() {
17071            Some((None, v))
17072        } else {
17073            None
17074        }
17075    }
17076
17077    /// Begin batch operations - defer CSR rebuilds for better performance
17078    pub fn begin_batch(&mut self) {
17079        self.graph.begin_batch();
17080    }
17081
17082    /// End batch operations and trigger CSR rebuild
17083    pub fn end_batch(&mut self) {
17084        self.graph.end_batch();
17085    }
17086
17087    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
17088    /// every edit's dirty propagation queues its sources instead of running
17089    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
17090    /// the union with ONE multi-source propagation (O(component) instead of
17091    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
17092    ///
17093    /// Callers MUST run `end_deferred_dirty` on every exit path, including
17094    /// error returns; evaluation entry points `debug_assert` no scope leaked.
17095    pub fn begin_deferred_dirty(&mut self) {
17096        self.graph.begin_deferred_dirty();
17097    }
17098
17099    /// End a deferred-dirty scope, flushing the queued propagation when the
17100    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
17101    pub fn end_deferred_dirty(&mut self) {
17102        let _ = self.graph.end_deferred_dirty();
17103    }
17104
17105    /// Total vertices processed by dirty-propagation BFS loops since graph
17106    /// creation. Perf-shape observability only (cross-crate tests assert
17107    /// batched edits propagate O(component), not O(edits × component)).
17108    pub fn dirty_propagation_visits(&self) -> u64 {
17109        self.graph.dirty_propagation_visits()
17110    }
17111
17112    /// Evaluate a single vertex.
17113    /// This is the core of the sequential evaluation logic for Milestone 3.1.
17114    #[inline]
17115    fn record_cell_if_changed(
17116        delta: &mut DeltaCollector,
17117        cell: &CellRef,
17118        old: &LiteralValue,
17119        new: &LiteralValue,
17120    ) {
17121        if old != new {
17122            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
17123        }
17124    }
17125
17126    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
17127        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
17128            engine.observe_function_semantic_epoch()?;
17129            // A direct request selects exactly one vertex, regardless of its formula kind.
17130            engine.resource_checkpoint(1)?;
17131            let is_formula = engine.graph.vertex_exists(vertex_id)
17132                && matches!(
17133                    engine.graph.get_vertex_kind(vertex_id),
17134                    VertexKind::FormulaScalar | VertexKind::FormulaArray
17135                );
17136            if is_formula {
17137                engine.begin_evaluation_request();
17138                engine.graph.flush_pending_edge_deltas();
17139                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
17140                    vertex_id,
17141                )];
17142                if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17143                    && engine.graph.formula_authority().active_span_count() > 0
17144                {
17145                    engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
17146                } else {
17147                    engine.evaluate_legacy_target_roots(&roots, None)?;
17148                }
17149            } else if engine.config.formula_plane_mode
17150                == FormulaPlaneMode::AuthoritativeExperimental
17151                && engine.graph.formula_authority().active_span_count() > 0
17152            {
17153                engine.begin_evaluation_request();
17154                engine.graph.flush_pending_edge_deltas();
17155                engine.evaluate_authoritative_formula_plane(None, None)?;
17156            }
17157            engine.evaluate_vertex_impl(vertex_id, None)
17158        })
17159    }
17160
17161    fn evaluate_vertex_impl(
17162        &mut self,
17163        vertex_id: VertexId,
17164        delta: Option<&mut DeltaCollector>,
17165    ) -> Result<LiteralValue, ExcelError> {
17166        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
17167        // names, and other non-formula vertices. Only formula publication needs the C1a final
17168        // deadline checkpoint and effects pipeline.
17169        if !self.graph.vertex_exists(vertex_id) {
17170            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
17171                .with_message(format!("Vertex not found: {vertex_id:?}")));
17172        }
17173        if self.active_resource_ledger.is_some()
17174            && matches!(
17175                self.graph.get_vertex_kind(vertex_id),
17176                VertexKind::FormulaScalar | VertexKind::FormulaArray
17177            )
17178        {
17179            let value = self
17180                .evaluate_vertex_immutable(vertex_id)
17181                .unwrap_or_else(LiteralValue::Error);
17182            let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
17183            // Do not publish the selected result until the outer request's deadline succeeds.
17184            self.resource_checkpoint(0)?;
17185            let mut delta = delta;
17186            for effect in &effects {
17187                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
17188            }
17189            return Ok(value);
17190        }
17191
17192        let mut delta = delta;
17193
17194        // Get vertex kind and check if it needs evaluation
17195        let kind = self.graph.get_vertex_kind(vertex_id);
17196        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
17197
17198        let ast_id = match kind {
17199            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
17200                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
17201                    ast_id
17202                } else {
17203                    return Ok(LiteralValue::Number(0.0));
17204                }
17205            }
17206            VertexKind::Empty | VertexKind::Cell => {
17207                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
17208                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17209                    let row = cell_ref.coord.row() + 1;
17210                    let col = cell_ref.coord.col() + 1;
17211                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
17212                        return Ok(v);
17213                    }
17214                }
17215                return Ok(LiteralValue::Number(0.0));
17216            }
17217            VertexKind::NamedScalar => {
17218                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
17219                return Ok(value);
17220            }
17221            VertexKind::NamedArray => {
17222                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
17223                return Ok(value);
17224            }
17225            VertexKind::InfiniteRange
17226            | VertexKind::Range
17227            | VertexKind::External
17228            | VertexKind::Table => {
17229                // Not directly evaluatable here.
17230                return Ok(LiteralValue::Number(0.0));
17231            }
17232        };
17233
17234        // The interpreter uses a reference to the engine as the context.
17235        let sheet_name = self.graph.sheet_name(sheet_id);
17236        let cell_ref = self
17237            .graph
17238            .get_cell_ref(vertex_id)
17239            .expect("cell ref for vertex");
17240        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17241
17242        let result =
17243            interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
17244
17245        // If array result, perform spill from the anchor cell
17246        match result {
17247            Ok(cv) => {
17248                let result_literal =
17249                    crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
17250                match result_literal {
17251                    LiteralValue::Array(rows) => {
17252                        // Update kind to FormulaArray for tracking
17253                        self.graph
17254                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
17255                        // Build target cells rectangle starting from anchor
17256                        let anchor = self
17257                            .graph
17258                            .get_cell_ref(vertex_id)
17259                            .expect("cell ref for vertex");
17260                        let sheet_id = anchor.sheet_id;
17261                        let h = rows.len() as u32;
17262                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
17263
17264                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
17265                        let spill_cells = (h as u64).saturating_mul(w as u64);
17266                        if spill_cells > self.config.spill.max_spill_cells as u64 {
17267                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17268                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17269                                .with_message("SpillTooLarge")
17270                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17271                                    expected_rows: h,
17272                                    expected_cols: w,
17273                                });
17274                            let spill_val = LiteralValue::Error(spill_err.clone());
17275                            if let Some(d) = delta.as_deref_mut() {
17276                                let old = self
17277                                    .read_cell_value(
17278                                        self.graph.sheet_name(anchor.sheet_id),
17279                                        anchor.coord.row() + 1,
17280                                        anchor.coord.col() + 1,
17281                                    )
17282                                    .unwrap_or(LiteralValue::Empty);
17283                                if old != spill_val {
17284                                    d.record_cell(
17285                                        anchor.sheet_id,
17286                                        anchor.coord.row(),
17287                                        anchor.coord.col(),
17288                                    );
17289                                }
17290                            }
17291                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
17292                            if self.config.arrow_storage_enabled
17293                                && self.config.delta_overlay_enabled
17294                                && self.config.write_formula_overlay_enabled
17295                            {
17296                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17297                                self.mirror_value_to_computed_overlay(
17298                                    &sheet_name,
17299                                    anchor.coord.row() + 1,
17300                                    anchor.coord.col() + 1,
17301                                    &spill_val,
17302                                );
17303                            }
17304                            return Ok(spill_val);
17305                        }
17306                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
17307                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
17308                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
17309                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
17310                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
17311                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
17312                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
17313                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17314                                .with_message("Spill exceeds sheet bounds")
17315                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17316                                    expected_rows: h,
17317                                    expected_cols: w,
17318                                });
17319                            let spill_val = LiteralValue::Error(spill_err.clone());
17320                            if let Some(d) = delta.as_deref_mut() {
17321                                let old = self
17322                                    .read_cell_value(
17323                                        self.graph.sheet_name(anchor.sheet_id),
17324                                        anchor.coord.row() + 1,
17325                                        anchor.coord.col() + 1,
17326                                    )
17327                                    .unwrap_or(LiteralValue::Empty);
17328                                if old != spill_val {
17329                                    d.record_cell(
17330                                        anchor.sheet_id,
17331                                        anchor.coord.row(),
17332                                        anchor.coord.col(),
17333                                    );
17334                                }
17335                            }
17336                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
17337                            if self.config.arrow_storage_enabled
17338                                && self.config.delta_overlay_enabled
17339                                && self.config.write_formula_overlay_enabled
17340                            {
17341                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17342                                self.mirror_value_to_computed_overlay(
17343                                    &sheet_name,
17344                                    anchor.coord.row() + 1,
17345                                    anchor.coord.col() + 1,
17346                                    &spill_val,
17347                                );
17348                            }
17349                            return Ok(spill_val);
17350                        }
17351                        let mut targets = Vec::new();
17352                        for r in 0..h {
17353                            for c in 0..w {
17354                                targets.push(self.graph.make_cell_ref_internal(
17355                                    sheet_id,
17356                                    anchor.coord.row() + r,
17357                                    anchor.coord.col() + c,
17358                                ));
17359                            }
17360                        }
17361
17362                        // Plan spill via spill manager shim
17363                        match self.spill_mgr.reserve(
17364                            vertex_id,
17365                            anchor,
17366                            SpillShape { rows: h, cols: w },
17367                            SpillMeta {
17368                                epoch: self.recalc_epoch,
17369                                config: self.config.spill,
17370                            },
17371                        ) {
17372                            Ok(()) => {
17373                                // Commit: write values to grid
17374                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
17375                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
17376                                if let Err(e) = self.commit_spill_and_mirror(
17377                                    vertex_id,
17378                                    &targets,
17379                                    rows.clone(),
17380                                    delta.as_deref_mut(),
17381                                    None,
17382                                ) {
17383                                    // If commit fails, mark as error
17384                                    self.clear_spill_projection_and_mirror(
17385                                        vertex_id,
17386                                        delta.as_deref_mut(),
17387                                    );
17388                                    if let Some(d) = delta.as_deref_mut() {
17389                                        let old = self
17390                                            .read_cell_value(
17391                                                self.graph.sheet_name(anchor.sheet_id),
17392                                                anchor.coord.row() + 1,
17393                                                anchor.coord.col() + 1,
17394                                            )
17395                                            .unwrap_or(LiteralValue::Empty);
17396                                        let new = LiteralValue::Error(e.clone());
17397                                        if old != new {
17398                                            d.record_cell(
17399                                                anchor.sheet_id,
17400                                                anchor.coord.row(),
17401                                                anchor.coord.col(),
17402                                            );
17403                                        }
17404                                    }
17405                                    let err_val = LiteralValue::Error(e.clone());
17406                                    self.graph.update_vertex_value(vertex_id, err_val.clone());
17407                                    if self.config.arrow_storage_enabled
17408                                        && self.config.delta_overlay_enabled
17409                                        && self.config.write_formula_overlay_enabled
17410                                    {
17411                                        let sheet_name =
17412                                            self.graph.sheet_name(anchor.sheet_id).to_string();
17413                                        self.mirror_value_to_computed_overlay(
17414                                            &sheet_name,
17415                                            anchor.coord.row() + 1,
17416                                            anchor.coord.col() + 1,
17417                                            &err_val,
17418                                        );
17419                                    }
17420                                    return Ok(err_val);
17421                                }
17422                                // Anchor shows the top-left value, like Excel
17423                                let top_left = rows
17424                                    .first()
17425                                    .and_then(|r| r.first())
17426                                    .cloned()
17427                                    .unwrap_or(LiteralValue::Empty);
17428                                self.graph.update_vertex_value(vertex_id, top_left.clone());
17429                                Ok(top_left)
17430                            }
17431                            Err(e) => {
17432                                self.clear_spill_projection_and_mirror(
17433                                    vertex_id,
17434                                    delta.as_deref_mut(),
17435                                );
17436                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
17437                                    .with_message(
17438                                        e.message.unwrap_or_else(|| "Spill blocked".to_string()),
17439                                    )
17440                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
17441                                        expected_rows: h,
17442                                        expected_cols: w,
17443                                    });
17444                                let spill_val = LiteralValue::Error(spill_err.clone());
17445                                if let Some(d) = delta.as_deref_mut() {
17446                                    let old = self
17447                                        .read_cell_value(
17448                                            self.graph.sheet_name(anchor.sheet_id),
17449                                            anchor.coord.row() + 1,
17450                                            anchor.coord.col() + 1,
17451                                        )
17452                                        .unwrap_or(LiteralValue::Empty);
17453                                    if old != spill_val {
17454                                        d.record_cell(
17455                                            anchor.sheet_id,
17456                                            anchor.coord.row(),
17457                                            anchor.coord.col(),
17458                                        );
17459                                    }
17460                                }
17461                                self.graph.update_vertex_value(vertex_id, spill_val.clone());
17462                                if self.config.arrow_storage_enabled
17463                                    && self.config.delta_overlay_enabled
17464                                    && self.config.write_formula_overlay_enabled
17465                                {
17466                                    let sheet_name =
17467                                        self.graph.sheet_name(anchor.sheet_id).to_string();
17468                                    self.mirror_value_to_computed_overlay(
17469                                        &sheet_name,
17470                                        anchor.coord.row() + 1,
17471                                        anchor.coord.col() + 1,
17472                                        &spill_val,
17473                                    );
17474                                }
17475                                Ok(spill_val)
17476                            }
17477                        }
17478                    }
17479                    other => {
17480                        // Scalar result: store value and ensure any previous spill is cleared
17481                        let spill_cells = self
17482                            .graph
17483                            .spill_cells_for_anchor(vertex_id)
17484                            .map(|cells| cells.to_vec())
17485                            .unwrap_or_default();
17486                        if let Some(d) = delta.as_deref_mut()
17487                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17488                        {
17489                            if spill_cells.is_empty() {
17490                                let old = self
17491                                    .read_cell_value(
17492                                        self.graph.sheet_name(anchor.sheet_id),
17493                                        anchor.coord.row() + 1,
17494                                        anchor.coord.col() + 1,
17495                                    )
17496                                    .unwrap_or(LiteralValue::Empty);
17497                                if old != other {
17498                                    d.record_cell(
17499                                        anchor.sheet_id,
17500                                        anchor.coord.row(),
17501                                        anchor.coord.col(),
17502                                    );
17503                                }
17504                            } else {
17505                                for cell in spill_cells.iter() {
17506                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
17507                                    let old = self
17508                                        .get_cell_value(
17509                                            sheet_name,
17510                                            cell.coord.row() + 1,
17511                                            cell.coord.col() + 1,
17512                                        )
17513                                        .unwrap_or(LiteralValue::Empty);
17514                                    let new = if cell.sheet_id == anchor.sheet_id
17515                                        && cell.coord.row() == anchor.coord.row()
17516                                        && cell.coord.col() == anchor.coord.col()
17517                                    {
17518                                        other.clone()
17519                                    } else {
17520                                        LiteralValue::Empty
17521                                    };
17522                                    Self::record_cell_if_changed(d, cell, &old, &new);
17523                                }
17524                            }
17525                        }
17526                        self.graph.clear_spill_region(vertex_id);
17527                        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17528                            self.record_formula_plane_structural_change(scope);
17529                        }
17530                        if self.config.arrow_storage_enabled
17531                            && self.config.delta_overlay_enabled
17532                            && self.config.write_formula_overlay_enabled
17533                        {
17534                            let empty = LiteralValue::Empty;
17535                            for cell in spill_cells.iter() {
17536                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17537                                self.mirror_value_to_computed_overlay(
17538                                    &sheet_name,
17539                                    cell.coord.row() + 1,
17540                                    cell.coord.col() + 1,
17541                                    &empty,
17542                                );
17543                            }
17544                        }
17545                        self.graph.update_vertex_value(vertex_id, other.clone());
17546                        // Optionally mirror into Arrow overlay for Arrow-backed reads
17547                        if self.config.arrow_storage_enabled
17548                            && self.config.delta_overlay_enabled
17549                            && self.config.write_formula_overlay_enabled
17550                        {
17551                            let anchor = self
17552                                .graph
17553                                .get_cell_ref(vertex_id)
17554                                .expect("cell ref for vertex");
17555                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17556                            self.mirror_value_to_computed_overlay(
17557                                &sheet_name,
17558                                anchor.coord.row() + 1,
17559                                anchor.coord.col() + 1,
17560                                &other,
17561                            );
17562                        }
17563                        Ok(other)
17564                    }
17565                }
17566            }
17567            Err(e) => {
17568                // Runtime Excel error: store as a cell value instead of propagating
17569                // as an exception so bulk eval paths don't fail the whole pass.
17570                let spill_cells = self
17571                    .graph
17572                    .spill_cells_for_anchor(vertex_id)
17573                    .map(|cells| cells.to_vec())
17574                    .unwrap_or_default();
17575                let err_val = LiteralValue::Error(e.clone());
17576                if let Some(d) = delta
17577                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
17578                {
17579                    if spill_cells.is_empty() {
17580                        let old = self
17581                            .read_cell_value(
17582                                self.graph.sheet_name(anchor.sheet_id),
17583                                anchor.coord.row() + 1,
17584                                anchor.coord.col() + 1,
17585                            )
17586                            .unwrap_or(LiteralValue::Empty);
17587                        if old != err_val {
17588                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
17589                        }
17590                    } else {
17591                        for cell in spill_cells.iter() {
17592                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
17593                            let old = self
17594                                .get_cell_value(
17595                                    sheet_name,
17596                                    cell.coord.row() + 1,
17597                                    cell.coord.col() + 1,
17598                                )
17599                                .unwrap_or(LiteralValue::Empty);
17600                            let new = if cell.sheet_id == anchor.sheet_id
17601                                && cell.coord.row() == anchor.coord.row()
17602                                && cell.coord.col() == anchor.coord.col()
17603                            {
17604                                err_val.clone()
17605                            } else {
17606                                LiteralValue::Empty
17607                            };
17608                            Self::record_cell_if_changed(d, cell, &old, &new);
17609                        }
17610                    }
17611                }
17612                self.graph.clear_spill_region(vertex_id);
17613                if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
17614                    self.record_formula_plane_structural_change(scope);
17615                }
17616                if self.config.arrow_storage_enabled
17617                    && self.config.delta_overlay_enabled
17618                    && self.config.write_formula_overlay_enabled
17619                {
17620                    let empty = LiteralValue::Empty;
17621                    for cell in spill_cells.iter() {
17622                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17623                        self.mirror_value_to_computed_overlay(
17624                            &sheet_name,
17625                            cell.coord.row() + 1,
17626                            cell.coord.col() + 1,
17627                            &empty,
17628                        );
17629                    }
17630                }
17631                self.graph.update_vertex_value(vertex_id, err_val.clone());
17632                if self.config.arrow_storage_enabled
17633                    && self.config.delta_overlay_enabled
17634                    && self.config.write_formula_overlay_enabled
17635                {
17636                    let anchor = self
17637                        .graph
17638                        .get_cell_ref(vertex_id)
17639                        .expect("cell ref for vertex");
17640                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
17641                    self.mirror_value_to_computed_overlay(
17642                        &sheet_name,
17643                        anchor.coord.row() + 1,
17644                        anchor.coord.col() + 1,
17645                        &err_val,
17646                    );
17647                }
17648                Ok(err_val)
17649            }
17650        }
17651    }
17652
17653    fn evaluate_named_scalar(
17654        &mut self,
17655        vertex_id: VertexId,
17656        sheet_id: SheetId,
17657    ) -> Result<LiteralValue, ExcelError> {
17658        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17659            ExcelError::new(ExcelErrorKind::Name)
17660                .with_message("Named range metadata missing".to_string())
17661        })?;
17662
17663        match &named_range.definition {
17664            NamedDefinition::Cell(cell_ref) => {
17665                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17666                let row = cell_ref.coord.row() + 1;
17667                let col = cell_ref.coord.col() + 1;
17668
17669                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
17670                    && matches!(
17671                        self.graph.get_vertex_kind(dep_vertex),
17672                        VertexKind::FormulaScalar | VertexKind::FormulaArray
17673                    )
17674                {
17675                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
17676                    let value = self.evaluate_vertex(dep_vertex)?;
17677                    self.graph.update_vertex_value(vertex_id, value.clone());
17678                    Ok(value)
17679                } else {
17680                    let value = self
17681                        .get_cell_value(sheet_name, row, col)
17682                        .unwrap_or(LiteralValue::Empty);
17683                    self.graph.update_vertex_value(vertex_id, value.clone());
17684                    Ok(value)
17685                }
17686            }
17687            NamedDefinition::Literal(v) => {
17688                let out = v.clone();
17689                self.graph.update_vertex_value(vertex_id, out.clone());
17690                Ok(out)
17691            }
17692            NamedDefinition::Formula { ast, .. } => {
17693                let context_sheet = match named_range.scope {
17694                    NameScope::Sheet(id) => id,
17695                    NameScope::Workbook => sheet_id,
17696                };
17697                let sheet_name = self.graph.sheet_name(context_sheet);
17698                let cell_ref = self
17699                    .graph
17700                    .get_cell_ref(vertex_id)
17701                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17702                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17703                match interpreter.evaluate_ast(ast) {
17704                    Ok(cv) => {
17705                        let value = cv.into_literal();
17706                        match value {
17707                            LiteralValue::Array(_) => {
17708                                let err = ExcelError::new(ExcelErrorKind::NImpl)
17709                                    .with_message("Array result in scalar named range".to_string());
17710                                let err_val = LiteralValue::Error(err.clone());
17711                                self.graph.update_vertex_value(vertex_id, err_val.clone());
17712                                Ok(err_val)
17713                            }
17714                            other => {
17715                                self.graph.update_vertex_value(vertex_id, other.clone());
17716                                Ok(other)
17717                            }
17718                        }
17719                    }
17720                    Err(err) => {
17721                        let err_val = LiteralValue::Error(err.clone());
17722                        self.graph.update_vertex_value(vertex_id, err_val.clone());
17723                        Ok(err_val)
17724                    }
17725                }
17726            }
17727            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
17728                .with_message("Range-valued name evaluated as scalar".to_string())),
17729        }
17730    }
17731
17732    fn evaluate_named_array(
17733        &mut self,
17734        vertex_id: VertexId,
17735        sheet_id: SheetId,
17736    ) -> Result<LiteralValue, ExcelError> {
17737        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
17738            ExcelError::new(ExcelErrorKind::Name)
17739                .with_message("Named range metadata missing".to_string())
17740        })?;
17741
17742        let out = match &named_range.definition {
17743            NamedDefinition::Range(range_ref) => {
17744                if range_ref.start.sheet_id != range_ref.end.sheet_id {
17745                    return Err(ExcelError::new(ExcelErrorKind::Ref)
17746                        .with_message("Named range cannot span sheets".to_string()));
17747                }
17748
17749                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
17750                let sr0 = range_ref.start.coord.row();
17751                let sc0 = range_ref.start.coord.col();
17752                let er0 = range_ref.end.coord.row();
17753                let ec0 = range_ref.end.coord.col();
17754                if sr0 > er0 || sc0 > ec0 {
17755                    return Err(ExcelError::new(ExcelErrorKind::Ref)
17756                        .with_message("Invalid named range bounds".to_string()));
17757                }
17758
17759                let h = (er0 - sr0 + 1) as usize;
17760                let w = (ec0 - sc0 + 1) as usize;
17761                let cell_count = (h as u64).saturating_mul(w as u64);
17762                if cell_count > self.config.spill.max_spill_cells as u64 {
17763                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
17764                        "Named range too large to materialize as an array".to_string(),
17765                    ));
17766                }
17767
17768                let mut rows = Vec::with_capacity(h);
17769                for r0 in sr0..=er0 {
17770                    let mut row = Vec::with_capacity(w);
17771                    for c0 in sc0..=ec0 {
17772                        let v = self
17773                            .get_cell_value(sheet_name, r0 + 1, c0 + 1)
17774                            .unwrap_or(LiteralValue::Empty);
17775                        row.push(v);
17776                    }
17777                    rows.push(row);
17778                }
17779                LiteralValue::Array(rows)
17780            }
17781            NamedDefinition::Cell(cell_ref) => {
17782                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
17783                let row = cell_ref.coord.row() + 1;
17784                let col = cell_ref.coord.col() + 1;
17785                let v = self
17786                    .get_cell_value(sheet_name, row, col)
17787                    .unwrap_or(LiteralValue::Empty);
17788                LiteralValue::Array(vec![vec![v]])
17789            }
17790            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
17791            NamedDefinition::Formula { ast, .. } => {
17792                let context_sheet = match named_range.scope {
17793                    NameScope::Sheet(id) => id,
17794                    NameScope::Workbook => sheet_id,
17795                };
17796                let sheet_name = self.graph.sheet_name(context_sheet);
17797                let cell_ref = self
17798                    .graph
17799                    .get_cell_ref(vertex_id)
17800                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
17801                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
17802                match interpreter.evaluate_ast(ast) {
17803                    Ok(cv) => {
17804                        let v = cv.into_literal();
17805                        match v {
17806                            LiteralValue::Array(_) => v,
17807                            other => LiteralValue::Array(vec![vec![other]]),
17808                        }
17809                    }
17810                    Err(err) => LiteralValue::Error(err),
17811                }
17812            }
17813        };
17814
17815        self.graph.update_vertex_value(vertex_id, out.clone());
17816        Ok(out)
17817    }
17818
17819    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
17820        crate::engine::ResourceLedgerError::Exhausted(
17821            formualizer_common::ResourceExhaustionDetail {
17822                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
17823                limit: limit as u64,
17824                observed: limit.saturating_add(1) as u64,
17825                request_id: self
17826                    .active_evaluation_resource_request
17827                    .as_ref()
17828                    .map(|stats| stats.request_id),
17829            },
17830        )
17831        .into_excel_error()
17832        .with_message(format!("{context} did not converge after {limit} replans"))
17833    }
17834
17835    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
17836        matches!(
17837            &error.extra,
17838            formualizer_common::ExcelErrorExtra::PreparationStale {
17839                reason: formualizer_common::PreparationStaleReason::Semantic
17840                    | formualizer_common::PreparationStaleReason::Provider
17841            }
17842        )
17843    }
17844
17845    fn prepare_graph_for_routed_evaluation(
17846        &mut self,
17847        targets: &[crate::engine::EvaluationTarget],
17848        options: &crate::engine::TargetEvalOptions<'_>,
17849    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
17850        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
17851        let mut retries = 0usize;
17852        loop {
17853            match self.prepare_graph_for_targets_unobserved(targets, options) {
17854                Err(error)
17855                    if Self::transient_target_preparation_stale(&error)
17856                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
17857                {
17858                    retries = retries.saturating_add(1);
17859                }
17860                result => return result,
17861            }
17862        }
17863    }
17864
17865    fn evaluate_mixed_targets(
17866        &mut self,
17867        targets: &[crate::engine::EvaluationTarget],
17868        delta: Option<&mut DeltaCollector>,
17869    ) -> Result<EvalResult, ExcelError> {
17870        let _source_cache = self.source_cache_session();
17871        let cancel = self.active_cancel_flag.clone();
17872        let options = crate::engine::TargetEvalOptions {
17873            request_id: self
17874                .active_evaluation_resource_request
17875                .as_ref()
17876                .map(|stats| stats.request_id),
17877            cancel,
17878            deadline: None,
17879            budgets: None,
17880            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
17881        };
17882        self.prepare_and_execute_target_recipe(targets, &options, delta)
17883    }
17884
17885    fn prepare_and_execute_target_recipe(
17886        &mut self,
17887        targets: &[crate::engine::EvaluationTarget],
17888        options: &crate::engine::TargetEvalOptions<'_>,
17889        delta: Option<&mut DeltaCollector>,
17890    ) -> Result<EvalResult, ExcelError> {
17891        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
17892        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
17893    }
17894
17895    fn execute_prepared_target_recipe(
17896        &mut self,
17897        targets: &[crate::engine::EvaluationTarget],
17898        scope: &crate::engine::PrepareScope,
17899        delta: Option<&mut DeltaCollector>,
17900    ) -> Result<EvalResult, ExcelError> {
17901        if matches!(scope, crate::engine::PrepareScope::Workbook)
17902            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
17903        {
17904            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
17905        }
17906        let mut roots = self.resolve_target_producers(targets)?;
17907        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
17908            let request_id = self
17909                .active_evaluation_resource_request
17910                .as_ref()
17911                .map(|request| request.request_id);
17912            let root_count = roots.len();
17913            let mut widened_roots =
17914                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
17915                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
17916            let sheet_ids = sheets
17917                .iter()
17918                .filter_map(|sheet| self.graph.sheet_id(sheet))
17919                .collect::<FxHashSet<_>>();
17920            for vertex in self.graph.formula_vertices() {
17921                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
17922                    widened_roots
17923                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
17924                            vertex,
17925                        ))
17926                        .map_err(|_| {
17927                            target_root_allocation_error(widened_roots.len() + 1, request_id)
17928                        })?;
17929                }
17930            }
17931            let authority = self.graph.formula_authority();
17932            for span_ref in authority.active_span_refs() {
17933                let Some(span) = authority.plane.spans.get(span_ref) else {
17934                    continue;
17935                };
17936                if sheet_ids.contains(&span.sheet_id) {
17937                    widened_roots
17938                        .push(crate::engine::target_preparation::TargetProducer::Span {
17939                            span_ref,
17940                            demanded: Region::from_domain(span.result_region.domain()),
17941                        })
17942                        .map_err(|_| {
17943                            target_root_allocation_error(widened_roots.len() + 1, request_id)
17944                        })?;
17945                }
17946            }
17947            roots = widened_roots.into_vec();
17948        }
17949        self.begin_evaluation_request();
17950        self.graph.flush_pending_edge_deltas();
17951        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
17952        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
17953            && self.graph.formula_authority().active_span_count() > 0
17954        {
17955            if workbook_scope {
17956                self.evaluate_authoritative_formula_plane(None, delta)
17957            } else {
17958                self.evaluate_authoritative_formula_plane_targets(&roots, delta)
17959            }
17960        } else {
17961            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
17962                self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
17963            }
17964            if workbook_scope {
17965                if let Some(delta) = delta {
17966                    self.evaluate_all_with_delta_collector(delta)
17967                } else {
17968                    self.evaluate_all_legacy_impl()
17969                }
17970            } else {
17971                self.evaluate_legacy_target_roots(&roots, delta)
17972            }
17973        }
17974    }
17975
17976    fn legacy_coordinate_targets(
17977        &mut self,
17978        targets: &[(&str, u32, u32)],
17979    ) -> Vec<crate::engine::EvaluationTarget> {
17980        targets
17981            .iter()
17982            .map(|(sheet, row, col)| {
17983                // Compatibility APIs historically interned an unknown target sheet
17984                // and returned an empty value rather than rejecting the target.
17985                self.graph.sheet_id_mut(sheet);
17986                crate::engine::EvaluationTarget::Cell {
17987                    sheet: (*sheet).to_string(),
17988                    row: *row,
17989                    col: *col,
17990                }
17991            })
17992            .collect()
17993    }
17994
17995    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
17996    pub fn evaluate_targets(
17997        &mut self,
17998        targets: &[crate::engine::EvaluationTarget],
17999    ) -> Result<EvalResult, ExcelError> {
18000        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18001            engine.observe_function_semantic_epoch()?;
18002            engine.validate_deterministic_mode()?;
18003            engine.evaluate_mixed_targets(targets, None)
18004        })
18005    }
18006
18007    /// Evaluate typed targets with explicit preparation policy and request controls.
18008    pub fn evaluate_targets_with_options(
18009        &mut self,
18010        targets: &[crate::engine::EvaluationTarget],
18011        options: crate::engine::TargetEvalOptions<'_>,
18012    ) -> Result<EvalResult, ExcelError> {
18013        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18014            engine.active_cancel_flag = options.cancel.clone();
18015            engine.active_evaluation_deadline = options.deadline;
18016            let result = (|| {
18017                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
18018                engine.observe_function_semantic_epoch()?;
18019                engine.validate_deterministic_mode()?;
18020                let _source_cache = engine.source_cache_session();
18021                engine.prepare_and_execute_target_recipe(targets, &options, None)
18022            })();
18023            engine.active_cancel_flag = None;
18024            engine.active_evaluation_deadline = None;
18025            result
18026        })
18027    }
18028
18029    /// Evaluate typed targets and return the versioned run/region delta for the request.
18030    pub fn evaluate_targets_with_delta(
18031        &mut self,
18032        targets: &[crate::engine::EvaluationTarget],
18033    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
18034        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
18035            engine.observe_function_semantic_epoch()?;
18036            engine.validate_deterministic_mode()?;
18037            let mut collector = DeltaCollector::new(DeltaMode::Cells);
18038            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
18039            Ok((result, collector.finish_target()))
18040        })
18041    }
18042
18043    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
18044    pub fn evaluate_until(
18045        &mut self,
18046        targets: &[(&str, u32, u32)],
18047    ) -> Result<EvalResult, ExcelError> {
18048        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
18049            engine.evaluate_until_unobserved(targets)
18050        })
18051    }
18052
18053    fn evaluate_until_unobserved(
18054        &mut self,
18055        targets: &[(&str, u32, u32)],
18056    ) -> Result<EvalResult, ExcelError> {
18057        self.observe_function_semantic_epoch()?;
18058        let targets = self.legacy_coordinate_targets(targets);
18059        self.evaluate_mixed_targets(&targets, None)
18060    }
18061
18062    fn evaluate_until_with_delta_collector(
18063        &mut self,
18064        targets: &[(&str, u32, u32)],
18065        delta: &mut DeltaCollector,
18066    ) -> Result<EvalResult, ExcelError> {
18067        let targets = self.legacy_coordinate_targets(targets);
18068        self.evaluate_mixed_targets(&targets, Some(delta))
18069    }
18070
18071    fn evaluate_legacy_target_roots(
18072        &mut self,
18073        roots: &[crate::engine::target_preparation::TargetProducer],
18074        mut delta: Option<&mut DeltaCollector>,
18075    ) -> Result<EvalResult, ExcelError> {
18076        use crate::engine::target_preparation::TargetProducer;
18077        let start = crate::instant::FzInstant::now();
18078        let root_vertices = roots
18079            .iter()
18080            .filter_map(|root| match root {
18081                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
18082                TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
18083            })
18084            .collect::<Vec<_>>();
18085        let mut computed_vertices = 0usize;
18086        let mut cycle_errors = 0usize;
18087        let mut replans = 0usize;
18088        const MAX_REPLAN: usize = 5;
18089        loop {
18090            let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
18091            if precedents_to_eval.is_empty() {
18092                break;
18093            }
18094            let scheduler = Scheduler::new(&self.graph);
18095            let schedule =
18096                scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
18097            for &unit in &schedule.units {
18098                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
18099                match unit {
18100                    ScheduleUnit::Cycle(index) => {
18101                        if self.handle_cycle_unit(
18102                            schedule.unit_cycle(index),
18103                            delta.as_deref_mut(),
18104                            None,
18105                            None,
18106                        )? > 0
18107                        {
18108                            cycle_errors = cycle_errors.saturating_add(1);
18109                        }
18110                    }
18111                    ScheduleUnit::Layer(index) => {
18112                        let layer = schedule.unit_layer(index);
18113                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
18114                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18115                                self.evaluate_layer_parallel_with_delta(layer, delta)?
18116                            } else {
18117                                self.evaluate_layer_sequential_with_delta(layer, delta)?
18118                            }
18119                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
18120                            self.evaluate_layer_parallel(layer)?
18121                        } else {
18122                            self.evaluate_layer_sequential(layer)?
18123                        };
18124                        computed_vertices = computed_vertices.saturating_add(evaluated);
18125                    }
18126                }
18127            }
18128            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
18129            self.resource_checkpoint(0)?;
18130            self.graph.clear_dirty_flags(&precedents_to_eval);
18131            for vertex in &changed {
18132                self.graph.set_dirty(*vertex, true);
18133            }
18134            if changed.is_empty() {
18135                break;
18136            }
18137            if replans >= MAX_REPLAN {
18138                return Err(self.replan_exhausted_error(
18139                    MAX_REPLAN,
18140                    "targeted legacy dynamic dependency evaluation",
18141                ));
18142            }
18143            replans = replans.saturating_add(1);
18144        }
18145        self.redirty_for_next_recalc();
18146        Ok(EvalResult {
18147            computed_vertices,
18148            cycle_errors,
18149            elapsed: start.elapsed(),
18150        })
18151    }
18152
18153    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
18154    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
18155        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
18156            return Err(
18157                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
18158                    "compatibility recalculation plans require all staged formulas to be prepared",
18159                ),
18160            );
18161        }
18162        let key = self.recalc_plan_key();
18163        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
18164        vertices.sort_unstable();
18165        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
18166        let schedule = if vertices.is_empty() {
18167            crate::engine::Schedule {
18168                units: Vec::new(),
18169                layers: Vec::new(),
18170                cycles: Vec::new(),
18171            }
18172        } else {
18173            self.create_evaluation_schedule_uncached(&vertices)?.0
18174        };
18175        self.validate_recalc_plan_key(&key)?;
18176        Ok(RecalcPlan {
18177            key,
18178            kind: RecalcPlanKind::CompatibilityFull {
18179                schedule,
18180                has_dynamic_refs,
18181            },
18182        })
18183    }
18184
18185    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
18186    pub fn build_recalc_plan_for_targets(
18187        &mut self,
18188        targets: &[crate::engine::EvaluationTarget],
18189    ) -> Result<RecalcPlan, ExcelError> {
18190        self.build_recalc_plan_for_targets_with_options(
18191            targets,
18192            crate::engine::TargetEvalOptions::default(),
18193        )
18194    }
18195
18196    pub fn build_recalc_plan_for_targets_with_options(
18197        &mut self,
18198        targets: &[crate::engine::EvaluationTarget],
18199        options: crate::engine::TargetEvalOptions<'_>,
18200    ) -> Result<RecalcPlan, ExcelError> {
18201        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18202            engine.observe_function_semantic_epoch()?;
18203            engine.validate_deterministic_mode()?;
18204            let _source_cache = engine.source_cache_session();
18205            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
18206            engine.graph.flush_pending_edge_deltas();
18207            let topology = if matches!(
18208                preparation.widened_scope,
18209                crate::engine::PrepareScope::Workbook
18210            ) {
18211                RecalcTopology::Workbook
18212            } else {
18213                RecalcTopology::RunLocalRecipe
18214            };
18215            Ok(RecalcPlan {
18216                key: engine.recalc_plan_key(),
18217                kind: RecalcPlanKind::Target {
18218                    targets: targets.to_vec(),
18219                    scope: preparation.widened_scope,
18220                    topology,
18221                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
18222                },
18223            })
18224        })
18225    }
18226
18227    /// Evaluate using a previously constructed compatibility or target plan.
18228    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
18229        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18230            engine.evaluate_recalc_plan_unobserved(plan)
18231        })
18232    }
18233
18234    pub fn evaluate_recalc_plan_with_controls(
18235        &mut self,
18236        plan: &RecalcPlan,
18237        cancel: Option<crate::engine::CancelToken>,
18238        deadline: Option<Instant>,
18239    ) -> Result<EvalResult, ExcelError> {
18240        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
18241            engine.active_cancel_flag = cancel.clone();
18242            engine.active_evaluation_deadline = deadline;
18243            let result = engine.evaluate_recalc_plan_unobserved(plan);
18244            engine.active_cancel_flag = None;
18245            engine.active_evaluation_deadline = None;
18246            result
18247        })
18248    }
18249
18250    fn evaluate_recalc_plan_unobserved(
18251        &mut self,
18252        plan: &RecalcPlan,
18253    ) -> Result<EvalResult, ExcelError> {
18254        self.graph.flush_pending_edge_deltas();
18255        self.validate_recalc_plan_key(&plan.key)?;
18256        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
18257        self.validate_deterministic_mode()?;
18258
18259        match &plan.kind {
18260            RecalcPlanKind::Target {
18261                targets,
18262                scope,
18263                topology,
18264                dynamic_policy,
18265            } => {
18266                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
18267                debug_assert_eq!(
18268                    matches!(topology, RecalcTopology::Workbook),
18269                    matches!(scope, crate::engine::PrepareScope::Workbook)
18270                );
18271                let _source_cache = self.source_cache_session();
18272                self.execute_prepared_target_recipe(targets, scope, None)
18273            }
18274            RecalcPlanKind::CompatibilityFull {
18275                schedule,
18276                has_dynamic_refs,
18277            } => {
18278                let _source_cache = self.source_cache_session();
18279                self.begin_evaluation_request();
18280                if self.graph.formula_authority().active_span_count() > 0 {
18281                    return self.evaluate_authoritative_formula_plane_all();
18282                }
18283                if *has_dynamic_refs {
18284                    self.virtual_dep_fallback_activations =
18285                        self.virtual_dep_fallback_activations.saturating_add(1);
18286                    return self.evaluate_all_coordinator();
18287                }
18288
18289                let start = crate::instant::FzInstant::now();
18290                let dirty_vertices = self.graph.get_evaluation_vertices();
18291                if dirty_vertices.is_empty() {
18292                    return Ok(EvalResult {
18293                        computed_vertices: 0,
18294                        cycle_errors: 0,
18295                        elapsed: start.elapsed(),
18296                    });
18297                }
18298
18299                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
18300                let mut computed_vertices = 0;
18301                let mut cycle_errors = 0;
18302                for &unit in &schedule.units {
18303                    self.cancellation_checkpoint(
18304                        "Evaluation cancelled before recalculation plan schedule unit",
18305                    )?;
18306                    match unit {
18307                        ScheduleUnit::Cycle(i) => {
18308                            let stamped = self.handle_cycle_unit(
18309                                schedule.unit_cycle(i),
18310                                None,
18311                                Some(&dirty_set),
18312                                None,
18313                            )?;
18314                            if stamped > 0 {
18315                                cycle_errors += 1;
18316                            }
18317                        }
18318                        ScheduleUnit::Layer(i) => {
18319                            let work: Vec<VertexId> = schedule
18320                                .unit_layer(i)
18321                                .vertices
18322                                .iter()
18323                                .copied()
18324                                .filter(|v| dirty_set.contains(v))
18325                                .collect();
18326                            if work.is_empty() {
18327                                continue;
18328                            }
18329                            let temp_layer = crate::engine::scheduler::Layer { vertices: work };
18330                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
18331                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
18332                            } else {
18333                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
18334                            }
18335                        }
18336                    }
18337                }
18338
18339                self.resource_checkpoint(0)?;
18340                self.graph.clear_dirty_flags(&dirty_vertices);
18341                self.redirty_for_next_recalc();
18342                Ok(EvalResult {
18343                    computed_vertices,
18344                    cycle_errors,
18345                    elapsed: start.elapsed(),
18346                })
18347            }
18348        }
18349    }
18350    fn evaluate_all_legacy_and_ack_dirty(
18351        &mut self,
18352        formula_dirty: FormulaDirtyLease,
18353    ) -> Result<EvalResult, ExcelError> {
18354        let result = self.evaluate_all_legacy_impl()?;
18355        self.checked_ack_formula_dirty_observed(formula_dirty)?;
18356        Ok(result)
18357    }
18358
18359    fn evaluate_formula_plane_capacity_fallback(
18360        &mut self,
18361        mut formula_dirty: FormulaDirtyLease,
18362        owned_dirty_events: &[usize],
18363        selected_span_refs: &[FormulaSpanRef],
18364        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18365        delta: Option<&mut DeltaCollector>,
18366    ) -> Result<EvalResult, ExcelError> {
18367        let leased_len = formula_dirty.len();
18368        if !selected_span_refs.is_empty() {
18369            let materialization_started = crate::instant::FzInstant::now();
18370            let prepared = self
18371                .prepare_formula_span_demotion(selected_span_refs)
18372                .map_err(|error| {
18373                    if let FormulaSpanDemotionError::Resource(error) = error {
18374                        error
18375                    } else {
18376                        ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18377                            "FormulaPlane capacity fallback demotion preparation failed: {error}"
18378                        ))
18379                    }
18380                })?;
18381            let report = self
18382                .commit_prepared_formula_span_demotion(prepared)
18383                .map_err(|error| {
18384                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
18385                        "FormulaPlane capacity fallback demotion commit failed: {error}"
18386                    ))
18387                })?;
18388            self.observe_materialization(
18389                report.placements_materialized,
18390                false,
18391                materialization_started.elapsed(),
18392            );
18393        }
18394        #[cfg(test)]
18395        {
18396            self.last_formula_plane_span_eval_report = None;
18397        }
18398        if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
18399            formula_dirty = extended;
18400        }
18401        let mut acknowledged = owned_dirty_events.to_vec();
18402        acknowledged.extend(leased_len..formula_dirty.len());
18403        let result = if let Some(target_roots) = target_roots {
18404            let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
18405            self.evaluate_legacy_target_roots(&refreshed, delta)?
18406        } else if let Some(delta) = delta {
18407            self.evaluate_all_with_delta_collector(delta)?
18408        } else {
18409            self.evaluate_all_legacy_impl()?
18410        };
18411        self.formula_plane_capacity_bailouts =
18412            self.formula_plane_capacity_bailouts.saturating_add(1);
18413        if target_roots.is_some() {
18414            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
18415        } else {
18416            self.checked_ack_formula_dirty_observed(formula_dirty)?;
18417        }
18418        Ok(result)
18419    }
18420
18421    fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
18422        self.evaluate_authoritative_formula_plane(None, None)
18423    }
18424
18425    fn evaluate_authoritative_formula_plane_targets(
18426        &mut self,
18427        roots: &[crate::engine::target_preparation::TargetProducer],
18428        delta: Option<&mut DeltaCollector>,
18429    ) -> Result<EvalResult, ExcelError> {
18430        self.evaluate_authoritative_formula_plane(Some(roots), delta)
18431    }
18432
18433    fn evaluate_authoritative_formula_plane(
18434        &mut self,
18435        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
18436        mut delta: Option<&mut DeltaCollector>,
18437    ) -> Result<EvalResult, ExcelError> {
18438        // The public/request coordinators begin exactly once. Every primitive
18439        // below is deliberately non-beginning and non-finalising so the dirty
18440        // lease, iterative accumulator and §7.11 clock sample have one owner.
18441        let mut formula_dirty = self.graph.lease_formula_dirty();
18442        self.observe_dirty_lease_acquired(formula_dirty.is_empty());
18443
18444        // SingletonUnique formulas intentionally remain legacy graph vertices;
18445        // when no spans are active, execute through the private legacy primitive.
18446        if self.graph.formula_authority().active_span_count() == 0 {
18447            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
18448            #[cfg(test)]
18449            {
18450                self.last_formula_plane_span_eval_report = None;
18451            }
18452            if let Some(target_roots) = target_roots {
18453                let result =
18454                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18455                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18456                return Ok(result);
18457            }
18458            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18459        }
18460
18461        // With no graph-owned span/region dirtiness, only sparse legacy work
18462        // (including volatiles) can remain. Preserve the warm no-op fast path.
18463        if formula_dirty.is_empty() {
18464            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
18465            #[cfg(test)]
18466            {
18467                self.last_formula_plane_span_eval_report = None;
18468            }
18469            if let Some(target_roots) = target_roots {
18470                let result =
18471                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
18472                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18473                return Ok(result);
18474            }
18475            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18476        }
18477
18478        let start = crate::instant::FzInstant::now();
18479        let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
18480        let mut workbook_exact_attempted = target_roots.is_none();
18481        // #CIRC stamps produced by demoting cyclic spans and resolving the
18482        // residual legacy-only cycle ahead of the mixed schedule (gotcha G8).
18483        let mut prepass_cycle_errors = 0usize;
18484        const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
18485        let mut cycle_demote_iters = 0usize;
18486        let mut include_dirty_regions = true;
18487        let mut retry_whole_spans = Vec::new();
18488        let mut computed_vertices = 0usize;
18489        let mut runtime_replan_iterations = 0usize;
18490        const MAX_RUNTIME_REPLAN: usize = 5;
18491        let mut virtual_telemetry = self
18492            .config
18493            .enable_virtual_dep_telemetry
18494            .then(|| self.start_virtual_dep_telemetry());
18495        'virtual_replan: loop {
18496            self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
18497            let (
18498                schedule,
18499                span_refs_by_id,
18500                plane_epoch,
18501                legacy_vertices,
18502                owned_dirty_events,
18503                island_plan,
18504            ) = loop {
18505                let (
18506                    schedule,
18507                    span_refs_by_id,
18508                    plane_epoch,
18509                    legacy_vertices,
18510                    owned_dirty_events,
18511                    island_plan,
18512                ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
18513                    self.build_formula_plane_target_schedule(
18514                        &formula_dirty,
18515                        &retry_whole_spans,
18516                        include_dirty_regions,
18517                        target_roots,
18518                    )?
18519                } else {
18520                    self.build_formula_plane_mixed_schedule(
18521                        &formula_dirty,
18522                        &retry_whole_spans,
18523                        include_dirty_regions,
18524                    )?
18525                };
18526                #[cfg(test)]
18527                let mut schedule = schedule;
18528                #[cfg(test)]
18529                if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
18530                    let producer = schedule
18531                        .layers
18532                        .iter()
18533                        .flat_map(|layer| layer.work.iter())
18534                        .map(|work| work.producer)
18535                        .next()
18536                        .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
18537                    schedule.fallbacks.push(
18538                        crate::formula_plane::scheduler::MixedScheduleFallback {
18539                            producer,
18540                            reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
18541                        },
18542                    );
18543                }
18544
18545                if schedule.is_authoritative_safe() {
18546                    break (
18547                        schedule,
18548                        span_refs_by_id,
18549                        plane_epoch,
18550                        legacy_vertices,
18551                        owned_dirty_events,
18552                        island_plan,
18553                    );
18554                }
18555
18556                // Non-cycle unsafe schedules cannot safely run legacy-only while a
18557                // scheduled span remains virtual. Refine exactly every span in this
18558                // request's scheduled work, then complete the request in one legacy
18559                // pass. Clean, unscheduled spans keep their authority and overlays.
18560                let has_cycle_fallback = schedule.stats.cycle_count > 0
18561                    || schedule
18562                        .fallbacks
18563                        .iter()
18564                        .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
18565                if !has_cycle_fallback {
18566                    let selected_span_refs = schedule
18567                        .layers
18568                        .iter()
18569                        .flat_map(|layer| layer.work.iter())
18570                        .filter_map(|work| match work.producer {
18571                            FormulaProducerId::Span(span_id) => {
18572                                span_refs_by_id.get(&span_id).copied()
18573                            }
18574                            FormulaProducerId::Legacy(_) => None,
18575                        })
18576                        .collect::<Vec<_>>();
18577
18578                    return self.evaluate_formula_plane_capacity_fallback(
18579                        formula_dirty,
18580                        &owned_dirty_events,
18581                        &selected_span_refs,
18582                        runtime_target_roots.as_deref(),
18583                        delta.as_deref_mut(),
18584                    );
18585                }
18586
18587                // Gotcha G8 (refs #112): a span whose member cell participates in a
18588                // statically-cyclic SCC must never be span-evaluated. Cross-cell
18589                // cycles that route through a span producer are invisible to the
18590                // legacy Tarjan pass (the span member has no graph vertex) and only
18591                // surface here, as `CycleDetected` fallbacks in the producer-bounded
18592                // mixed schedule. Demote the cyclic spans to legacy graph vertices
18593                // so the cycle members move onto the legacy SCC path, then resolve
18594                // the now legacy-only cycle ahead of the schedule and rebuild.
18595                // Spans that do not touch the cycle are left untouched.
18596                let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
18597                if !cyclic_spans.is_empty() {
18598                    self.demote_cyclic_spans(&cyclic_spans)?;
18599                }
18600
18601                if self.graph.formula_authority().active_span_count() == 0 {
18602                    // All spans demoted; nothing left for the FP coordinator. The
18603                    // legacy evaluator resolves the (now fully legacy) cycle.
18604                    if let Some(target_roots) = runtime_target_roots.as_deref() {
18605                        let refreshed =
18606                            self.resolve_target_producers_from_existing_roots(target_roots)?;
18607                        let result =
18608                            self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
18609                        self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
18610                        return Ok(result);
18611                    }
18612                    return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
18613                }
18614
18615                // Resolve the residual legacy-only cycle (`handle_cycle_unit`
18616                // honors Static vs Runtime) before rebuilding so the mixed schedule
18617                // is cycle-free and the surviving spans still get evaluated.
18618                prepass_cycle_errors =
18619                    prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
18620
18621                // Re-seed every surviving span explicitly through the graph-owned
18622                // dirty authority, then renew the lease so successful completion
18623                // acknowledges both the original prefix and retry seeds.
18624                retry_whole_spans = if runtime_target_roots.is_some() {
18625                    let active = self
18626                        .graph
18627                        .formula_authority()
18628                        .active_span_refs()
18629                        .into_iter()
18630                        .map(|span_ref| (span_ref.id, span_ref))
18631                        .collect::<BTreeMap<_, _>>();
18632                    let mut refs = schedule
18633                        .layers
18634                        .iter()
18635                        .flat_map(|layer| layer.work.iter())
18636                        .filter_map(|work| match work.producer {
18637                            FormulaProducerId::Span(id) => active.get(&id).copied(),
18638                            FormulaProducerId::Legacy(_) => None,
18639                        })
18640                        .collect::<Vec<_>>();
18641                    refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
18642                        match fallback.producer {
18643                            FormulaProducerId::Span(id) => active.get(&id).copied(),
18644                            FormulaProducerId::Legacy(_) => None,
18645                        }
18646                    }));
18647                    refs.sort_by_key(|span_ref| span_ref.id);
18648                    refs.dedup();
18649                    refs
18650                } else {
18651                    self.graph.formula_authority().active_span_refs()
18652                };
18653                self.graph.mark_formula_spans_dirty(
18654                    retry_whole_spans.iter().copied(),
18655                    WholeSpanDirtyReason::CycleRetry,
18656                );
18657                // Only the first renewal can enlarge this generation's owned
18658                // prefix. If another cycle iteration is required, its redundant
18659                // retry seeds stay pending rather than risking acknowledgement of
18660                // work recorded after the first renewal.
18661                if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
18662                {
18663                    formula_dirty = extended;
18664                    #[cfg(test)]
18665                    if std::mem::take(
18666                        &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
18667                    ) {
18668                        self.graph.mark_formula_spans_dirty(
18669                            retry_whole_spans.iter().copied().take(1),
18670                            WholeSpanDirtyReason::GlobalInvalidation,
18671                        );
18672                    }
18673                }
18674                include_dirty_regions = false;
18675
18676                cycle_demote_iters += 1;
18677                if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
18678                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
18679                        "FormulaPlane cyclic demotion did not converge".to_string(),
18680                    ));
18681                }
18682            };
18683            self.observe_formula_plane_route(
18684                &island_plan,
18685                FormulaPlaneRoutePhase::Planned,
18686                FormulaPlaneRouteTransitionReason::ProvenIsolated,
18687                cycle_demote_iters,
18688                runtime_replan_iterations,
18689            );
18690            if !island_plan.dirty_vertices.is_empty() {
18691                self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
18692                let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
18693                    &island_plan.dirty_vertices,
18694                    delta.as_deref_mut(),
18695                )?;
18696                computed_vertices = computed_vertices.saturating_add(island_computed);
18697                prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
18698                self.observe_formula_plane_route(
18699                    &island_plan,
18700                    FormulaPlaneRoutePhase::Executed,
18701                    FormulaPlaneRouteTransitionReason::ProvenIsolated,
18702                    cycle_demote_iters,
18703                    runtime_replan_iterations,
18704                );
18705                self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
18706            }
18707            let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
18708            let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18709
18710            #[cfg(test)]
18711            {
18712                self.last_formula_plane_span_eval_report = None;
18713            }
18714            for layer in schedule.layers {
18715                self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
18716                let mut buffer = ComputedWriteBuffer::default();
18717                let mut sink = SpanComputedWriteSink::new(&mut buffer);
18718                let work_items = layer.work;
18719                let mut work_index = 0usize;
18720                while work_index < work_items.len() {
18721                    match work_items[work_index].producer {
18722                        FormulaProducerId::Span(span_id) => {
18723                            let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
18724                                ExcelError::new(ExcelErrorKind::NImpl)
18725                                    .with_message("FormulaPlane schedule referenced a stale span")
18726                            })?;
18727                            let sheet_id = {
18728                                let authority = self.graph.formula_authority();
18729                                let span =
18730                                    authority.plane.spans.get(span_ref).ok_or_else(|| {
18731                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
18732                                            "FormulaPlane schedule referenced a stale span",
18733                                        )
18734                                    })?;
18735                                span.sheet_id
18736                            };
18737                            let current_sheet = self.graph.sheet_name(sheet_id);
18738                            let authority = self.graph.formula_authority();
18739                            let evaluator = SpanEvaluator::new_with_cancel(
18740                                &authority.plane,
18741                                self,
18742                                current_sheet,
18743                                self.graph.data_store(),
18744                                self.graph.sheet_reg(),
18745                                self.active_cancel_flag.as_ref(),
18746                            );
18747                            #[cfg(test)]
18748                            let mut last_group_report = None;
18749                            let mut selected_group_work = 0_u64;
18750                            while work_index < work_items.len() {
18751                                let FormulaProducerId::Span(group_span_id) =
18752                                    work_items[work_index].producer
18753                                else {
18754                                    break;
18755                                };
18756                                let group_span_ref =
18757                                    *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
18758                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
18759                                            "FormulaPlane schedule referenced a stale span",
18760                                        )
18761                                    })?;
18762                                let group_sheet_id = {
18763                                    let authority = self.graph.formula_authority();
18764                                    let span =
18765                                        authority.plane.spans.get(group_span_ref).ok_or_else(
18766                                            || {
18767                                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
18768                                                    "FormulaPlane schedule referenced a stale span",
18769                                                )
18770                                            },
18771                                        )?;
18772                                    span.sheet_id
18773                                };
18774                                if group_sheet_id != sheet_id {
18775                                    break;
18776                                }
18777
18778                                let dirty = producer_dirty_to_span_dirty(
18779                                    work_items[work_index].dirty.clone(),
18780                                    group_span_ref,
18781                                );
18782                                let task = SpanEvalTask {
18783                                    span: group_span_ref,
18784                                    dirty,
18785                                    plane_epoch,
18786                                };
18787                                self.cancellation_checkpoint(
18788                                    "Evaluation cancelled during FormulaPlane span",
18789                                )?;
18790                                let report = evaluator.evaluate_task(&task, &mut sink).map_err(
18791                                    |err| {
18792                                        if err
18793                                            == crate::formula_plane::span_eval::SpanEvalError::Cancelled
18794                                        {
18795                                            ExcelError::new(ExcelErrorKind::Cancelled).with_message(
18796                                                "Evaluation cancelled during FormulaPlane span",
18797                                            )
18798                                        } else {
18799                                            ExcelError::new(ExcelErrorKind::NImpl).with_message(
18800                                                format!(
18801                                                    "FormulaPlane span evaluation failed: {err:?}"
18802                                                ),
18803                                            )
18804                                        }
18805                                    },
18806                                )?;
18807                                #[cfg(test)]
18808                                {
18809                                    last_group_report = Some(report.clone());
18810                                }
18811                                selected_group_work = selected_group_work
18812                                    .saturating_add(report.span_eval_placement_count);
18813                                computed_vertices = computed_vertices
18814                                    .saturating_add(report.span_eval_placement_count as usize);
18815                                work_index = work_index.saturating_add(1);
18816                            }
18817                            // Charge the exact selected placement regions before
18818                            // their transaction-local value buffer is published.
18819                            self.charge_bounded_work(selected_group_work)?;
18820                            #[cfg(test)]
18821                            {
18822                                if let Some(report) = last_group_report {
18823                                    self.last_formula_plane_span_eval_report = Some(report);
18824                                }
18825                            }
18826                        }
18827                        FormulaProducerId::Legacy(_) => {
18828                            // Batch the contiguous run of legacy work items into a
18829                            // synthetic layer and evaluate it through the same
18830                            // coalesced effects pipeline as the legacy scheduler.
18831                            // Items in one mixed layer have no edges between them
18832                            // (same invariant the legacy Kahn layers rely on), so
18833                            // batching preserves ordering semantics while
18834                            // amortizing per-write overlay mirroring that makes
18835                            // one-vertex-at-a-time evaluation ~30x slower.
18836                            let mut vertices = Vec::new();
18837                            while work_index < work_items.len() {
18838                                let FormulaProducerId::Legacy(vertex_id) =
18839                                    work_items[work_index].producer
18840                                else {
18841                                    break;
18842                                };
18843                                vertices.push(vertex_id);
18844                                work_index = work_index.saturating_add(1);
18845                            }
18846                            let legacy_layer = crate::engine::scheduler::Layer { vertices };
18847                            let evaluated = if let Some(delta) = delta.as_deref_mut() {
18848                                if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
18849                                    self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
18850                                } else {
18851                                    self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
18852                                }
18853                            } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
18854                            {
18855                                self.evaluate_layer_parallel(&legacy_layer)?
18856                            } else {
18857                                self.evaluate_layer_sequential(&legacy_layer)?
18858                            };
18859                            computed_vertices = computed_vertices.saturating_add(evaluated);
18860                        }
18861                    }
18862                }
18863                self.resource_checkpoint(0)?;
18864                self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
18865                if let Some(delta) = delta.as_deref_mut() {
18866                    self.record_computed_write_buffer_delta(&buffer, delta);
18867                }
18868                self.flush_computed_write_buffer(&mut buffer)?;
18869            }
18870
18871            let mut changed_virtual =
18872                self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
18873            let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
18874            let mut dynamic_candidates = old_dynamic_regions
18875                .keys()
18876                .chain(new_dynamic_regions.keys())
18877                .copied()
18878                .collect::<FxHashSet<_>>();
18879            for vertex in dynamic_candidates.drain() {
18880                if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
18881                    && !changed_virtual.contains(&vertex)
18882                {
18883                    changed_virtual.push(vertex);
18884                }
18885            }
18886            if let Some(telemetry) = virtual_telemetry.as_mut() {
18887                telemetry.changed_vdeps_total = telemetry
18888                    .changed_vdeps_total
18889                    .saturating_add(changed_virtual.len());
18890            }
18891            self.resource_checkpoint(0)?;
18892            self.graph.clear_dirty_flags(&legacy_vertices);
18893            if !changed_virtual.is_empty() {
18894                for vertex in &changed_virtual {
18895                    self.graph.set_dirty(*vertex, true);
18896                }
18897                let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
18898                let can_attempt_workbook =
18899                    !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
18900                if can_replan || can_attempt_workbook {
18901                    if can_replan {
18902                        if let Some(roots) = runtime_target_roots.as_mut() {
18903                            self.extend_target_roots_with_dynamic_regions(
18904                                roots,
18905                                old_dynamic_regions
18906                                    .values()
18907                                    .flatten()
18908                                    .chain(new_dynamic_regions.values().flatten()),
18909                            )?;
18910                        }
18911                        runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
18912                    } else {
18913                        runtime_target_roots = None;
18914                        workbook_exact_attempted = true;
18915                    }
18916                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
18917                        if can_replan {
18918                            stats.runtime_replan_rounds =
18919                                stats.runtime_replan_rounds.saturating_add(1);
18920                        }
18921                        stats.runtime_widening_rounds =
18922                            stats.runtime_widening_rounds.saturating_add(1);
18923                        if can_attempt_workbook {
18924                            stats.workbook_exact_attempts = 1;
18925                        }
18926                    }
18927                    self.cached_mixed_topology = None;
18928                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
18929                        ledger
18930                            .account_mixed_cache(0)
18931                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
18932                    }
18933                    include_dirty_regions = true;
18934                    continue 'virtual_replan;
18935                }
18936                if let Some(telemetry) = virtual_telemetry.as_mut() {
18937                    telemetry.replan_iterations = runtime_replan_iterations;
18938                    telemetry.bailout_reason = Some("max_replan");
18939                    self.last_virtual_dep_telemetry = telemetry.clone();
18940                }
18941                return Err(self.replan_exhausted_error(
18942                    MAX_RUNTIME_REPLAN,
18943                    "mixed dynamic dependency evaluation",
18944                ));
18945            }
18946            if let Some(mut telemetry) = virtual_telemetry {
18947                telemetry.replan_iterations = runtime_replan_iterations;
18948                telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
18949                    "converged"
18950                } else {
18951                    "max_replan"
18952                });
18953                self.last_virtual_dep_telemetry = telemetry;
18954            }
18955            // Drop dirty flags on any newly-scheduled FP runtime cells whose graph
18956            // vertices weren't in the dirty subset (e.g. recently-introduced span
18957            // result cells); legacy clear_dirty_flags is safe over the full set.
18958            self.redirty_for_next_recalc();
18959            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
18960            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
18961            return Ok(EvalResult {
18962                computed_vertices,
18963                cycle_errors: prepass_cycle_errors,
18964                elapsed: start.elapsed(),
18965            });
18966        }
18967    }
18968
18969    fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
18970        let budgets = &self.evaluation_resource_budgets;
18971        let candidate_limit = budgets
18972            .optimization
18973            .mixed_cache_candidates
18974            .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
18975                limit.min(self.config.max_formula_plane_cache_candidates)
18976            });
18977        let edge_limit = budgets
18978            .optimization
18979            .mixed_cache_edges
18980            .map_or(self.config.max_formula_plane_cache_edges, |limit| {
18981                limit.min(self.config.max_formula_plane_cache_edges)
18982            });
18983        let mut byte_limit =
18984            u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
18985        if let Some(limit) = budgets.retained.total_bytes {
18986            byte_limit = byte_limit.min(limit);
18987        }
18988        if let Some(limit) = budgets.retained.mixed_cache_bytes {
18989            byte_limit = byte_limit.min(limit);
18990        }
18991        (
18992            candidate_limit,
18993            edge_limit,
18994            usize::try_from(byte_limit).unwrap_or(usize::MAX),
18995        )
18996    }
18997
18998    fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
18999        let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
19000        MixedTopologyCacheKey {
19001            engine_topology_epoch: self.topology_epoch,
19002            graph_topology_revision: self.graph.topology_revision(),
19003            authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
19004            legacy_island_revision: self.graph.topology_revision(),
19005            boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
19006            function_semantic_epoch: self.function_semantic_epoch_seen,
19007            function_provider_revision: self.function_provider_revision_seen,
19008            max_candidates,
19009            max_edges,
19010            max_memory_bytes,
19011        }
19012    }
19013
19014    fn mixed_topology_index_preflight_bytes(&self) -> u64 {
19015        use crate::formula_plane::region_index::AxisRange;
19016
19017        const FIXED_BYTES: u64 = 8 * 1024;
19018        const ENTRY_BYTES: u64 = 512;
19019        const INDEX_UNIT_BYTES: u64 = 1024;
19020        const BINDING_BYTES: u64 = 128;
19021        let index_units = |region: Region| -> u64 {
19022            match region.normalized().axis_ranges() {
19023                (AxisRange::Point(_), AxisRange::Point(_)) => 2,
19024                (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
19025                    let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
19026                    let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
19027                    row_buckets.saturating_mul(col_buckets).saturating_mul(2)
19028                }
19029                _ => 1,
19030            }
19031        };
19032
19033        let authority = self.graph.formula_authority();
19034        let span_refs = authority.active_span_refs();
19035        let legacy_vertices = self.graph.formula_vertices();
19036        let mut entry_count = 0_u64;
19037        let mut units = 0_u64;
19038        for span_ref in &span_refs {
19039            if let Some(span) = authority.plane.spans.get(*span_ref) {
19040                let result_region = Region::from_domain(span.result_region.domain());
19041                entry_count = entry_count.saturating_add(1);
19042                units = units.saturating_add(index_units(result_region));
19043                if let Some(summary_id) = span.read_summary_id
19044                    && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
19045                {
19046                    for dependency in &summary.dependencies {
19047                        entry_count = entry_count.saturating_add(1);
19048                        units = units.saturating_add(index_units(dependency.read_region));
19049                    }
19050                }
19051            }
19052        }
19053        for &vertex in &legacy_vertices {
19054            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
19055                let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19056                entry_count = entry_count.saturating_add(1);
19057                units = units.saturating_add(index_units(point));
19058            }
19059            for dependency in self.graph.get_dependencies(vertex) {
19060                if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
19061                    let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19062                    entry_count = entry_count.saturating_add(1);
19063                    units = units.saturating_add(index_units(point));
19064                }
19065            }
19066            let vertex_sheet = self.graph.get_vertex_sheet_id(vertex);
19067            if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
19068                for range in ranges {
19069                    if let Ok(Some(region)) =
19070                        self.shared_range_to_region_pattern(range, vertex_sheet)
19071                    {
19072                        entry_count = entry_count.saturating_add(1);
19073                        units = units.saturating_add(index_units(region));
19074                    }
19075                }
19076            }
19077        }
19078        FIXED_BYTES
19079            .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
19080            .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
19081            .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
19082    }
19083
19084    fn prove_sheet_disjoint_legacy_island(
19085        &self,
19086        span_results: &FormulaProducerResultIndex,
19087        span_reads: &FormulaConsumerReadIndex,
19088        legacy_vertices: &[VertexId],
19089    ) -> Option<LegacyIslandPlan> {
19090        if !self.legacy_island_structural_summaries_trusted
19091            || self.graph.named_ranges_iter().next().is_some()
19092            || self.graph.sheet_named_ranges_iter().next().is_some()
19093            || self.graph.spill_registry_counts() != (0, 0)
19094            || legacy_vertices.iter().copied().any(|vertex| {
19095                self.graph.is_dynamic(vertex)
19096                    || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19097            })
19098        {
19099            return None;
19100        }
19101        let span_result_sheets = span_results
19102            .entries()
19103            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19104            .map(|entry| entry.result_region.sheet_id())
19105            .collect::<FxHashSet<_>>();
19106        let span_read_sheets = span_reads
19107            .entries()
19108            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
19109            .map(|entry| entry.read_region.sheet_id())
19110            .collect::<FxHashSet<_>>();
19111
19112        let span_boundary_sheets = span_result_sheets
19113            .union(&span_read_sheets)
19114            .copied()
19115            .collect::<FxHashSet<_>>();
19116        for vertex in legacy_vertices {
19117            let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
19118            if span_boundary_sheets.contains(&cell.sheet_id) {
19119                return None;
19120            }
19121            for dependency in self.graph.get_dependencies(*vertex) {
19122                let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
19123                if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
19124                    return None;
19125                }
19126            }
19127            if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19128                for range in ranges {
19129                    let region = self
19130                        .shared_range_to_region_pattern(range, cell.sheet_id)
19131                        .ok()??;
19132                    if span_boundary_sheets.contains(&region.sheet_id()) {
19133                        return None;
19134                    }
19135                }
19136            }
19137        }
19138
19139        let mut membership = legacy_vertices.to_vec();
19140        membership.sort_unstable();
19141        if membership.is_empty() {
19142            return None;
19143        }
19144        let mut sheet_ids = membership
19145            .iter()
19146            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19147            .map(|cell| cell.sheet_id)
19148            .collect::<Vec<_>>();
19149        sheet_ids.sort_unstable();
19150        sheet_ids.dedup();
19151        let island_id = membership
19152            .iter()
19153            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19154                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19155            });
19156        let retained_bytes = membership
19157            .capacity()
19158            .saturating_mul(std::mem::size_of::<VertexId>())
19159            .saturating_add(
19160                sheet_ids
19161                    .capacity()
19162                    .saturating_mul(std::mem::size_of::<SheetId>()),
19163            );
19164        Some(LegacyIslandPlan {
19165            membership,
19166            dirty_vertices: Vec::new(),
19167            sheet_ids,
19168            island_id,
19169            retained_bytes,
19170            boundary_relationships: 0,
19171            omitted_relationships: legacy_vertices.len(),
19172            omitted_direct_relationships: 0,
19173        })
19174    }
19175
19176    fn contract_legacy_islands(
19177        &self,
19178        full_results: &FormulaProducerResultIndex,
19179        full_reads: &FormulaConsumerReadIndex,
19180        legacy_reads_complete: bool,
19181        candidate_cap: usize,
19182    ) -> Option<(
19183        FormulaProducerResultIndex,
19184        FormulaConsumerReadIndex,
19185        LegacyIslandPlan,
19186    )> {
19187        use crate::formula_plane::producer::ProjectionResult;
19188        use crate::formula_plane::region_index::BoundedRegionQueryResult;
19189
19190        if !self.legacy_island_structural_summaries_trusted
19191            || !legacy_reads_complete
19192            || self.graph.named_ranges_iter().next().is_some()
19193            || self.graph.sheet_named_ranges_iter().next().is_some()
19194            || self.graph.spill_registry_counts() != (0, 0)
19195        {
19196            return None;
19197        }
19198        let legacy_vertices = self.graph.formula_vertices();
19199        if legacy_vertices.iter().copied().any(|vertex| {
19200            self.graph.is_dynamic(vertex)
19201                || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
19202        }) {
19203            return None;
19204        }
19205
19206        let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
19207        for entry in full_reads.entries() {
19208            if let FormulaProducerId::Legacy(vertex) = entry.consumer {
19209                reads_by_consumer
19210                    .entry(vertex)
19211                    .or_default()
19212                    .push(entry.read_region);
19213            }
19214        }
19215
19216        let mut connected = FxHashSet::default();
19217        let mut queue = VecDeque::new();
19218        let mut observed_candidates = 0usize;
19219        let mut boundary_relationships = 0usize;
19220        let add_connected = |vertex: VertexId,
19221                             connected: &mut FxHashSet<VertexId>,
19222                             queue: &mut VecDeque<VertexId>| {
19223            if connected.insert(vertex) {
19224                queue.push_back(vertex);
19225            }
19226        };
19227
19228        // Span results -> legacy consumers.
19229        for result in full_results.entries() {
19230            if !matches!(result.producer, FormulaProducerId::Span(_)) {
19231                continue;
19232            }
19233            let remaining = candidate_cap.saturating_sub(observed_candidates);
19234            let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
19235            let BoundedRegionQueryResult::Complete(query) = query else {
19236                return None;
19237            };
19238            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19239            for matched in query.matches {
19240                if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
19241                    match matched.value.dirty {
19242                        ProjectionResult::NoIntersection => {}
19243                        ProjectionResult::Unsupported(_) => return None,
19244                        ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19245                            boundary_relationships = boundary_relationships.saturating_add(1);
19246                            add_connected(vertex, &mut connected, &mut queue);
19247                        }
19248                    }
19249                }
19250            }
19251        }
19252
19253        // Span reads -> legacy producers.
19254        for read in full_reads.entries() {
19255            if !matches!(read.consumer, FormulaProducerId::Span(_)) {
19256                continue;
19257            }
19258            let remaining = candidate_cap.saturating_sub(observed_candidates);
19259            let query = full_results.query_bounded(read.read_region, remaining);
19260            let BoundedRegionQueryResult::Complete(query) = query else {
19261                return None;
19262            };
19263            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
19264            for matched in query.matches {
19265                if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
19266                    boundary_relationships = boundary_relationships.saturating_add(1);
19267                    add_connected(vertex, &mut connected, &mut queue);
19268                }
19269            }
19270        }
19271
19272        // Undirected closure through legacy relationships. Every vertex in this
19273        // closure remains in the mixed scheduler, preserving the only detector
19274        // for cycles whose path crosses a virtual span member.
19275        while let Some(vertex) = queue.pop_front() {
19276            let producer = FormulaProducerId::Legacy(vertex);
19277            if let Some(result_region) = full_results.producer_result_region(producer) {
19278                let remaining = candidate_cap.saturating_sub(observed_candidates);
19279                let query = full_reads.query_changed_region_bounded(result_region, remaining);
19280                let BoundedRegionQueryResult::Complete(query) = query else {
19281                    return None;
19282                };
19283                observed_candidates =
19284                    observed_candidates.saturating_add(query.stats.candidate_count);
19285                for matched in query.matches {
19286                    if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
19287                        match matched.value.dirty {
19288                            ProjectionResult::NoIntersection => {}
19289                            ProjectionResult::Unsupported(_) => return None,
19290                            ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
19291                                add_connected(consumer, &mut connected, &mut queue);
19292                            }
19293                        }
19294                    }
19295                }
19296            }
19297            for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
19298                let remaining = candidate_cap.saturating_sub(observed_candidates);
19299                let query = full_results.query_bounded(*read_region, remaining);
19300                let BoundedRegionQueryResult::Complete(query) = query else {
19301                    return None;
19302                };
19303                observed_candidates =
19304                    observed_candidates.saturating_add(query.stats.candidate_count);
19305                for matched in query.matches {
19306                    if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
19307                        add_connected(precedent, &mut connected, &mut queue);
19308                    }
19309                }
19310            }
19311        }
19312
19313        let mut membership = legacy_vertices
19314            .iter()
19315            .copied()
19316            .filter(|vertex| !connected.contains(vertex))
19317            .collect::<Vec<_>>();
19318        membership.sort_unstable();
19319        if membership.is_empty() {
19320            return None;
19321        }
19322
19323        let mut producer_results = FormulaProducerResultIndex::default();
19324        let mut consumer_reads = FormulaConsumerReadIndex::default();
19325        let retained_span_results = full_results
19326            .entries()
19327            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
19328            .count();
19329        producer_results
19330            .try_reserve(retained_span_results.saturating_add(connected.len()))
19331            .ok()?;
19332        for entry in full_results.entries() {
19333            let retain = match entry.producer {
19334                FormulaProducerId::Span(_) => true,
19335                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19336            };
19337            if retain {
19338                producer_results.insert_producer(entry.producer, entry.result_region);
19339            }
19340        }
19341        for entry in full_reads.entries() {
19342            let retain = match entry.consumer {
19343                FormulaProducerId::Span(_) => true,
19344                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
19345            };
19346            if retain {
19347                consumer_reads.try_reserve(1).ok()?;
19348                consumer_reads.insert_read(
19349                    entry.consumer,
19350                    entry.read_region,
19351                    entry.consumer_result_region,
19352                    entry.projection,
19353                );
19354            }
19355        }
19356
19357        let mut sheet_ids = membership
19358            .iter()
19359            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
19360            .map(|cell| cell.sheet_id)
19361            .collect::<Vec<_>>();
19362        sheet_ids.sort_unstable();
19363        sheet_ids.dedup();
19364        let island_id = membership
19365            .iter()
19366            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
19367                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
19368            });
19369        let retained_bytes = membership
19370            .capacity()
19371            .saturating_mul(std::mem::size_of::<VertexId>())
19372            .saturating_add(
19373                sheet_ids
19374                    .capacity()
19375                    .saturating_mul(std::mem::size_of::<SheetId>()),
19376            );
19377        let omitted_relationships = full_reads
19378            .entries()
19379            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
19380            .count();
19381        let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
19382        let omitted_direct_relationships = membership
19383            .iter()
19384            .map(|vertex| {
19385                self.graph
19386                    .get_dependencies(*vertex)
19387                    .into_iter()
19388                    .filter(|dependency| membership_set.contains(dependency))
19389                    .count()
19390            })
19391            .sum();
19392        Some((
19393            producer_results,
19394            consumer_reads,
19395            LegacyIslandPlan {
19396                membership,
19397                dirty_vertices: Vec::new(),
19398                sheet_ids,
19399                island_id,
19400                retained_bytes,
19401                boundary_relationships,
19402                omitted_relationships,
19403                omitted_direct_relationships,
19404            },
19405        ))
19406    }
19407
19408    fn compile_formula_plane_mixed_topology(
19409        &mut self,
19410        key: MixedTopologyCacheKey,
19411    ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
19412        #[cfg(test)]
19413        {
19414            self.mixed_topology_index_builds_for_test =
19415                self.mixed_topology_index_builds_for_test.saturating_add(1);
19416        }
19417        let authority = self.graph.formula_authority();
19418        let mut producer_results = FormulaProducerResultIndex::default();
19419        let mut consumer_reads = FormulaConsumerReadIndex::default();
19420        let mut legacy_reads_complete = true;
19421        let span_refs = authority.active_span_refs();
19422        let legacy_vertices = self.graph.formula_vertices();
19423        let _producer_count = span_refs
19424            .len()
19425            .checked_add(legacy_vertices.len())
19426            .ok_or_else(|| {
19427                ExcelError::new(ExcelErrorKind::NImpl)
19428                    .with_message("FormulaPlane cache producer count overflow")
19429            })?;
19430        producer_results.try_reserve(_producer_count).map_err(|_| {
19431            ExcelError::new(ExcelErrorKind::NImpl)
19432                .with_message("FormulaPlane cache producer index reservation failed")
19433        })?;
19434        let span_refs_by_id = span_refs
19435            .iter()
19436            .copied()
19437            .map(|span_ref| (span_ref.id, span_ref))
19438            .collect::<BTreeMap<_, _>>();
19439        for span_ref in &span_refs {
19440            let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
19441                ExcelError::new(ExcelErrorKind::NImpl)
19442                    .with_message("FormulaPlane active span ref is stale")
19443            })?;
19444            let result_region = Region::from_domain(span.result_region.domain());
19445            producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
19446            let Some(read_summary_id) = span.read_summary_id else {
19447                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19448                    .with_message("FormulaPlane active span is missing read summary"));
19449            };
19450            let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
19451            else {
19452                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19453                    .with_message("FormulaPlane active span has stale read summary"));
19454            };
19455            if read_summary.result_region != result_region {
19456                return Err(ExcelError::new(ExcelErrorKind::NImpl)
19457                    .with_message("FormulaPlane active span read summary is stale"));
19458            }
19459            for dependency in &read_summary.dependencies {
19460                consumer_reads.try_reserve(1).map_err(|_| {
19461                    ExcelError::new(ExcelErrorKind::NImpl)
19462                        .with_message("FormulaPlane cache read-index reservation failed")
19463                })?;
19464                consumer_reads.insert_read(
19465                    FormulaProducerId::Span(span.id),
19466                    dependency.read_region,
19467                    read_summary.result_region,
19468                    dependency.projection,
19469                );
19470            }
19471        }
19472
19473        let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
19474            &producer_results,
19475            &consumer_reads,
19476            &legacy_vertices,
19477        );
19478        if sheet_disjoint_island.is_none() {
19479            for vertex in &legacy_vertices {
19480                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19481                    continue;
19482                };
19483                producer_results.insert_producer(
19484                    FormulaProducerId::Legacy(*vertex),
19485                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
19486                );
19487            }
19488            for vertex in &legacy_vertices {
19489                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
19490                    continue;
19491                };
19492                let result_region =
19493                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19494                let mut seen = rustc_hash::FxHashSet::default();
19495                for dep in self.graph.get_dependencies(*vertex) {
19496                    let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
19497                        // Named/table/source vertices do not have a trustworthy
19498                        // regional boundary summary. Preserve the global planner.
19499                        legacy_reads_complete = false;
19500                        continue;
19501                    };
19502                    let read_region = Region::point(
19503                        dep_cell.sheet_id,
19504                        dep_cell.coord.row(),
19505                        dep_cell.coord.col(),
19506                    );
19507                    if seen.insert(read_region) {
19508                        consumer_reads.try_reserve(1).map_err(|_| {
19509                            ExcelError::new(ExcelErrorKind::NImpl)
19510                                .with_message("FormulaPlane cache read-index reservation failed")
19511                        })?;
19512                        consumer_reads.insert_read(
19513                            FormulaProducerId::Legacy(*vertex),
19514                            read_region,
19515                            result_region,
19516                            DirtyProjectionRule::WholeResult,
19517                        );
19518                    }
19519                }
19520                if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
19521                    for range in ranges {
19522                        let Some(read_region) =
19523                            self.shared_range_to_region_pattern(range, cell.sheet_id)?
19524                        else {
19525                            legacy_reads_complete = false;
19526                            continue;
19527                        };
19528                        if seen.insert(read_region) {
19529                            consumer_reads.try_reserve(1).map_err(|_| {
19530                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
19531                                    "FormulaPlane cache read-index reservation failed",
19532                                )
19533                            })?;
19534                            consumer_reads.insert_read(
19535                                FormulaProducerId::Legacy(*vertex),
19536                                read_region,
19537                                result_region,
19538                                DirtyProjectionRule::WholeResult,
19539                            );
19540                        }
19541                    }
19542                }
19543                let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
19544                for dependency in virtual_dependencies
19545                    .get(vertex)
19546                    .into_iter()
19547                    .flatten()
19548                    .copied()
19549                {
19550                    let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
19551                        continue;
19552                    };
19553                    let read_region =
19554                        Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
19555                    if seen.insert(read_region) {
19556                        consumer_reads.try_reserve(1).map_err(|_| {
19557                            ExcelError::new(ExcelErrorKind::NImpl)
19558                                .with_message("FormulaPlane cache virtual read reservation failed")
19559                        })?;
19560                        consumer_reads.insert_read(
19561                            FormulaProducerId::Legacy(*vertex),
19562                            read_region,
19563                            result_region,
19564                            DirtyProjectionRule::WholeResult,
19565                        );
19566                    }
19567                }
19568                for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
19569                {
19570                    if seen.insert(read_region) {
19571                        consumer_reads.try_reserve(1).map_err(|_| {
19572                            ExcelError::new(ExcelErrorKind::NImpl)
19573                                .with_message("FormulaPlane dynamic-region reservation failed")
19574                        })?;
19575                        consumer_reads.insert_read(
19576                            FormulaProducerId::Legacy(*vertex),
19577                            read_region,
19578                            result_region,
19579                            DirtyProjectionRule::WholeResult,
19580                        );
19581                    }
19582                }
19583            }
19584        }
19585
19586        // Boundary discovery uses complete, request-local indexes. Only the
19587        // span-connected closure is retained and compiled; disconnected legacy
19588        // membership is contracted into the legacy scheduler primitive.
19589        let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
19590        {
19591            (producer_results, consumer_reads, island)
19592        } else {
19593            self.contract_legacy_islands(
19594                &producer_results,
19595                &consumer_reads,
19596                legacy_reads_complete,
19597                key.max_candidates,
19598            )
19599            .unwrap_or((
19600                producer_results,
19601                consumer_reads,
19602                LegacyIslandPlan::default(),
19603            ))
19604        };
19605
19606        let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
19607            .checked_add(
19608                producer_results
19609                    .estimated_memory_bytes()
19610                    .unwrap_or(usize::MAX),
19611            )
19612            .and_then(|bytes| {
19613                bytes.checked_add(
19614                    consumer_reads
19615                        .estimated_memory_bytes()
19616                        .unwrap_or(usize::MAX),
19617                )
19618            })
19619            .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
19620            .and_then(|bytes| bytes.checked_add(island.retained_bytes))
19621            .unwrap_or(usize::MAX);
19622        let config = MixedTopologyConfig {
19623            max_candidates: key.max_candidates,
19624            max_edges: key.max_edges,
19625            max_memory_bytes: key.max_memory_bytes,
19626            retained_memory_bytes,
19627        };
19628        let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
19629        // `producers_observed` remains a discovery counter, not merely the
19630        // post-contraction mixed-node count. Preserve that established meaning
19631        // while exposing omissions separately in route telemetry.
19632        match &mut compiled {
19633            MixedTopologyCompileResult::Cached(topology) => {
19634                topology.stats.producers = topology
19635                    .stats
19636                    .producers
19637                    .saturating_add(island.membership.len());
19638                topology.stats.candidates = topology
19639                    .stats
19640                    .candidates
19641                    .saturating_add(island.omitted_direct_relationships);
19642                topology.stats.relationships = topology
19643                    .stats
19644                    .relationships
19645                    .saturating_add(island.omitted_direct_relationships);
19646            }
19647            MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
19648                observed.producers = observed.producers.saturating_add(island.membership.len());
19649                observed.candidates = observed
19650                    .candidates
19651                    .saturating_add(island.omitted_direct_relationships);
19652                observed.relationships = observed
19653                    .relationships
19654                    .saturating_add(island.omitted_direct_relationships);
19655            }
19656        }
19657        let plane_epoch = authority.plane.epoch().0;
19658        Ok(match compiled {
19659            MixedTopologyCompileResult::Cached(topology) => {
19660                FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
19661                    key,
19662                    topology,
19663                    producer_results,
19664                    consumer_reads,
19665                    span_refs_by_id,
19666                    plane_epoch,
19667                    island,
19668                })
19669            }
19670            MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
19671                FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
19672                    reason,
19673                    observed,
19674                    producer_results,
19675                    consumer_reads,
19676                    span_refs_by_id,
19677                    plane_epoch,
19678                    island,
19679                })
19680            }
19681        })
19682    }
19683
19684    fn build_formula_plane_mixed_schedule(
19685        &mut self,
19686        formula_dirty: &FormulaDirtyLease,
19687        retry_whole_spans: &[FormulaSpanRef],
19688        include_dirty_regions: bool,
19689    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19690        self.build_formula_plane_mixed_schedule_inner(
19691            formula_dirty,
19692            retry_whole_spans,
19693            include_dirty_regions,
19694            None,
19695        )
19696    }
19697
19698    fn build_formula_plane_target_schedule(
19699        &mut self,
19700        formula_dirty: &FormulaDirtyLease,
19701        retry_whole_spans: &[FormulaSpanRef],
19702        include_dirty_regions: bool,
19703        target_roots: &[crate::engine::target_preparation::TargetProducer],
19704    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19705        self.build_formula_plane_mixed_schedule_inner(
19706            formula_dirty,
19707            retry_whole_spans,
19708            include_dirty_regions,
19709            Some(target_roots),
19710        )
19711    }
19712
19713    fn build_formula_plane_mixed_schedule_inner(
19714        &mut self,
19715        formula_dirty: &FormulaDirtyLease,
19716        retry_whole_spans: &[FormulaSpanRef],
19717        include_dirty_regions: bool,
19718        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19719    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19720        let baseline = self.graph.baseline_stats();
19721        self.charge_bounded_work(
19722            baseline
19723                .graph_vertex_count
19724                .saturating_add(baseline.graph_edge_count) as u64,
19725        )?;
19726        let key = self.mixed_topology_cache_key();
19727        let active_refs = self
19728            .graph
19729            .formula_authority()
19730            .active_span_refs()
19731            .into_iter()
19732            .map(|span_ref| (span_ref.id, span_ref))
19733            .collect::<BTreeMap<_, _>>();
19734        let has_dynamic_legacy = self
19735            .graph
19736            .formula_vertices()
19737            .into_iter()
19738            .any(|vertex| self.graph.is_dynamic(vertex));
19739        let cache_hit = !has_dynamic_legacy
19740            && self
19741                .cached_mixed_topology
19742                .as_ref()
19743                .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
19744        let mut skipped_topology = None;
19745        let mut index_scratch_reserved = 0_u64;
19746        let mut demand_scratch_reserved = 0_u64;
19747        let topology_started = crate::instant::FzInstant::now();
19748        let (cache_outcome, strategy, compile_stats) = if cache_hit {
19749            self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
19750            self.mixed_topology_cache_skip_streak = 0;
19751            let cached = self
19752                .cached_mixed_topology
19753                .as_ref()
19754                .expect("cache hit has topology");
19755            let stats = cached.topology.stats.clone();
19756            let strategy = if cached.topology.is_complete() {
19757                FormulaPlaneTopologyStrategy::Cached
19758            } else {
19759                FormulaPlaneTopologyStrategy::ExactPagedIndexed
19760            };
19761            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19762                ledger
19763                    .account_mixed_cache(stats.estimated_memory_bytes as u64)
19764                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19765            }
19766            (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
19767        } else {
19768            // A key/cap miss makes the old retained topology stale. Drop it before building its
19769            // replacement so the request never accounts or owns both generations concurrently.
19770            self.cached_mixed_topology = None;
19771            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19772                ledger
19773                    .account_mixed_cache(0)
19774                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19775            }
19776
19777            let preflight = self.mixed_topology_index_preflight_bytes();
19778            if let Some(ledger) = self.active_resource_ledger.as_mut() {
19779                ledger
19780                    .reserve_schedule_discovery(preflight)
19781                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19782            }
19783            index_scratch_reserved = preflight;
19784            let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
19785                Ok(compiled) => compiled,
19786                Err(error) => {
19787                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19788                        let _ = ledger.release_scratch(index_scratch_reserved);
19789                    }
19790                    return Err(error);
19791                }
19792            };
19793            let actual_index_bytes = match &compiled {
19794                FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
19795                    .producer_results
19796                    .estimated_memory_bytes()
19797                    .and_then(|bytes| {
19798                        bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
19799                    })
19800                    .and_then(|bytes| {
19801                        bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
19802                    })
19803                    .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
19804                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
19805                    .producer_results
19806                    .estimated_memory_bytes()
19807                    .and_then(|bytes| {
19808                        bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
19809                    })
19810                    .and_then(|bytes| {
19811                        bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
19812                    })
19813                    .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
19814            }
19815            .unwrap_or(usize::MAX) as u64;
19816            debug_assert!(
19817                actual_index_bytes <= index_scratch_reserved,
19818                "mixed topology index preflight underestimated actual allocation"
19819            );
19820            if actual_index_bytes > index_scratch_reserved {
19821                let additional = actual_index_bytes - index_scratch_reserved;
19822                let reserve_result = self
19823                    .active_resource_ledger
19824                    .as_mut()
19825                    .map_or(Ok(()), |ledger| {
19826                        ledger.reserve_schedule_discovery(additional)
19827                    });
19828                if let Err(error) = reserve_result {
19829                    drop(compiled);
19830                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19831                        let _ = ledger.release_scratch(index_scratch_reserved);
19832                    }
19833                    return Err(error.into_excel_error());
19834                }
19835            } else if actual_index_bytes < index_scratch_reserved
19836                && let Some(ledger) = self.active_resource_ledger.as_mut()
19837            {
19838                ledger
19839                    .release_scratch(index_scratch_reserved - actual_index_bytes)
19840                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19841            }
19842            index_scratch_reserved = actual_index_bytes;
19843
19844            self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
19845            match compiled {
19846                FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
19847                    let stats = compiled.topology.stats.clone();
19848                    self.mixed_topology_cache_skip_streak =
19849                        self.mixed_topology_cache_skip_streak.saturating_add(1);
19850                    skipped_topology = Some(SkippedMixedTopology {
19851                        reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
19852                        observed: stats.clone(),
19853                        producer_results: compiled.producer_results,
19854                        consumer_reads: compiled.consumer_reads,
19855                        span_refs_by_id: compiled.span_refs_by_id,
19856                        plane_epoch: compiled.plane_epoch,
19857                        island: compiled.island,
19858                    });
19859                    (
19860                        FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
19861                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19862                        stats,
19863                    )
19864                }
19865                FormulaPlaneTopologyCompileResult::Cached(compiled) => {
19866                    let stats = compiled.topology.stats.clone();
19867                    let strategy = if compiled.topology.is_complete() {
19868                        FormulaPlaneTopologyStrategy::CompiledAndCached
19869                    } else {
19870                        FormulaPlaneTopologyStrategy::ExactPagedIndexed
19871                    };
19872                    let retained_result = self
19873                        .active_resource_ledger
19874                        .as_mut()
19875                        .map_or(Ok(()), |ledger| {
19876                            ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
19877                        });
19878                    if let Err(error) = retained_result {
19879                        if let Some(ledger) = self.active_resource_ledger.as_mut() {
19880                            let _ = ledger.release_scratch(index_scratch_reserved);
19881                        }
19882                        return Err(error.into_excel_error());
19883                    }
19884                    self.mixed_topology_cache_skip_streak = 0;
19885                    self.cached_mixed_topology = Some(compiled);
19886                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
19887                        ledger
19888                            .release_scratch(index_scratch_reserved)
19889                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
19890                    }
19891                    index_scratch_reserved = 0;
19892                    (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
19893                }
19894                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
19895                    debug_assert!(self.cached_mixed_topology.is_none());
19896                    self.mixed_topology_cache_overflows =
19897                        self.mixed_topology_cache_overflows.saturating_add(1);
19898                    self.mixed_topology_cache_skip_streak =
19899                        self.mixed_topology_cache_skip_streak.saturating_add(1);
19900                    let stats = skipped.observed.clone();
19901                    skipped_topology = Some(skipped);
19902                    (
19903                        FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
19904                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
19905                        stats,
19906                    )
19907                }
19908            }
19909        };
19910        self.observe_topology(
19911            cache_outcome,
19912            strategy,
19913            &compile_stats,
19914            topology_started.elapsed(),
19915        );
19916        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
19917            stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
19918            stats.topology.candidate_cap = Some(key.max_candidates as u64);
19919            stats.topology.edge_cap = Some(key.max_edges as u64);
19920            stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
19921            if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
19922                stats.topology.operator_guidance =
19923                    Some("dynamic legacy dependencies require an exact request topology rebuild");
19924            } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
19925                stats.topology.operator_guidance =
19926                    Some(if compile_stats.candidate_overflow_count > 0 {
19927                        "raise the mixed-cache candidate limit to avoid exact request rebuilds"
19928                    } else if compile_stats.edge_overflow_count > 0 {
19929                        "raise the mixed-cache edge limit to avoid exact request rebuilds"
19930                    } else {
19931                        "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
19932                    });
19933            }
19934        }
19935
19936        let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
19937        let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
19938            let (
19939                producer_results,
19940                consumer_reads,
19941                span_refs_by_id,
19942                plane_epoch,
19943                cached_topology,
19944                partial_topology,
19945                island,
19946            ) = if let Some(skipped) = skipped_topology.as_ref() {
19947                (
19948                    &skipped.producer_results,
19949                    &skipped.consumer_reads,
19950                    &skipped.span_refs_by_id,
19951                    skipped.plane_epoch,
19952                    None,
19953                    None,
19954                    &skipped.island,
19955                )
19956            } else {
19957                let cached = self
19958                    .cached_mixed_topology
19959                    .as_ref()
19960                    .expect("mixed topology available for request");
19961                let (complete, partial) = if cached.topology.is_complete() {
19962                    (Some(&cached.topology), None)
19963                } else {
19964                    (None, Some(&cached.topology))
19965                };
19966                (
19967                    &cached.producer_results,
19968                    &cached.consumer_reads,
19969                    &cached.span_refs_by_id,
19970                    cached.plane_epoch,
19971                    complete,
19972                    partial,
19973                    &cached.island,
19974                )
19975            };
19976            let demand = if let Some(target_roots) = target_roots {
19977                use crate::engine::target_preparation::TargetProducer;
19978                let mut roots = Vec::new();
19979                let mut graph_roots = Vec::new();
19980                for root in target_roots {
19981                    match *root {
19982                        TargetProducer::Span { span_ref, demanded } => {
19983                            roots.push((FormulaProducerId::Span(span_ref.id), demanded))
19984                        }
19985                        TargetProducer::Legacy(vertex) => {
19986                            graph_roots.push(vertex);
19987                            if let Some(region) = producer_results
19988                                .producer_result_region(FormulaProducerId::Legacy(vertex))
19989                            {
19990                                roots.push((FormulaProducerId::Legacy(vertex), region));
19991                            }
19992                        }
19993                        TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
19994                        TargetProducer::ValueOnly(_) => {}
19995                    }
19996                }
19997                if !graph_roots.is_empty() {
19998                    let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
19999                    for vertex in legacy_demand {
20000                        if let Some(region) = producer_results
20001                            .producer_result_region(FormulaProducerId::Legacy(vertex))
20002                        {
20003                            roots.push((FormulaProducerId::Legacy(vertex), region));
20004                        }
20005                    }
20006                }
20007                let roots = roots;
20008                let closure_base = (producer_results.len() as u64)
20009                    .saturating_mul(128)
20010                    .saturating_add(4096);
20011                let scratch_estimates =
20012                    exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
20013                let native_allowed = native_exact_demand_allowed(
20014                    self.active_resource_ledger
20015                        .as_ref()
20016                        .and_then(ResourceLedger::disk_scratch_policy),
20017                    !cfg!(target_arch = "wasm32"),
20018                );
20019                let (selected_strategy, reservation) = if cached_topology.is_some() {
20020                    (None, closure_base)
20021                } else {
20022                    let (strategy, reservation) =
20023                        select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
20024                            self.can_reserve_topology_scratch(bytes)
20025                        });
20026                    (Some(strategy), reservation)
20027                };
20028                if let Some(ledger) = self.active_resource_ledger.as_mut() {
20029                    ledger
20030                        .reserve_schedule_discovery(reservation)
20031                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20032                }
20033                demand_scratch_reserved = reservation;
20034                #[allow(unused_mut)]
20035                let mut native_disk_bytes = 0_u64;
20036                let (demand, actual_strategy, passes) = match selected_strategy {
20037                    None => (
20038                        build_demand_closure_cached(
20039                            roots,
20040                            producer_results,
20041                            cached_topology.expect("cached demand strategy has topology"),
20042                        ),
20043                        None,
20044                        0,
20045                    ),
20046                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20047                        let (demand, passes) = build_demand_closure_paged(
20048                            roots,
20049                            producer_results,
20050                            consumer_reads,
20051                            |units| {
20052                                Self::resource_loop_checkpoint(
20053                                    &mut self.active_resource_ledger,
20054                                    units,
20055                                )
20056                            },
20057                        )?;
20058                        (demand, selected_strategy, passes)
20059                    }
20060                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20061                        let (demand, passes) = build_demand_closure_in_memory_runs(
20062                            roots,
20063                            producer_results,
20064                            consumer_reads,
20065                            |units| {
20066                                Self::resource_loop_checkpoint(
20067                                    &mut self.active_resource_ledger,
20068                                    units,
20069                                )
20070                            },
20071                        )?;
20072                        (demand, selected_strategy, passes)
20073                    }
20074                    #[cfg(not(target_arch = "wasm32"))]
20075                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20076                        let request_id = self
20077                            .active_evaluation_resource_request
20078                            .as_ref()
20079                            .map_or(0, |stats| stats.request_id);
20080                        match NativeTopologyScratch::create(request_id) {
20081                            Ok(mut native) => match build_demand_closure_native(
20082                                roots.clone(),
20083                                producer_results,
20084                                consumer_reads,
20085                                native.file.as_mut().expect("native scratch file"),
20086                                native
20087                                    .auxiliary_file
20088                                    .as_mut()
20089                                    .expect("native auxiliary scratch file"),
20090                                |units| {
20091                                    Self::resource_loop_checkpoint(
20092                                        &mut self.active_resource_ledger,
20093                                        units,
20094                                    )
20095                                },
20096                            ) {
20097                                Ok((demand, passes, bytes)) => {
20098                                    native_disk_bytes = bytes;
20099                                    (demand, selected_strategy, passes)
20100                                }
20101                                Err(NativeExactDemandError::Work(error)) => return Err(error),
20102                                Err(NativeExactDemandError::Io(_)) => {
20103                                    let (demand, passes) = build_demand_closure_repeated_passes(
20104                                        roots,
20105                                        producer_results,
20106                                        consumer_reads,
20107                                        |units| {
20108                                            Self::resource_loop_checkpoint(
20109                                                &mut self.active_resource_ledger,
20110                                                units,
20111                                            )
20112                                        },
20113                                    )?;
20114                                    (
20115                                        demand,
20116                                        Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20117                                        passes,
20118                                    )
20119                                }
20120                            },
20121                            Err(_) => {
20122                                let (demand, passes) = build_demand_closure_repeated_passes(
20123                                    roots,
20124                                    producer_results,
20125                                    consumer_reads,
20126                                    |units| {
20127                                        Self::resource_loop_checkpoint(
20128                                            &mut self.active_resource_ledger,
20129                                            units,
20130                                        )
20131                                    },
20132                                )?;
20133                                (
20134                                    demand,
20135                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20136                                    passes,
20137                                )
20138                            }
20139                        }
20140                    }
20141                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20142                        let (demand, passes) = build_demand_closure_repeated_passes(
20143                            roots,
20144                            producer_results,
20145                            consumer_reads,
20146                            |units| {
20147                                Self::resource_loop_checkpoint(
20148                                    &mut self.active_resource_ledger,
20149                                    units,
20150                                )
20151                            },
20152                        )?;
20153                        (demand, selected_strategy, passes)
20154                    }
20155                    _ => unreachable!("exact demand strategy selection is exhaustive"),
20156                };
20157                let strategy_memory = match actual_strategy {
20158                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
20159                        scratch_estimates.paged.saturating_sub(closure_base)
20160                    }
20161                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
20162                        scratch_estimates.runs.saturating_sub(closure_base)
20163                    }
20164                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
20165                        scratch_estimates.native.saturating_sub(closure_base)
20166                    }
20167                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
20168                        scratch_estimates.repeated.saturating_sub(closure_base)
20169                    }
20170                    None => 0,
20171                    _ => 0,
20172                };
20173                let actual_scratch = demand
20174                    .estimated_memory_bytes()
20175                    .saturating_add(strategy_memory);
20176                if actual_scratch > demand_scratch_reserved {
20177                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
20178                        ledger
20179                            .reserve_schedule_discovery(
20180                                actual_scratch.saturating_sub(demand_scratch_reserved),
20181                            )
20182                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20183                    }
20184                } else if actual_scratch < demand_scratch_reserved
20185                    && let Some(ledger) = self.active_resource_ledger.as_mut()
20186                {
20187                    ledger
20188                        .release_scratch(demand_scratch_reserved - actual_scratch)
20189                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20190                }
20191                demand_scratch_reserved = actual_scratch;
20192                if let Some(strategy) = actual_strategy
20193                    && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20194                {
20195                    if strategy.severity() > stats.topology.strategy.severity() {
20196                        stats.topology.strategy = strategy;
20197                    }
20198                    stats.topology.exact_pass_count = stats
20199                        .topology
20200                        .exact_pass_count
20201                        .saturating_add(passes.max(1));
20202                    stats.topology.native_topology_disk_bytes = stats
20203                        .topology
20204                        .native_topology_disk_bytes
20205                        .saturating_add(native_disk_bytes);
20206                }
20207                Some(demand)
20208            } else {
20209                None
20210            };
20211
20212            let dirty_legacy = self.graph.get_evaluation_vertices();
20213            let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
20214            let island_target_demand = target_roots.map(|roots| {
20215                use crate::engine::target_preparation::TargetProducer;
20216                let graph_roots = roots
20217                    .iter()
20218                    .filter_map(|root| match *root {
20219                        TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
20220                            Some(vertex)
20221                        }
20222                        TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
20223                    })
20224                    .collect::<Vec<_>>();
20225                self.build_demand_subgraph(&graph_roots)
20226                    .0
20227                    .into_iter()
20228                    .collect::<FxHashSet<_>>()
20229            });
20230            let mut island_plan = island.clone();
20231            island_plan.dirty_vertices = island_plan
20232                .membership
20233                .iter()
20234                .copied()
20235                .filter(|vertex| dirty_legacy_set.contains(vertex))
20236                .filter(|vertex| {
20237                    island_target_demand
20238                        .as_ref()
20239                        .is_none_or(|demand| demand.contains(vertex))
20240                })
20241                .collect();
20242            let mut work = Vec::new();
20243            let mut scheduled_legacy_vertices = Vec::new();
20244            let demanded_dirty = |producer: FormulaProducerId,
20245                                  dirty: ProducerDirtyDomain|
20246             -> Option<ProducerDirtyDomain> {
20247                let Some(demand) = demand.as_ref() else {
20248                    return Some(dirty);
20249                };
20250                let result = producer_results.producer_result_region(producer)?;
20251                dirty.intersect_demanded(result, demand.regions(producer))
20252            };
20253            for (span_ref, region) in formula_dirty.span_regions() {
20254                let producer = FormulaProducerId::Span(span_ref.id);
20255                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20256                    && let Some(dirty) =
20257                        demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
20258                {
20259                    work.push(FormulaProducerWork { producer, dirty });
20260                }
20261            }
20262            for span_ref in formula_dirty
20263                .whole_spans()
20264                .chain(retry_whole_spans.iter().copied())
20265            {
20266                let producer = FormulaProducerId::Span(span_ref.id);
20267                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
20268                    && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20269                {
20270                    work.push(FormulaProducerWork { producer, dirty });
20271                }
20272            }
20273            for vertex in dirty_legacy {
20274                let producer = FormulaProducerId::Legacy(vertex);
20275                if producer_results.producer_result_region(producer).is_some()
20276                    && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
20277                {
20278                    scheduled_legacy_vertices.push(vertex);
20279                    work.push(FormulaProducerWork { producer, dirty });
20280                }
20281            }
20282
20283            let pending_changed_regions = formula_dirty
20284                .regions()
20285                .chain(formula_dirty.span_regions().map(|(_, region)| region))
20286                .collect::<Vec<_>>();
20287            if include_dirty_regions && !pending_changed_regions.is_empty() {
20288                use crate::formula_plane::producer::compute_dirty_closure_checked;
20289                let closure = compute_dirty_closure_checked(
20290                    consumer_reads,
20291                    pending_changed_regions.iter().copied(),
20292                    |producer| producer_results.producer_result_region(producer),
20293                    |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
20294                )?;
20295                if closure.incomplete {
20296                    // The fixed-point guard discards partial closure output. Complete conservatively
20297                    // in the same topology/schedule path; it must not become a new demotion route.
20298                    work.clear();
20299                    scheduled_legacy_vertices.clear();
20300                    if let Some(demand) = demand.as_ref() {
20301                        for producer in demand.producers() {
20302                            if let Some(dirty) =
20303                                demanded_dirty(producer, ProducerDirtyDomain::Whole)
20304                            {
20305                                if let FormulaProducerId::Legacy(vertex) = producer {
20306                                    scheduled_legacy_vertices.push(vertex);
20307                                }
20308                                work.push(FormulaProducerWork { producer, dirty });
20309                            }
20310                        }
20311                    } else {
20312                        work.extend(span_refs_by_id.keys().copied().map(|id| {
20313                            FormulaProducerWork {
20314                                producer: FormulaProducerId::Span(id),
20315                                dirty: ProducerDirtyDomain::Whole,
20316                            }
20317                        }));
20318                        for vertex in self.graph.formula_vertices() {
20319                            let producer = FormulaProducerId::Legacy(vertex);
20320                            if producer_results.producer_result_region(producer).is_some() {
20321                                scheduled_legacy_vertices.push(vertex);
20322                                work.push(FormulaProducerWork {
20323                                    producer,
20324                                    dirty: ProducerDirtyDomain::Whole,
20325                                });
20326                            }
20327                        }
20328                    }
20329                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20330                        stats.topology.incomplete_reason =
20331                            Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
20332                    }
20333                } else if let Some(demand) = demand.as_ref() {
20334                    for item in closure.work {
20335                        if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
20336                            work.push(FormulaProducerWork {
20337                                producer: item.producer,
20338                                dirty,
20339                            });
20340                        }
20341                    }
20342                } else {
20343                    work.extend(closure.work);
20344                }
20345                if !closure.incomplete && !closure.fallbacks.is_empty() {
20346                    let mut already_whole = work
20347                        .iter()
20348                        .filter_map(|item| match (item.producer, &item.dirty) {
20349                            (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
20350                            _ => None,
20351                        })
20352                        .collect::<rustc_hash::FxHashSet<_>>();
20353                    for fallback in closure.fallbacks {
20354                        if let FormulaProducerId::Span(id) = fallback.consumer
20355                            && already_whole.insert(id)
20356                        {
20357                            work.push(FormulaProducerWork {
20358                                producer: FormulaProducerId::Span(id),
20359                                dirty: ProducerDirtyDomain::Whole,
20360                            });
20361                        }
20362                    }
20363                }
20364            }
20365
20366            let owned_dirty_events = if let Some(demand) = demand.as_ref() {
20367                use crate::formula_plane::producer::compute_dirty_closure_checked;
20368                let mut owned = Vec::new();
20369                for (index, event) in formula_dirty.events() {
20370                    let (seed_regions, initial) = match event {
20371                        FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
20372                        FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
20373                            vec![region],
20374                            Some(FormulaProducerWork {
20375                                producer: FormulaProducerId::Span(span_ref.id),
20376                                dirty: ProducerDirtyDomain::Regions(vec![region]),
20377                            }),
20378                        ),
20379                        FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
20380                            let producer = FormulaProducerId::Span(span_ref.id);
20381                            let Some(result) = producer_results.producer_result_region(producer)
20382                            else {
20383                                continue;
20384                            };
20385                            (
20386                                vec![result],
20387                                Some(FormulaProducerWork {
20388                                    producer,
20389                                    dirty: ProducerDirtyDomain::Whole,
20390                                }),
20391                            )
20392                        }
20393                    };
20394                    let closure = compute_dirty_closure_checked(
20395                        consumer_reads,
20396                        seed_regions,
20397                        |producer| producer_results.producer_result_region(producer),
20398                        |units| {
20399                            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
20400                        },
20401                    )?;
20402                    if closure.incomplete || !closure.fallbacks.is_empty() {
20403                        continue;
20404                    }
20405                    let fully_demanded =
20406                        initial
20407                            .into_iter()
20408                            .chain(closure.work.into_iter())
20409                            .all(|item| {
20410                                let Some(result) =
20411                                    producer_results.producer_result_region(item.producer)
20412                                else {
20413                                    return false;
20414                                };
20415                                item.dirty
20416                                    .result_regions(result)
20417                                    .into_iter()
20418                                    .all(|dirty_region| {
20419                                        demand
20420                                            .regions(item.producer)
20421                                            .iter()
20422                                            .any(|demanded| demanded.contains_region(dirty_region))
20423                                    })
20424                            });
20425                    if fully_demanded {
20426                        owned.push(index);
20427                    }
20428                }
20429                owned
20430            } else {
20431                (0..formula_dirty.len()).collect()
20432            };
20433
20434            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
20435            let producer_count = producer_results.len();
20436            let dirty_units = work.iter().fold(0usize, |total, item| {
20437                total.saturating_add(match &item.dirty {
20438                    ProducerDirtyDomain::Whole => 0,
20439                    ProducerDirtyDomain::Cells(cells) => cells.len(),
20440                    ProducerDirtyDomain::Regions(regions) => regions.len(),
20441                })
20442            });
20443            let work_bytes = work
20444                .len()
20445                .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
20446                .saturating_add(
20447                    dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
20448                        std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
20449                    )),
20450                ) as u64;
20451            let binding_bytes =
20452                estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
20453            let schedule_bytes = work_bytes
20454                .saturating_mul(4)
20455                .saturating_add((producer_count as u64).saturating_mul(192))
20456                .saturating_add(binding_bytes)
20457                .saturating_add(1024);
20458            let maximum_edges =
20459                producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
20460            // Producer/read indexes are either retained by the cache or covered by the live index
20461            // preflight reservation. Strategy reservations therefore cover only additional scheduling
20462            // structures and must not charge those indexes a second time.
20463            let paged_bytes = schedule_bytes
20464                .saturating_add(maximum_edges.saturating_mul(160))
20465                .saturating_add(16 * 1024);
20466            let runs_bytes = schedule_bytes
20467                .saturating_add(maximum_edges.saturating_mul(48))
20468                .saturating_add(8192);
20469            let native_bytes = schedule_bytes
20470                .saturating_add((producer_count as u64).saturating_mul(128))
20471                .saturating_add(2048);
20472            let repeated_bytes = schedule_bytes
20473                .saturating_add((producer_count as u64).saturating_mul(96))
20474                .saturating_add(1024);
20475            let native_allowed = self
20476                .active_resource_ledger
20477                .as_ref()
20478                .and_then(ResourceLedger::disk_scratch_policy)
20479                == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
20480                && !cfg!(target_arch = "wasm32");
20481
20482            let cached_schedule_bytes = schedule_bytes
20483                .saturating_add(
20484                    cached_topology
20485                        .map_or(0, |topology| topology.stats.relationships as u64)
20486                        .saturating_mul(160),
20487                )
20488                .saturating_add(1024);
20489            let use_cached_schedule = cached_topology.is_some()
20490                && self.can_reserve_topology_scratch(cached_schedule_bytes);
20491            let (selected_strategy, scratch) = if use_cached_schedule {
20492                (None, cached_schedule_bytes)
20493            } else if self.can_reserve_topology_scratch(paged_bytes) {
20494                (
20495                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
20496                    paged_bytes,
20497                )
20498            } else if self.can_reserve_topology_scratch(runs_bytes) {
20499                (
20500                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
20501                    runs_bytes,
20502                )
20503            } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
20504                (
20505                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20506                    native_bytes,
20507                )
20508            } else if self.can_reserve_topology_scratch(repeated_bytes) {
20509                (
20510                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20511                    repeated_bytes,
20512                )
20513            } else {
20514                if let Some(ledger) = self.active_resource_ledger.as_mut() {
20515                    ledger
20516                        .reserve_schedule_discovery(repeated_bytes)
20517                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20518                }
20519                unreachable!("failed scratch reservation must return a typed error")
20520            };
20521            if let Some(ledger) = self.active_resource_ledger.as_mut() {
20522                ledger
20523                    .reserve_schedule_discovery(scratch)
20524                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20525            }
20526            let request_id = self
20527                .active_evaluation_resource_request
20528                .as_ref()
20529                .map_or(0, |stats| stats.request_id);
20530            let build_result = (|| -> Result<_, ExcelError> {
20531                let (schedule, actual_strategy, passes, native_disk_bytes) =
20532                    match (cached_topology, selected_strategy) {
20533                        (Some(topology), None) => (
20534                            schedule_dirty_work(work, producer_results, topology, 256),
20535                            None,
20536                            0,
20537                            0,
20538                        ),
20539                        (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
20540                            let (schedule, passes) = schedule_dirty_work_paged_hybrid(
20541                                work,
20542                                producer_results,
20543                                consumer_reads,
20544                                partial_topology.filter(|_| reuse_partial_topology),
20545                                256,
20546                                |units| {
20547                                    Self::resource_loop_checkpoint(
20548                                        &mut self.active_resource_ledger,
20549                                        units,
20550                                    )
20551                                },
20552                            )?;
20553                            (schedule, selected_strategy, passes.max(1), 0)
20554                        }
20555                        (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
20556                            let (schedule, passes) = schedule_dirty_work_in_memory_runs(
20557                                work,
20558                                producer_results,
20559                                consumer_reads,
20560                                256,
20561                                |units| {
20562                                    Self::resource_loop_checkpoint(
20563                                        &mut self.active_resource_ledger,
20564                                        units,
20565                                    )
20566                                },
20567                            )?;
20568                            (schedule, selected_strategy, passes.max(1), 0)
20569                        }
20570                        #[cfg(not(target_arch = "wasm32"))]
20571                        (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
20572                            let native = NativeTopologyScratch::create(request_id);
20573                            if let Ok(mut native) = native {
20574                                match schedule_dirty_work_native(
20575                                    work.clone(),
20576                                    producer_results,
20577                                    consumer_reads,
20578                                    256,
20579                                    native.file.as_mut().expect("native scratch file"),
20580                                    |units| {
20581                                        Self::resource_loop_checkpoint(
20582                                            &mut self.active_resource_ledger,
20583                                            units,
20584                                        )
20585                                    },
20586                                ) {
20587                                    Ok((schedule, passes, topology_bytes)) => (
20588                                        schedule,
20589                                        Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
20590                                        passes.max(1),
20591                                        topology_bytes,
20592                                    ),
20593                                    Err(NativeExactScheduleError::Work(error)) => {
20594                                        return Err(error);
20595                                    }
20596                                    Err(NativeExactScheduleError::Io(_)) => {
20597                                        let (schedule, passes) =
20598                                            schedule_dirty_work_repeated_passes(
20599                                                work,
20600                                                producer_results,
20601                                                consumer_reads,
20602                                                256,
20603                                                |units| {
20604                                                    Self::resource_loop_checkpoint(
20605                                                        &mut self.active_resource_ledger,
20606                                                        units,
20607                                                    )
20608                                                },
20609                                            )?;
20610                                        (
20611                                            schedule,
20612                                            Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20613                                            passes.max(1),
20614                                            0,
20615                                        )
20616                                    }
20617                                }
20618                            } else {
20619                                let (schedule, passes) = schedule_dirty_work_repeated_passes(
20620                                    work,
20621                                    producer_results,
20622                                    consumer_reads,
20623                                    256,
20624                                    |units| {
20625                                        Self::resource_loop_checkpoint(
20626                                            &mut self.active_resource_ledger,
20627                                            units,
20628                                        )
20629                                    },
20630                                )?;
20631                                (
20632                                    schedule,
20633                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
20634                                    passes.max(1),
20635                                    0,
20636                                )
20637                            }
20638                        }
20639                        (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
20640                            let (schedule, passes) = schedule_dirty_work_repeated_passes(
20641                                work,
20642                                producer_results,
20643                                consumer_reads,
20644                                256,
20645                                |units| {
20646                                    Self::resource_loop_checkpoint(
20647                                        &mut self.active_resource_ledger,
20648                                        units,
20649                                    )
20650                                },
20651                            )?;
20652                            (schedule, selected_strategy, passes.max(1), 0)
20653                        }
20654                        _ => unreachable!(
20655                            "topology cache and exact strategy selection are exhaustive"
20656                        ),
20657                    };
20658                Ok((
20659                    schedule,
20660                    actual_strategy,
20661                    passes,
20662                    native_disk_bytes,
20663                    span_refs_by_id.clone(),
20664                ))
20665            })();
20666            let release_result = self
20667                .active_resource_ledger
20668                .as_mut()
20669                .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
20670                .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20671            let (schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
20672                match build_result {
20673                    Ok(output) => {
20674                        release_result?;
20675                        output
20676                    }
20677                    Err(error) => {
20678                        let _ = release_result;
20679                        return Err(error);
20680                    }
20681                };
20682            if let Some(strategy) = exact_strategy
20683                && let Some(stats) = self.active_evaluation_resource_request.as_mut()
20684            {
20685                if strategy.severity() > stats.topology.strategy.severity() {
20686                    stats.topology.strategy = strategy;
20687                }
20688                stats.topology.exact_pass_count = pass_count;
20689                stats.topology.native_topology_disk_bytes = stats
20690                    .topology
20691                    .native_topology_disk_bytes
20692                    .saturating_add(native_disk_bytes);
20693            }
20694            Ok((
20695                schedule,
20696                result_span_refs,
20697                plane_epoch,
20698                scheduled_legacy_vertices,
20699                owned_dirty_events,
20700                island_plan,
20701            ))
20702        })();
20703        let scratch_release_result = self
20704            .active_resource_ledger
20705            .as_mut()
20706            .map_or(Ok(()), |ledger| {
20707                ledger
20708                    .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
20709            })
20710            .map_err(crate::engine::ResourceLedgerError::into_excel_error);
20711        match schedule_result {
20712            Ok(output) => {
20713                scratch_release_result?;
20714                Ok(output)
20715            }
20716            Err(error) => {
20717                let _ = scratch_release_result;
20718                Err(error)
20719            }
20720        }
20721    }
20722}
20723
20724impl<R> Engine<R>
20725where
20726    R: EvaluationContext,
20727{
20728    /// The [`Region`] a compressed range dependency covers.
20729    ///
20730    /// `context_sheet` is the sheet the dependency's formula lives on, which is
20731    /// what a `Current` locator means. It must be supplied by the caller: this
20732    /// method has no way to recover it, and substituting the workbook default
20733    /// sheet pointed the region at an unrelated sheet (issue #110). `None` means
20734    /// "no region pattern"; callers treat that as an incomplete summary and fall
20735    /// back to the conservative global path.
20736    fn shared_range_to_region_pattern(
20737        &self,
20738        range: &crate::reference::SharedRangeRef<'static>,
20739        context_sheet: SheetId,
20740    ) -> Result<Option<Region>, ExcelError> {
20741        assert!(
20742            matches!(range.sheet, crate::reference::SharedSheetLocator::Id(_)),
20743            "PROBE: non-Id locator reached shared_range_to_region_pattern: {:?}",
20744            range.sheet
20745        );
20746        let Ok(sheet_id) = self.resolve_sheet_locator(&range.sheet, context_sheet) else {
20747            // Unresolvable sheet name: no trustworthy region.
20748            return Ok(None);
20749        };
20750        match (
20751            range.start_row,
20752            range.end_row,
20753            range.start_col,
20754            range.end_col,
20755        ) {
20756            (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
20757                sheet_id, sr.index, er.index, sc.index, ec.index,
20758            ))),
20759            (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
20760                Ok(Some(Region::whole_col(sheet_id, sc.index)))
20761            }
20762            (Some(sr), Some(er), None, None) if sr.index == er.index => {
20763                Ok(Some(Region::whole_row(sheet_id, sr.index)))
20764            }
20765            _ => Ok(None),
20766        }
20767    }
20768
20769    /// Evaluate all dirty/volatile vertices
20770    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
20771        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
20772            engine.evaluate_all_unobserved()
20773        })
20774    }
20775
20776    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
20777        debug_assert!(
20778            !self.graph.deferred_dirty_active(),
20779            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
20780             was not balanced by end_deferred_dirty"
20781        );
20782        self.observe_function_semantic_epoch()?;
20783        self.lookup_index_cache.reset_counters();
20784        let _source_cache = self.source_cache_session();
20785        self.validate_deterministic_mode()?;
20786        if self.config.defer_graph_building {
20787            // Build graph for all staged formulas before evaluating
20788            self.build_graph_all()?;
20789        }
20790        self.evaluate_all_coordinator()
20791    }
20792
20793    /// Central FormulaPlane-aware coordinator for `evaluate_all`. In
20794    /// `AuthoritativeExperimental` mode every call enters the FormulaPlane
20795    /// coordinator; the coordinator itself composes with private legacy
20796    /// primitives for legacy-only work.
20797    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
20798        self.begin_evaluation_request();
20799        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
20800            return self.evaluate_authoritative_formula_plane_all();
20801        }
20802        self.evaluate_all_legacy_impl()
20803    }
20804
20805    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
20806    /// at its position and evaluate each layer (parallel when enabled).
20807    ///
20808    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
20809    /// number of Cycle units walked (the former `schedule.cycles.len()`).
20810    fn legacy_pass_run_units(
20811        &mut self,
20812        schedule: &crate::engine::scheduler::Schedule,
20813    ) -> Result<(usize, usize), ExcelError> {
20814        let mut computed_vertices = 0;
20815        let mut cycle_count = 0;
20816        for &unit in &schedule.units {
20817            match unit {
20818                ScheduleUnit::Cycle(i) => {
20819                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
20820                        cycle_count += 1;
20821                    }
20822                }
20823                ScheduleUnit::Layer(i) => {
20824                    let layer = schedule.unit_layer(i);
20825                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20826                        computed_vertices += self.evaluate_layer_parallel(layer)?;
20827                    } else {
20828                        computed_vertices += self.evaluate_layer_sequential(layer)?;
20829                    }
20830                }
20831            }
20832        }
20833        Ok((computed_vertices, cycle_count))
20834    }
20835
20836    /// Execute a proven span-disconnected legacy island without beginning or
20837    /// finalising an evaluation request. The FormulaPlane coordinator retains
20838    /// sole ownership of the dirty lease, volatile redirty, acknowledgement,
20839    /// clock sample, and recalc-epoch advance.
20840    fn evaluate_legacy_island_non_finalizing(
20841        &mut self,
20842        vertices: &[VertexId],
20843        mut delta: Option<&mut DeltaCollector>,
20844    ) -> Result<(usize, usize), ExcelError> {
20845        if vertices.is_empty() {
20846            return Ok((0, 0));
20847        }
20848        let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
20849        let mut computed = 0usize;
20850        let mut cycles = 0usize;
20851        for &unit in &schedule.units {
20852            self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
20853            match unit {
20854                ScheduleUnit::Cycle(index) => {
20855                    if self.handle_cycle_unit(
20856                        schedule.unit_cycle(index),
20857                        delta.as_deref_mut(),
20858                        None,
20859                        None,
20860                    )? > 0
20861                    {
20862                        cycles = cycles.saturating_add(1);
20863                    }
20864                }
20865                ScheduleUnit::Layer(index) => {
20866                    let layer = schedule.unit_layer(index);
20867                    let evaluated = if let Some(delta) = delta.as_deref_mut() {
20868                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20869                            self.evaluate_layer_parallel_with_delta(layer, delta)?
20870                        } else {
20871                            self.evaluate_layer_sequential_with_delta(layer, delta)?
20872                        }
20873                    } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
20874                        self.evaluate_layer_parallel(layer)?
20875                    } else {
20876                        self.evaluate_layer_sequential(layer)?
20877                    };
20878                    computed = computed.saturating_add(evaluated);
20879                }
20880            }
20881        }
20882        self.graph.clear_dirty_flags(vertices);
20883        Ok((computed, cycles))
20884    }
20885
20886    /// Legacy `evaluate_all` body, reachable from the FormulaPlane coordinator
20887    /// when no active spans exist or FormulaPlane authority is not in
20888    /// `AuthoritativeExperimental` mode. This is now an internal primitive; it
20889    /// must not be invoked directly from public APIs.
20890    ///
20891    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
20892    /// per-recalc clock sample): the FormulaPlane coordinator composes this
20893    /// primitive *after* `evaluate_legacy_cycle_prepass` may have accumulated
20894    /// counts (G8 demotion path), and both sub-passes belong to ONE request /
20895    /// one clock sample; request begin happens at the public entry points /
20896    /// coordinators instead.
20897    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
20898        self.reset_virtual_dep_telemetry_if_disabled();
20899        #[cfg(feature = "tracing")]
20900        let _span_eval = tracing::info_span!("evaluate_all").entered();
20901        let start = crate::instant::FzInstant::now();
20902        let mut computed_vertices = 0;
20903        let mut cycle_errors = 0;
20904        let mut replan_iterations = 0;
20905        const MAX_REPLAN: usize = 5;
20906        let mut telemetry = self
20907            .config
20908            .enable_virtual_dep_telemetry
20909            .then(|| self.start_virtual_dep_telemetry());
20910
20911        loop {
20912            let to_evaluate = self.graph.get_evaluation_vertices();
20913            if to_evaluate.is_empty() {
20914                if let Some(t) = telemetry.as_mut()
20915                    && t.bailout_reason.is_none()
20916                {
20917                    t.bailout_reason = Some("no_work");
20918                }
20919                break;
20920            }
20921
20922            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
20923            if let Some(t) = telemetry.as_mut() {
20924                Self::accumulate_schedule_meta(t, &meta);
20925            }
20926
20927            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
20928            computed_vertices += pass_computed;
20929            cycle_errors += pass_cycles;
20930
20931            // Check if dynamic dependencies changed
20932            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
20933            if let Some(t) = telemetry.as_mut() {
20934                t.changed_vdeps_total += changed_vertices.len();
20935            }
20936
20937            self.resource_checkpoint(0)?;
20938            self.graph.clear_dirty_flags(&to_evaluate);
20939            for v in &changed_vertices {
20940                self.graph.set_dirty(*v, true);
20941            }
20942
20943            if changed_vertices.is_empty() {
20944                if let Some(t) = telemetry.as_mut() {
20945                    t.bailout_reason = Some("converged");
20946                }
20947                break;
20948            }
20949            if replan_iterations >= MAX_REPLAN {
20950                if let Some(mut t) = telemetry.take() {
20951                    t.bailout_reason = Some("max_replan");
20952                    t.replan_iterations = replan_iterations;
20953                    self.last_virtual_dep_telemetry = t;
20954                }
20955                return Err(
20956                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
20957                );
20958            }
20959
20960            replan_iterations += 1;
20961        }
20962
20963        if let Some(mut t) = telemetry {
20964            t.replan_iterations = replan_iterations;
20965            self.last_virtual_dep_telemetry = t;
20966        }
20967
20968        // Re-dirty volatile vertices for the next evaluation cycle
20969        self.redirty_for_next_recalc();
20970
20971        // Advance recalc epoch after a full evaluation pass finishes
20972        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20973
20974        Ok(EvalResult {
20975            computed_vertices,
20976            cycle_errors,
20977            elapsed: start.elapsed(),
20978        })
20979    }
20980
20981    pub fn evaluate_all_with_target_delta(
20982        &mut self,
20983    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
20984        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
20985            engine.observe_function_semantic_epoch()?;
20986            let mut collector = DeltaCollector::new(DeltaMode::Cells);
20987            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
20988            Ok((result, collector.finish_target()))
20989        })
20990    }
20991
20992    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
20993        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
20994    }
20995
20996    pub fn evaluate_all_with_delta_policy(
20997        &mut self,
20998        policy: EvalDeltaCompatibilityPolicy,
20999    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
21000        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
21001            engine.observe_function_semantic_epoch()?;
21002            let mut collector = DeltaCollector::new(DeltaMode::Cells);
21003            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
21004            Ok((result, collector.finish_with_policy(policy)?))
21005        })
21006    }
21007
21008    fn evaluate_all_with_delta_collector(
21009        &mut self,
21010        delta: &mut DeltaCollector,
21011    ) -> Result<EvalResult, ExcelError> {
21012        self.begin_evaluation_request();
21013        let _source_cache = self.source_cache_session();
21014        if self.config.defer_graph_building {
21015            self.build_graph_all()?;
21016        }
21017        if self.graph.formula_authority().active_span_count() > 0 {
21018            return self.evaluate_authoritative_formula_plane(None, Some(delta));
21019        }
21020        self.reset_virtual_dep_telemetry_if_disabled();
21021        #[cfg(feature = "tracing")]
21022        let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
21023        let start = crate::instant::FzInstant::now();
21024        let mut computed_vertices = 0;
21025        let mut cycle_errors = 0;
21026
21027        let mut replan_iterations = 0;
21028        const MAX_REPLAN: usize = 5;
21029        let mut telemetry = self
21030            .config
21031            .enable_virtual_dep_telemetry
21032            .then(|| self.start_virtual_dep_telemetry());
21033
21034        loop {
21035            let to_evaluate = self.graph.get_evaluation_vertices();
21036            if to_evaluate.is_empty() {
21037                if let Some(t) = telemetry.as_mut()
21038                    && t.bailout_reason.is_none()
21039                {
21040                    t.bailout_reason = Some("no_work");
21041                }
21042                break;
21043            }
21044
21045            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21046            if let Some(t) = telemetry.as_mut() {
21047                Self::accumulate_schedule_meta(t, &meta);
21048            }
21049
21050            for &unit in &schedule.units {
21051                match unit {
21052                    ScheduleUnit::Cycle(i) => {
21053                        if self.handle_cycle_unit(
21054                            schedule.unit_cycle(i),
21055                            Some(delta),
21056                            None,
21057                            None,
21058                        )? > 0
21059                        {
21060                            cycle_errors += 1;
21061                        }
21062                    }
21063                    ScheduleUnit::Layer(i) => {
21064                        let layer = schedule.unit_layer(i);
21065                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21066                            computed_vertices +=
21067                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
21068                        } else {
21069                            computed_vertices +=
21070                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
21071                        }
21072                    }
21073                }
21074            }
21075
21076            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21077            if let Some(t) = telemetry.as_mut() {
21078                t.changed_vdeps_total += changed_vertices.len();
21079            }
21080            self.resource_checkpoint(0)?;
21081            self.graph.clear_dirty_flags(&to_evaluate);
21082            for v in &changed_vertices {
21083                self.graph.set_dirty(*v, true);
21084            }
21085
21086            if changed_vertices.is_empty() {
21087                if let Some(t) = telemetry.as_mut() {
21088                    t.bailout_reason = Some("converged");
21089                }
21090                break;
21091            }
21092            if replan_iterations >= MAX_REPLAN {
21093                if let Some(mut t) = telemetry.take() {
21094                    t.bailout_reason = Some("max_replan");
21095                    t.replan_iterations = replan_iterations;
21096                    self.last_virtual_dep_telemetry = t;
21097                }
21098                return Err(
21099                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21100                );
21101            }
21102            replan_iterations += 1;
21103        }
21104
21105        if let Some(mut t) = telemetry {
21106            t.replan_iterations = replan_iterations;
21107            self.last_virtual_dep_telemetry = t;
21108        }
21109
21110        self.redirty_for_next_recalc();
21111        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21112
21113        Ok(EvalResult {
21114            computed_vertices,
21115            cycle_errors,
21116            elapsed: start.elapsed(),
21117        })
21118    }
21119
21120    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
21121    ///
21122    /// This will evaluate only the minimal set of dirty / volatile precedents required
21123    /// to bring the target cell up-to-date (as if a user asked for that single value),
21124    /// rather than scheduling a full workbook recalc. If the cell is already clean and
21125    /// non-volatile, no vertices will be recomputed.
21126    ///
21127    /// Returns the (possibly newly computed) value stored for the cell afterwards.
21128    /// Empty cells return None. Errors are surfaced via the Result type.
21129    pub fn evaluate_cell(
21130        &mut self,
21131        sheet: &str,
21132        row: u32,
21133        col: u32,
21134    ) -> Result<Option<LiteralValue>, ExcelError> {
21135        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
21136            engine.evaluate_cell_unobserved(sheet, row, col)
21137        })
21138    }
21139
21140    fn evaluate_cell_unobserved(
21141        &mut self,
21142        sheet: &str,
21143        row: u32,
21144        col: u32,
21145    ) -> Result<Option<LiteralValue>, ExcelError> {
21146        if row == 0 || col == 0 {
21147            return Err(ExcelError::new(ExcelErrorKind::Ref)
21148                .with_message("Row and column must be >= 1".to_string()));
21149        }
21150
21151        let result = self.evaluate_cells(&[(sheet, row, col)])?;
21152
21153        match result.len() {
21154            0 => Ok(None),
21155            1 => {
21156                let v = result.into_iter().next().unwrap();
21157                Ok(v)
21158            }
21159            _ => unreachable!("evaluate_cells returned unexpected length"),
21160        }
21161    }
21162
21163    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
21164    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
21165    /// is computed once and evaluated, which is typically faster than calling
21166    /// `evaluate_cell` repeatedly for a related set of targets.
21167    ///
21168    /// Returns the resulting values for each requested target in the same order.
21169    pub fn evaluate_cells(
21170        &mut self,
21171        targets: &[(&str, u32, u32)],
21172    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21173        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
21174            engine.evaluate_cells_unobserved(targets)
21175        })
21176    }
21177
21178    fn evaluate_cells_unobserved(
21179        &mut self,
21180        targets: &[(&str, u32, u32)],
21181    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21182        self.observe_function_semantic_epoch()?;
21183        debug_assert!(
21184            !self.graph.deferred_dirty_active(),
21185            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
21186             was not balanced by end_deferred_dirty"
21187        );
21188        self.validate_deterministic_mode()?;
21189        if targets.is_empty() {
21190            return Ok(Vec::new());
21191        }
21192        let typed_targets = self.legacy_coordinate_targets(targets);
21193        self.evaluate_mixed_targets(&typed_targets, None)?;
21194        Ok(targets
21195            .iter()
21196            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21197            .collect())
21198    }
21199
21200    pub fn evaluate_cells_cancellable(
21201        &mut self,
21202        targets: &[(&str, u32, u32)],
21203        cancel: crate::engine::CancelToken,
21204    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21205        self.observe_evaluation_resource_request(
21206            EvaluationRequestKind::CellsCancellable,
21207            |engine| {
21208                engine.observe_function_semantic_epoch()?;
21209                engine.active_cancel_flag = Some(cancel.clone());
21210                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
21211                engine.active_cancel_flag = None;
21212                res
21213            },
21214        )
21215    }
21216
21217    fn evaluate_cells_cancellable_impl(
21218        &mut self,
21219        targets: &[(&str, u32, u32)],
21220        cancel_flag: &AtomicBool,
21221    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
21222        self.validate_deterministic_mode()?;
21223        if targets.is_empty() {
21224            return Ok(Vec::new());
21225        }
21226        if cancel_flag.load(Ordering::Relaxed) {
21227            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21228                .with_message("Evaluation cancelled before target preparation"));
21229        }
21230        let typed_targets = self.legacy_coordinate_targets(targets);
21231        self.evaluate_mixed_targets(&typed_targets, None)?;
21232        Ok(targets
21233            .iter()
21234            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
21235            .collect())
21236    }
21237
21238    pub fn evaluate_cells_with_target_delta(
21239        &mut self,
21240        targets: &[(&str, u32, u32)],
21241    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
21242        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21243            engine.observe_function_semantic_epoch()?;
21244            engine.validate_deterministic_mode()?;
21245            if targets.is_empty() {
21246                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
21247            }
21248            let mut collector = DeltaCollector::new(DeltaMode::Cells);
21249            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
21250            let values = targets
21251                .iter()
21252                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
21253                .collect();
21254            Ok((values, collector.finish_target()))
21255        })
21256    }
21257
21258    pub fn evaluate_cells_with_delta(
21259        &mut self,
21260        targets: &[(&str, u32, u32)],
21261    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21262        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
21263    }
21264
21265    pub fn evaluate_cells_with_delta_policy(
21266        &mut self,
21267        targets: &[(&str, u32, u32)],
21268        policy: EvalDeltaCompatibilityPolicy,
21269    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21270        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
21271            engine.evaluate_cells_with_delta_unobserved(targets, policy)
21272        })
21273    }
21274
21275    fn evaluate_cells_with_delta_unobserved(
21276        &mut self,
21277        targets: &[(&str, u32, u32)],
21278        policy: EvalDeltaCompatibilityPolicy,
21279    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
21280        self.observe_function_semantic_epoch()?;
21281        self.validate_deterministic_mode()?;
21282        if targets.is_empty() {
21283            return Ok((Vec::new(), EvalDelta::default()));
21284        }
21285        let mut collector = DeltaCollector::new(DeltaMode::Cells);
21286        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
21287        let values = targets
21288            .iter()
21289            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
21290            .collect();
21291        Ok((values, collector.finish_with_policy(policy)?))
21292    }
21293
21294    /// Get the evaluation plan for target cells without actually evaluating them
21295    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
21296        if targets.is_empty() {
21297            return Ok(EvalPlan {
21298                total_vertices_to_evaluate: 0,
21299                layers: Vec::new(),
21300                cycles_detected: 0,
21301                dirty_count: 0,
21302                volatile_count: 0,
21303                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21304                estimated_parallel_layers: 0,
21305                target_cells: Vec::new(),
21306            });
21307        }
21308        if self.config.defer_graph_building && self.has_staged_formulas() {
21309            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
21310                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
21311            ));
21312        }
21313
21314        // Convert targets to A1 notation for consistency
21315        let addresses: Vec<String> = targets
21316            .iter()
21317            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
21318            .collect();
21319
21320        // Parse target cell addresses
21321        let mut target_addrs = Vec::new();
21322        for (sheet, row, col) in targets {
21323            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
21324                let coord = Coord::from_excel(*row, *col, true, true);
21325                target_addrs.push(CellRef::new(sheet_id, coord));
21326            }
21327        }
21328
21329        // Find vertex IDs for targets
21330        let mut target_vertex_ids = Vec::new();
21331        for addr in &target_addrs {
21332            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
21333                target_vertex_ids.push(*vertex_id);
21334            }
21335        }
21336
21337        if target_vertex_ids.is_empty() {
21338            return Ok(EvalPlan {
21339                total_vertices_to_evaluate: 0,
21340                layers: Vec::new(),
21341                cycles_detected: 0,
21342                dirty_count: 0,
21343                volatile_count: 0,
21344                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21345                estimated_parallel_layers: 0,
21346                target_cells: addresses,
21347            });
21348        }
21349
21350        // Build demand subgraph with virtual edges (same as evaluate_until)
21351        let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
21352
21353        if precedents_to_eval.is_empty() {
21354            return Ok(EvalPlan {
21355                total_vertices_to_evaluate: 0,
21356                layers: Vec::new(),
21357                cycles_detected: 0,
21358                dirty_count: 0,
21359                volatile_count: 0,
21360                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
21361                estimated_parallel_layers: 0,
21362                target_cells: addresses,
21363            });
21364        }
21365
21366        // Count dirty and volatile vertices
21367        let mut dirty_count = 0;
21368        let mut volatile_count = 0;
21369        for &vertex_id in &precedents_to_eval {
21370            if self.graph.is_dirty(vertex_id) {
21371                dirty_count += 1;
21372            }
21373            if self.graph.is_volatile(vertex_id) {
21374                volatile_count += 1;
21375            }
21376        }
21377
21378        // Create schedule for the minimal subgraph honoring virtual edges
21379        let scheduler = Scheduler::new(&self.graph);
21380        let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
21381
21382        // Build layer information
21383        let mut layers = Vec::new();
21384        let mut estimated_parallel_layers = 0;
21385        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
21386
21387        for layer in &schedule.layers {
21388            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
21389            if parallel_eligible {
21390                estimated_parallel_layers += 1;
21391            }
21392
21393            // Get sample cell addresses (up to 5)
21394            let sample_cells: Vec<String> = layer
21395                .vertices
21396                .iter()
21397                .take(5)
21398                .filter_map(|&vertex_id| {
21399                    self.graph
21400                        .get_cell_ref_for_vertex(vertex_id)
21401                        .map(|cell_ref| {
21402                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
21403                            format!(
21404                                "{}!{}{}",
21405                                sheet_name,
21406                                Self::col_to_letters(cell_ref.coord.col()),
21407                                cell_ref.coord.row() + 1
21408                            )
21409                        })
21410                })
21411                .collect();
21412
21413            layers.push(LayerInfo {
21414                vertex_count: layer.vertices.len(),
21415                parallel_eligible,
21416                sample_cells,
21417            });
21418        }
21419
21420        Ok(EvalPlan {
21421            total_vertices_to_evaluate: precedents_to_eval.len(),
21422            layers,
21423            cycles_detected: schedule.cycles.len(),
21424            dirty_count,
21425            volatile_count,
21426            parallel_enabled,
21427            estimated_parallel_layers,
21428            target_cells: addresses,
21429        })
21430    }
21431    /// Helper to create a schedule, integrating virtual dependencies automatically.
21432    fn create_evaluation_schedule(
21433        &mut self,
21434        to_evaluate: &[VertexId],
21435    ) -> Result<ScheduleBuildOutput, ExcelError> {
21436        // Fold pending edge deltas once per schedule build so traversal uses
21437        // the zero-allocation CSR slices (#125).
21438        self.graph.flush_pending_edge_deltas();
21439        if self.can_use_static_schedule_cache(to_evaluate) {
21440            if let Some(cached) = self.cached_static_schedule.as_ref()
21441                && cached.topology_epoch == self.topology_epoch
21442                && cached.candidate_vertices.as_slice() == to_evaluate
21443            {
21444                let meta = ScheduleBuildMeta {
21445                    candidate_vertices: to_evaluate.len(),
21446                    vdeps_vertices: 0,
21447                    vdeps_edges: 0,
21448                    builder_elapsed_ms: 0,
21449                    used_virtual_schedule: false,
21450                    schedule_cache_hit: true,
21451                    schedule_cache_eligible: true,
21452                };
21453                return Ok((cached.schedule.clone(), FxHashMap::default(), meta));
21454            }
21455
21456            let (schedule, vdeps, mut meta) =
21457                self.create_evaluation_schedule_uncached(to_evaluate)?;
21458            meta.schedule_cache_hit = false;
21459            meta.schedule_cache_eligible = true;
21460            if vdeps.is_empty() {
21461                self.cached_static_schedule = Some(CachedScheduleEntry {
21462                    topology_epoch: self.topology_epoch,
21463                    candidate_vertices: to_evaluate.to_vec(),
21464                    schedule: schedule.clone(),
21465                });
21466            }
21467            return Ok((schedule, vdeps, meta));
21468        }
21469
21470        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
21471        meta.schedule_cache_hit = false;
21472        meta.schedule_cache_eligible = false;
21473        Ok((schedule, vdeps, meta))
21474    }
21475
21476    fn create_evaluation_schedule_uncached(
21477        &self,
21478        to_evaluate: &[VertexId],
21479    ) -> Result<ScheduleBuildOutput, ExcelError> {
21480        let builder = VirtualDepBuilder::new(self);
21481        let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
21482            if self.config.enable_virtual_dep_telemetry {
21483                let build_started = crate::instant::FzInstant::now();
21484                let (vdeps, augmented) = builder.build(to_evaluate);
21485                let builder_elapsed_ms = build_started.elapsed().as_millis();
21486                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
21487                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
21488            } else {
21489                let (vdeps, augmented) = builder.build(to_evaluate);
21490                (vdeps, augmented, 0, 0)
21491            };
21492
21493        let mut final_evaluate = to_evaluate.to_vec();
21494        if !augmented.is_empty() {
21495            final_evaluate.extend(augmented);
21496            final_evaluate.sort_unstable();
21497            final_evaluate.dedup();
21498        }
21499
21500        let use_virtual = !vdeps.is_empty();
21501
21502        let scheduler = Scheduler::new(&self.graph);
21503        let schedule = if use_virtual {
21504            scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
21505        } else {
21506            scheduler.create_schedule(&final_evaluate)?
21507        };
21508
21509        let meta = ScheduleBuildMeta {
21510            candidate_vertices: to_evaluate.len(),
21511            vdeps_vertices: vdeps.len(),
21512            vdeps_edges,
21513            builder_elapsed_ms,
21514            used_virtual_schedule: use_virtual,
21515            schedule_cache_hit: false,
21516            schedule_cache_eligible: false,
21517        };
21518
21519        Ok((schedule, vdeps, meta))
21520    }
21521
21522    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
21523        !to_evaluate.is_empty()
21524            && to_evaluate.iter().copied().all(|v| {
21525                !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
21526            })
21527    }
21528
21529    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
21530        VirtualDepTelemetry {
21531            fallback_mode_activations: self.virtual_dep_fallback_activations,
21532            ..VirtualDepTelemetry::default()
21533        }
21534    }
21535
21536    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
21537        telemetry.candidate_vertices_total += meta.candidate_vertices;
21538        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
21539        telemetry.vdeps_edges_total += meta.vdeps_edges;
21540        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
21541        if meta.schedule_cache_eligible {
21542            if meta.schedule_cache_hit {
21543                telemetry.schedule_cache_hits += 1;
21544                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
21545            } else {
21546                telemetry.schedule_cache_misses += 1;
21547            }
21548        }
21549        if meta.used_virtual_schedule {
21550            telemetry.schedule_virtual_passes += 1;
21551        } else {
21552            telemetry.schedule_static_passes += 1;
21553        }
21554    }
21555
21556    fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
21557        vertices
21558            .iter()
21559            .copied()
21560            .filter(|vertex| self.graph.is_dynamic(*vertex))
21561            .filter_map(|vertex| {
21562                let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
21563                (!regions.is_empty()).then_some((vertex, regions))
21564            })
21565            .collect()
21566    }
21567
21568    fn extend_target_roots_with_dynamic_regions<'a>(
21569        &self,
21570        roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
21571        regions: impl Iterator<Item = &'a Region>,
21572    ) -> Result<(), ExcelError> {
21573        use crate::engine::target_preparation::TargetProducer;
21574        let request_id = self
21575            .active_evaluation_resource_request
21576            .as_ref()
21577            .map(|request| request.request_id);
21578        let root_count = roots.len();
21579        let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
21580            .map_err(|_| target_root_allocation_error(root_count, request_id))?;
21581        for region in regions {
21582            for vertex in self.graph.formula_vertices() {
21583                let key = crate::formula_plane::region_index::RegionKey {
21584                    sheet_id: self.graph.get_vertex_sheet_id(vertex),
21585                    row: self.graph.vertex_coord(vertex).row(),
21586                    col: self.graph.vertex_coord(vertex).col(),
21587                };
21588                if region.contains_key(key) {
21589                    extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
21590                        target_root_allocation_error(extended.len() + 1, request_id)
21591                    })?;
21592                }
21593            }
21594            let authority = self.graph.formula_authority();
21595            for span_ref in authority.active_span_refs() {
21596                let Some(span) = authority.plane.spans.get(span_ref) else {
21597                    continue;
21598                };
21599                let Some(demanded) =
21600                    Region::from_domain(span.result_region.domain()).intersection(*region)
21601                else {
21602                    continue;
21603                };
21604                extended
21605                    .push(TargetProducer::Span { span_ref, demanded })
21606                    .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
21607            }
21608        }
21609        *roots = extended.into_vec();
21610        Ok(())
21611    }
21612
21613    fn changed_virtual_dep_vertices(
21614        &mut self,
21615        to_evaluate: &[VertexId],
21616        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
21617    ) -> Vec<VertexId> {
21618        #[cfg(test)]
21619        if self.force_virtual_dep_changes_remaining_for_test > 0
21620            && let Some(vertex) = to_evaluate.first().copied()
21621        {
21622            self.force_virtual_dep_changes_remaining_for_test -= 1;
21623            return vec![vertex];
21624        }
21625        if !to_evaluate
21626            .iter()
21627            .copied()
21628            .any(|v| self.graph.is_dynamic(v))
21629        {
21630            return Vec::new();
21631        }
21632
21633        let builder = VirtualDepBuilder::new(self);
21634        let (new_vdeps, _) = builder.build(to_evaluate);
21635
21636        let mut candidates = FxHashSet::default();
21637        candidates.extend(old_vdeps.keys().copied());
21638        candidates.extend(new_vdeps.keys().copied());
21639
21640        let mut changed = Vec::new();
21641        for v in candidates {
21642            if old_vdeps.get(&v) != new_vdeps.get(&v) {
21643                changed.push(v);
21644            }
21645        }
21646        changed
21647    }
21648
21649    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
21650    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
21651    fn build_demand_subgraph(
21652        &self,
21653        target_vertices: &[VertexId],
21654    ) -> (
21655        Vec<VertexId>,
21656        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
21657    ) {
21658        #[cfg(feature = "tracing")]
21659        let _span =
21660            tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
21661        use rustc_hash::{FxHashMap, FxHashSet};
21662
21663        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
21664        let mut visited: FxHashSet<VertexId> = FxHashSet::default();
21665        let mut stack: Vec<VertexId> = Vec::new();
21666        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); // incoming deps per vertex
21667
21668        for &t in target_vertices {
21669            stack.push(t);
21670        }
21671
21672        while let Some(v) = stack.pop() {
21673            if !visited.insert(v) {
21674                continue;
21675            }
21676            if !self.graph.vertex_exists(v) {
21677                continue;
21678            }
21679            // Schedule dirty/volatile formulas. Also schedule pass-through
21680            // Named*/Range vertices so the scheduler honours the
21681            // topological position of any formula cells that sit underneath
21682            // them — without these in `vertex_set` the scheduler skips the
21683            // edges that route a target through a named-range vertex into
21684            // its underlying cells, and the underlying cells then end up
21685            // in the same (or an earlier) layer as the target.
21686            match self.graph.get_vertex_kind(v) {
21687                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
21688                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
21689                        to_evaluate.insert(v);
21690                    }
21691                }
21692                VertexKind::NamedScalar
21693                | VertexKind::NamedArray
21694                | VertexKind::Range
21695                | VertexKind::InfiniteRange => {
21696                    to_evaluate.insert(v);
21697                }
21698                _ => {}
21699            }
21700
21701            // Explicit dependencies (graph edges). We push *every* dep onto
21702            // the stack — not just formulas — because intermediate vertices
21703            // (NamedScalar, NamedArray, Range) are pass-through nodes whose
21704            // own dependencies point at the actual formula cells. Filtering
21705            // by kind here previously caused DN-range refs to be dropped
21706            // from the demand subgraph, so a target like
21707            // ``=SUM(named_range_pointing_at_dirty_cells)`` would evaluate
21708            // using stale values for those cells. The kind check at the top
21709            // of the loop still gates which vertices end up in
21710            // ``to_evaluate``; only Formula vertices are scheduled.
21711            if let Some(dependencies) = self.graph.dependencies_slice(v) {
21712                for &dep in dependencies {
21713                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21714                        stack.push(dep);
21715                    }
21716                }
21717            } else {
21718                for dep in self.graph.get_dependencies(v) {
21719                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
21720                        stack.push(dep);
21721                    }
21722                }
21723            } // Virtual dependencies (compressed ranges + dynamic like INDIRECT)
21724            let builder = VirtualDepBuilder::new(self);
21725            let (vdeps_map, _) = builder.build(&[v]);
21726            if let Some(deps) = vdeps_map.get(&v) {
21727                for &u in deps {
21728                    vdeps.entry(v).or_default().push(u);
21729                    if !visited.contains(&u) {
21730                        stack.push(u);
21731                    }
21732                }
21733            }
21734        }
21735
21736        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
21737        result.sort_unstable();
21738        // Dedup virtual deps
21739        for deps in vdeps.values_mut() {
21740            deps.sort_unstable();
21741            deps.dedup();
21742        }
21743        (result, vdeps)
21744    }
21745
21746    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
21747    fn col_to_letters(col: u32) -> String {
21748        col_letters_from_1based(col).expect("column index must be >= 1")
21749    }
21750
21751    /// Evaluate all dirty/volatile vertices with cancellation support
21752    pub fn evaluate_all_cancellable(
21753        &mut self,
21754        cancel: crate::engine::CancelToken,
21755    ) -> Result<EvalResult, ExcelError> {
21756        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
21757            engine.observe_function_semantic_epoch()?;
21758            engine.active_cancel_flag = Some(cancel.clone());
21759            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
21760            engine.active_cancel_flag = None;
21761            res
21762        })
21763    }
21764
21765    fn evaluate_all_cancellable_impl(
21766        &mut self,
21767        cancel_flag: &AtomicBool,
21768    ) -> Result<EvalResult, ExcelError> {
21769        self.begin_evaluation_request();
21770        let _source_cache = self.source_cache_session();
21771        self.validate_deterministic_mode()?;
21772        if self.config.defer_graph_building {
21773            self.build_graph_all()?;
21774        }
21775        if self.graph.formula_authority().active_span_count() > 0 {
21776            if cancel_flag.load(Ordering::Relaxed) {
21777                return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21778                    "Evaluation cancelled before FormulaPlane scheduling".to_string(),
21779                ));
21780            }
21781            return self.evaluate_authoritative_formula_plane_all();
21782        }
21783        self.reset_virtual_dep_telemetry_if_disabled();
21784        let start = crate::instant::FzInstant::now();
21785        let mut computed_vertices = 0;
21786        let mut cycle_errors = 0;
21787
21788        let mut replan_iterations = 0;
21789        const MAX_REPLAN: usize = 5;
21790        let mut telemetry = self
21791            .config
21792            .enable_virtual_dep_telemetry
21793            .then(|| self.start_virtual_dep_telemetry());
21794
21795        loop {
21796            if cancel_flag.load(Ordering::Relaxed) {
21797                if let Some(mut t) = telemetry {
21798                    t.bailout_reason = Some("cancelled");
21799                    t.replan_iterations = replan_iterations;
21800                    self.last_virtual_dep_telemetry = t;
21801                }
21802                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21803                    .with_message("Evaluation cancelled before scheduling".to_string()));
21804            }
21805
21806            let to_evaluate = self.graph.get_evaluation_vertices();
21807            if to_evaluate.is_empty() {
21808                if let Some(t) = telemetry.as_mut()
21809                    && t.bailout_reason.is_none()
21810                {
21811                    t.bailout_reason = Some("no_work");
21812                }
21813                break;
21814            }
21815
21816            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
21817            if let Some(t) = telemetry.as_mut() {
21818                Self::accumulate_schedule_meta(t, &meta);
21819            }
21820
21821            // Walk units in condensation order, checking cancellation between
21822            // units (formerly between cycles and between layers).
21823            for &unit in &schedule.units {
21824                match unit {
21825                    ScheduleUnit::Cycle(i) => {
21826                        // Check cancellation between cycles
21827                        if cancel_flag.load(Ordering::Relaxed) {
21828                            if let Some(mut t) = telemetry {
21829                                t.bailout_reason = Some("cancelled");
21830                                t.replan_iterations = replan_iterations;
21831                                self.last_virtual_dep_telemetry = t;
21832                            }
21833                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
21834                                "Evaluation cancelled during cycle handling".to_string(),
21835                            ));
21836                        }
21837
21838                        if self.handle_cycle_unit(
21839                            schedule.unit_cycle(i),
21840                            None,
21841                            None,
21842                            Some(cancel_flag),
21843                        )? > 0
21844                        {
21845                            cycle_errors += 1;
21846                        }
21847                    }
21848                    ScheduleUnit::Layer(i) => {
21849                        let layer = schedule.unit_layer(i);
21850                        // Check cancellation between layers
21851                        if cancel_flag.load(Ordering::Relaxed) {
21852                            if let Some(mut t) = telemetry {
21853                                t.bailout_reason = Some("cancelled");
21854                                t.replan_iterations = replan_iterations;
21855                                self.last_virtual_dep_telemetry = t;
21856                            }
21857                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21858                                .with_message("Evaluation cancelled between layers".to_string()));
21859                        }
21860
21861                        // Evaluate vertices in this layer (parallel or sequential)
21862                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
21863                            computed_vertices +=
21864                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
21865                        } else {
21866                            computed_vertices +=
21867                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
21868                        }
21869                    }
21870                }
21871            }
21872
21873            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
21874            if let Some(t) = telemetry.as_mut() {
21875                t.changed_vdeps_total += changed_vertices.len();
21876            }
21877            self.resource_checkpoint(0)?;
21878            self.graph.clear_dirty_flags(&to_evaluate);
21879            for v in &changed_vertices {
21880                self.graph.set_dirty(*v, true);
21881            }
21882
21883            if changed_vertices.is_empty() {
21884                if let Some(t) = telemetry.as_mut() {
21885                    t.bailout_reason = Some("converged");
21886                }
21887                break;
21888            }
21889            if replan_iterations >= MAX_REPLAN {
21890                if let Some(mut t) = telemetry.take() {
21891                    t.bailout_reason = Some("max_replan");
21892                    t.replan_iterations = replan_iterations;
21893                    self.last_virtual_dep_telemetry = t;
21894                }
21895                return Err(
21896                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
21897                );
21898            }
21899            replan_iterations += 1;
21900        }
21901
21902        if let Some(mut t) = telemetry {
21903            t.replan_iterations = replan_iterations;
21904            self.last_virtual_dep_telemetry = t;
21905        }
21906
21907        // Re-dirty volatile vertices for the next evaluation cycle
21908        self.redirty_for_next_recalc();
21909        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
21910
21911        Ok(EvalResult {
21912            computed_vertices,
21913            cycle_errors,
21914            elapsed: start.elapsed(),
21915        })
21916    }
21917
21918    /// Evaluate only the necessary precedents for specific target cells with cancellation support
21919    pub fn evaluate_until_cancellable(
21920        &mut self,
21921        targets: &[&str],
21922        cancel: crate::engine::CancelToken,
21923    ) -> Result<EvalResult, ExcelError> {
21924        self.observe_evaluation_resource_request(
21925            EvaluationRequestKind::TargetedCancellable,
21926            |engine| {
21927                engine.observe_function_semantic_epoch()?;
21928                engine.active_cancel_flag = Some(cancel.clone());
21929                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
21930                engine.active_cancel_flag = None;
21931                res
21932            },
21933        )
21934    }
21935
21936    fn evaluate_until_cancellable_impl(
21937        &mut self,
21938        targets: &[&str],
21939        cancel_flag: &AtomicBool,
21940    ) -> Result<EvalResult, ExcelError> {
21941        if cancel_flag.load(Ordering::Relaxed) {
21942            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
21943                .with_message("Evaluation cancelled before target preparation"));
21944        }
21945        let mut typed_targets = Vec::with_capacity(targets.len());
21946        for target in targets {
21947            let (sheet, row, col) = self.parse_a1_notation(target)?;
21948            self.graph.sheet_id_mut(&sheet);
21949            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
21950        }
21951        self.evaluate_mixed_targets(&typed_targets, None)
21952    }
21953
21954    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
21955        let mut quoted = false;
21956        let mut separator = None;
21957        let bytes = address.as_bytes();
21958        let mut index = 0usize;
21959        while index < bytes.len() {
21960            match bytes[index] {
21961                b'\'' => {
21962                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
21963                        index = index.saturating_add(1);
21964                    } else {
21965                        quoted = !quoted;
21966                    }
21967                }
21968                b'!' if !quoted => separator = Some(index),
21969                _ => {}
21970            }
21971            index = index.saturating_add(1);
21972        }
21973        if quoted {
21974            return Err(ExcelError::new(ExcelErrorKind::Ref)
21975                .with_message(format!("Invalid quoted sheet reference `{address}`")));
21976        }
21977        let (sheet, cell_part) = match separator {
21978            Some(separator) => {
21979                let raw_sheet = &address[..separator];
21980                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
21981                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
21982                } else {
21983                    raw_sheet.to_string()
21984                };
21985                (sheet, &address[separator + 1..])
21986            }
21987            None => (self.default_sheet_name().to_string(), address),
21988        };
21989
21990        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
21991            ExcelError::new(ExcelErrorKind::Ref)
21992                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
21993        })?;
21994
21995        Ok((sheet, row, col))
21996    }
21997
21998    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
21999    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
22000        use formualizer_parse::parser::ASTNodeType;
22001        match &ast.node_type {
22002            ASTNodeType::Function { name, args, .. } => {
22003                if let Some(func) = self
22004                    .get_function("", name)
22005                    .or_else(|| crate::function_registry::get("", name))
22006                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
22007                {
22008                    return true;
22009                }
22010                args.iter()
22011                    .any(|arg| self.is_ast_volatile_with_provider(arg))
22012            }
22013            ASTNodeType::BinaryOp { left, right, .. } => {
22014                self.is_ast_volatile_with_provider(left)
22015                    || self.is_ast_volatile_with_provider(right)
22016            }
22017            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
22018            ASTNodeType::Array(rows) => rows.iter().any(|row| {
22019                row.iter()
22020                    .any(|cell| self.is_ast_volatile_with_provider(cell))
22021            }),
22022            _ => false,
22023        }
22024    }
22025
22026    /// Find dirty precedents that need evaluation for the given target vertices
22027    fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
22028        let mut to_evaluate = FxHashSet::default();
22029        let mut visited = FxHashSet::default();
22030        let mut stack = Vec::new();
22031
22032        // Start reverse traversal from target vertices
22033        for &target in target_vertices {
22034            stack.push(target);
22035        }
22036
22037        while let Some(vertex_id) = stack.pop() {
22038            if !visited.insert(vertex_id) {
22039                continue; // Already processed
22040            }
22041
22042            if self.graph.vertex_exists(vertex_id) {
22043                // Check if this vertex needs evaluation
22044                let kind = self.graph.get_vertex_kind(vertex_id);
22045                let needs_eval = match kind {
22046                    super::vertex::VertexKind::FormulaScalar
22047                    | super::vertex::VertexKind::FormulaArray => {
22048                        self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
22049                    }
22050                    _ => false, // Values and empty cells don't need evaluation
22051                };
22052
22053                if needs_eval {
22054                    to_evaluate.insert(vertex_id);
22055                }
22056
22057                // Continue traversal to dependencies (precedents)
22058                if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
22059                    for &dep_id in dependencies {
22060                        if !visited.contains(&dep_id) {
22061                            stack.push(dep_id);
22062                        }
22063                    }
22064                } else {
22065                    let dependencies = self.graph.get_dependencies(vertex_id);
22066                    for dep_id in dependencies {
22067                        if !visited.contains(&dep_id) {
22068                            stack.push(dep_id);
22069                        }
22070                    }
22071                }
22072            }
22073        }
22074
22075        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
22076        result.sort_unstable();
22077        result
22078    }
22079
22080    /// Evaluate a layer sequentially
22081    fn evaluate_layer_sequential(
22082        &mut self,
22083        layer: &super::scheduler::Layer,
22084    ) -> Result<usize, ExcelError> {
22085        self.resource_checkpoint(layer.vertices.len() as u64)?;
22086        self.evaluate_layer_sequential_effects(layer)
22087    }
22088
22089    fn update_vertex_value_with_delta(
22090        &mut self,
22091        vertex_id: VertexId,
22092        new_value: LiteralValue,
22093        delta: &mut DeltaCollector,
22094    ) {
22095        if delta.mode != DeltaMode::Off
22096            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
22097        {
22098            let sheet_name = self.graph.sheet_name(cell.sheet_id);
22099            let old = self
22100                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22101                .unwrap_or(LiteralValue::Empty);
22102            if old != new_value {
22103                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
22104            }
22105        }
22106        self.graph.update_vertex_value(vertex_id, new_value.clone());
22107        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
22108    }
22109
22110    fn evaluate_layer_sequential_with_delta(
22111        &mut self,
22112        layer: &super::scheduler::Layer,
22113        delta: &mut DeltaCollector,
22114    ) -> Result<usize, ExcelError> {
22115        self.resource_checkpoint(layer.vertices.len() as u64)?;
22116        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
22117    }
22118
22119    /// Evaluate a layer sequentially with cancellation support
22120    fn evaluate_layer_sequential_cancellable(
22121        &mut self,
22122        layer: &super::scheduler::Layer,
22123        cancel_flag: &AtomicBool,
22124    ) -> Result<usize, ExcelError> {
22125        self.resource_checkpoint(layer.vertices.len() as u64)?;
22126        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
22127    }
22128
22129    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
22130    fn evaluate_layer_sequential_cancellable_demand_driven(
22131        &mut self,
22132        layer: &super::scheduler::Layer,
22133        cancel_flag: &AtomicBool,
22134    ) -> Result<usize, ExcelError> {
22135        self.resource_checkpoint(layer.vertices.len() as u64)?;
22136        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
22137    }
22138
22139    /// Evaluate a layer in parallel using the thread pool
22140    fn evaluate_layer_parallel(
22141        &mut self,
22142        layer: &super::scheduler::Layer,
22143    ) -> Result<usize, ExcelError> {
22144        self.resource_checkpoint(layer.vertices.len() as u64)?;
22145        self.evaluate_layer_parallel_effects(layer)
22146    }
22147
22148    fn evaluate_layer_parallel_with_delta(
22149        &mut self,
22150        layer: &super::scheduler::Layer,
22151        delta: &mut DeltaCollector,
22152    ) -> Result<usize, ExcelError> {
22153        self.resource_checkpoint(layer.vertices.len() as u64)?;
22154        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
22155    }
22156
22157    /// Evaluate a layer in parallel with cancellation support
22158    fn evaluate_layer_parallel_cancellable(
22159        &mut self,
22160        layer: &super::scheduler::Layer,
22161        cancel_flag: &AtomicBool,
22162    ) -> Result<usize, ExcelError> {
22163        self.resource_checkpoint(layer.vertices.len() as u64)?;
22164        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
22165    }
22166
22167    /// Apply a computed result produced by `evaluate_vertex_immutable()`.
22168    ///
22169    /// This is the parallel equivalent of the "apply" portion of `evaluate_vertex_impl`.
22170    /// We keep apply sequential for correctness (spill commit is inherently stateful).
22171    fn apply_parallel_vertex_result(
22172        &mut self,
22173        vertex_id: VertexId,
22174        result: LiteralValue,
22175        mut delta: Option<&mut DeltaCollector>,
22176        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22177    ) -> Result<(), ExcelError> {
22178        // If this vertex's cell is currently covered by a spill from a different anchor,
22179        // ignore the computed result. The spill's committed values own the grid.
22180        if let Some(cell) = self.graph.get_cell_ref(vertex_id)
22181            && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
22182            && owner != vertex_id
22183        {
22184            return Ok(());
22185        }
22186
22187        let kind = self.graph.get_vertex_kind(vertex_id);
22188
22189        // Only formula vertices spill dynamic arrays into the grid.
22190        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
22191        if is_formula {
22192            match result {
22193                LiteralValue::Array(rows) => {
22194                    self.apply_array_result_from_parallel(
22195                        vertex_id,
22196                        rows,
22197                        delta.as_deref_mut(),
22198                        overwritable_formulas,
22199                    )?;
22200                }
22201                other => {
22202                    self.apply_non_array_result_from_parallel(
22203                        vertex_id,
22204                        other,
22205                        delta.as_deref_mut(),
22206                    );
22207                }
22208            }
22209            return Ok(());
22210        }
22211
22212        // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
22213        if let Some(d) = delta {
22214            self.update_vertex_value_with_delta(vertex_id, result, d);
22215        } else {
22216            self.graph.update_vertex_value(vertex_id, result.clone());
22217            self.mirror_vertex_value_to_overlay(vertex_id, &result);
22218        }
22219        Ok(())
22220    }
22221
22222    fn apply_non_array_result_from_parallel(
22223        &mut self,
22224        vertex_id: VertexId,
22225        value: LiteralValue,
22226        delta: Option<&mut DeltaCollector>,
22227    ) {
22228        // Scalar/error result: store value and ensure any previous spill is cleared.
22229        // This mirrors the sequential behavior in `evaluate_vertex_impl`.
22230        let spill_cells = self
22231            .graph
22232            .spill_cells_for_anchor(vertex_id)
22233            .map(|cells| cells.to_vec())
22234            .unwrap_or_default();
22235
22236        if let Some(d) = delta
22237            && d.mode != DeltaMode::Off
22238            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
22239        {
22240            if spill_cells.is_empty() {
22241                let old = self
22242                    .read_cell_value(
22243                        self.graph.sheet_name(anchor.sheet_id),
22244                        anchor.coord.row() + 1,
22245                        anchor.coord.col() + 1,
22246                    )
22247                    .unwrap_or(LiteralValue::Empty);
22248                if old != value {
22249                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22250                }
22251            } else {
22252                for cell in spill_cells.iter() {
22253                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
22254                    let old = self
22255                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
22256                        .unwrap_or(LiteralValue::Empty);
22257                    let new = if cell.sheet_id == anchor.sheet_id
22258                        && cell.coord.row() == anchor.coord.row()
22259                        && cell.coord.col() == anchor.coord.col()
22260                    {
22261                        value.clone()
22262                    } else {
22263                        LiteralValue::Empty
22264                    };
22265                    Self::record_cell_if_changed(d, cell, &old, &new);
22266                }
22267            }
22268        }
22269
22270        self.graph.clear_spill_region(vertex_id);
22271        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
22272            self.record_formula_plane_structural_change(scope);
22273        }
22274
22275        if self.config.arrow_storage_enabled
22276            && self.config.delta_overlay_enabled
22277            && self.config.write_formula_overlay_enabled
22278        {
22279            let empty = LiteralValue::Empty;
22280            for cell in spill_cells.iter() {
22281                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
22282                self.mirror_value_to_computed_overlay(
22283                    &sheet_name,
22284                    cell.coord.row() + 1,
22285                    cell.coord.col() + 1,
22286                    &empty,
22287                );
22288            }
22289        }
22290
22291        self.graph.update_vertex_value(vertex_id, value.clone());
22292        self.mirror_vertex_value_to_overlay(vertex_id, &value);
22293    }
22294
22295    fn apply_array_result_from_parallel(
22296        &mut self,
22297        vertex_id: VertexId,
22298        rows: Vec<Vec<LiteralValue>>,
22299        mut delta: Option<&mut DeltaCollector>,
22300        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22301    ) -> Result<(), ExcelError> {
22302        // Keep behavior consistent with the sequential spill path in `evaluate_vertex_impl`.
22303        self.graph
22304            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
22305
22306        let anchor = self
22307            .graph
22308            .get_cell_ref(vertex_id)
22309            .expect("cell ref for vertex");
22310        let sheet_id = anchor.sheet_id;
22311        let h = rows.len() as u32;
22312        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
22313
22314        // Hard cap to avoid vertex explosion from huge dynamic arrays.
22315        let spill_cells = (h as u64).saturating_mul(w as u64);
22316        if spill_cells > self.config.spill.max_spill_cells as u64 {
22317            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22318            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22319                .with_message("SpillTooLarge")
22320                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22321                    expected_rows: h,
22322                    expected_cols: w,
22323                });
22324            let spill_val = LiteralValue::Error(spill_err.clone());
22325            if let Some(d) = delta.as_deref_mut()
22326                && d.mode != DeltaMode::Off
22327            {
22328                let old = self
22329                    .read_cell_value(
22330                        self.graph.sheet_name(anchor.sheet_id),
22331                        anchor.coord.row() + 1,
22332                        anchor.coord.col() + 1,
22333                    )
22334                    .unwrap_or(LiteralValue::Empty);
22335                if old != spill_val {
22336                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22337                }
22338            }
22339            self.graph.update_vertex_value(vertex_id, spill_val.clone());
22340            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22341            return Ok(());
22342        }
22343
22344        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
22345        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
22346        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
22347        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
22348        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
22349        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
22350            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22351            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22352                .with_message("Spill exceeds sheet bounds")
22353                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22354                    expected_rows: h,
22355                    expected_cols: w,
22356                });
22357            let spill_val = LiteralValue::Error(spill_err.clone());
22358            if let Some(d) = delta.as_deref_mut()
22359                && d.mode != DeltaMode::Off
22360            {
22361                let old = self
22362                    .read_cell_value(
22363                        self.graph.sheet_name(anchor.sheet_id),
22364                        anchor.coord.row() + 1,
22365                        anchor.coord.col() + 1,
22366                    )
22367                    .unwrap_or(LiteralValue::Empty);
22368                if old != spill_val {
22369                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22370                }
22371            }
22372            self.graph.update_vertex_value(vertex_id, spill_val.clone());
22373            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22374            return Ok(());
22375        }
22376
22377        let mut targets = Vec::new();
22378        for r in 0..h {
22379            for c in 0..w {
22380                targets.push(self.graph.make_cell_ref_internal(
22381                    sheet_id,
22382                    anchor.coord.row() + r,
22383                    anchor.coord.col() + c,
22384                ));
22385            }
22386        }
22387
22388        match self.spill_mgr.reserve(
22389            vertex_id,
22390            anchor,
22391            SpillShape { rows: h, cols: w },
22392            SpillMeta {
22393                epoch: self.recalc_epoch,
22394                config: self.config.spill,
22395            },
22396        ) {
22397            Ok(()) => {
22398                if let Err(e) = self.commit_spill_and_mirror(
22399                    vertex_id,
22400                    &targets,
22401                    rows.clone(),
22402                    delta.as_deref_mut(),
22403                    overwritable_formulas,
22404                ) {
22405                    self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22406                    let err_val = LiteralValue::Error(e.clone());
22407                    if let Some(d) = delta.as_deref_mut()
22408                        && d.mode != DeltaMode::Off
22409                    {
22410                        let old = self
22411                            .read_cell_value(
22412                                self.graph.sheet_name(anchor.sheet_id),
22413                                anchor.coord.row() + 1,
22414                                anchor.coord.col() + 1,
22415                            )
22416                            .unwrap_or(LiteralValue::Empty);
22417                        if old != err_val {
22418                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22419                        }
22420                    }
22421                    self.graph.update_vertex_value(vertex_id, err_val.clone());
22422                    self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
22423                    return Ok(());
22424                }
22425
22426                // Anchor shows the top-left value, like Excel
22427                let top_left = rows
22428                    .first()
22429                    .and_then(|r| r.first())
22430                    .cloned()
22431                    .unwrap_or(LiteralValue::Empty);
22432                self.graph.update_vertex_value(vertex_id, top_left.clone());
22433                self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
22434                Ok(())
22435            }
22436            Err(e) => {
22437                self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
22438                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
22439                    .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
22440                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
22441                        expected_rows: h,
22442                        expected_cols: w,
22443                    });
22444                let spill_val = LiteralValue::Error(spill_err.clone());
22445                if let Some(d) = delta
22446                    && d.mode != DeltaMode::Off
22447                {
22448                    let old = self
22449                        .read_cell_value(
22450                            self.graph.sheet_name(anchor.sheet_id),
22451                            anchor.coord.row() + 1,
22452                            anchor.coord.col() + 1,
22453                        )
22454                        .unwrap_or(LiteralValue::Empty);
22455                    if old != spill_val {
22456                        d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
22457                    }
22458                }
22459                self.graph.update_vertex_value(vertex_id, spill_val.clone());
22460                self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
22461                Ok(())
22462            }
22463        }
22464    }
22465
22466    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
22467    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
22468        // Check if vertex exists
22469        if !self.graph.vertex_exists(vertex_id) {
22470            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
22471                .with_message(format!("Vertex not found: {vertex_id:?}")));
22472        }
22473
22474        // Get vertex kind and check if it needs evaluation
22475        let kind = self.graph.get_vertex_kind(vertex_id);
22476        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
22477
22478        let ast_id = match kind {
22479            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
22480                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
22481                    ast_id
22482                } else {
22483                    return Ok(LiteralValue::Number(0.0));
22484                }
22485            }
22486            VertexKind::Empty | VertexKind::Cell => {
22487                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
22488                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22489                    let row = cell_ref.coord.row() + 1;
22490                    let col = cell_ref.coord.col() + 1;
22491                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
22492                        return Ok(v);
22493                    }
22494                }
22495                return Ok(LiteralValue::Number(0.0));
22496            }
22497            VertexKind::NamedScalar => {
22498                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22499                    ExcelError::new(ExcelErrorKind::Name)
22500                        .with_message("Named range metadata missing".to_string())
22501                })?;
22502
22503                return match &named_range.definition {
22504                    NamedDefinition::Cell(cell_ref) => {
22505                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22506                        Ok(self
22507                            .get_cell_value(
22508                                sheet_name,
22509                                cell_ref.coord.row() + 1,
22510                                cell_ref.coord.col() + 1,
22511                            )
22512                            .unwrap_or(LiteralValue::Empty))
22513                    }
22514                    NamedDefinition::Literal(v) => Ok(v.clone()),
22515                    NamedDefinition::Formula { ast, .. } => {
22516                        let context_sheet = match named_range.scope {
22517                            NameScope::Sheet(id) => id,
22518                            NameScope::Workbook => sheet_id,
22519                        };
22520                        let sheet_name = self.graph.sheet_name(context_sheet);
22521                        let cell_ref = self
22522                            .graph
22523                            .get_cell_ref(vertex_id)
22524                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22525                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22526                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
22527                    }
22528                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
22529                        .with_message("Range-valued name evaluated as scalar".to_string())),
22530                };
22531            }
22532            VertexKind::NamedArray => {
22533                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
22534                    ExcelError::new(ExcelErrorKind::Name)
22535                        .with_message("Named range metadata missing".to_string())
22536                })?;
22537
22538                return match &named_range.definition {
22539                    NamedDefinition::Range(range_ref) => {
22540                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
22541                            return Err(ExcelError::new(ExcelErrorKind::Ref)
22542                                .with_message("Named range cannot span sheets".to_string()));
22543                        }
22544                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
22545                        let sr0 = range_ref.start.coord.row();
22546                        let sc0 = range_ref.start.coord.col();
22547                        let er0 = range_ref.end.coord.row();
22548                        let ec0 = range_ref.end.coord.col();
22549                        if sr0 > er0 || sc0 > ec0 {
22550                            return Err(ExcelError::new(ExcelErrorKind::Ref)
22551                                .with_message("Invalid named range bounds".to_string()));
22552                        }
22553
22554                        let h = (er0 - sr0 + 1) as usize;
22555                        let w = (ec0 - sc0 + 1) as usize;
22556                        let cell_count = (h as u64).saturating_mul(w as u64);
22557                        if cell_count > self.config.spill.max_spill_cells as u64 {
22558                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
22559                                "Named range too large to materialize as an array".to_string(),
22560                            ));
22561                        }
22562
22563                        let mut rows = Vec::with_capacity(h);
22564                        for r0 in sr0..=er0 {
22565                            let mut row = Vec::with_capacity(w);
22566                            for c0 in sc0..=ec0 {
22567                                let v = self
22568                                    .get_cell_value(sheet_name, r0 + 1, c0 + 1)
22569                                    .unwrap_or(LiteralValue::Empty);
22570                                row.push(v);
22571                            }
22572                            rows.push(row);
22573                        }
22574                        Ok(LiteralValue::Array(rows))
22575                    }
22576                    NamedDefinition::Cell(cell_ref) => {
22577                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
22578                        let row = cell_ref.coord.row() + 1;
22579                        let col = cell_ref.coord.col() + 1;
22580                        let v = self
22581                            .get_cell_value(sheet_name, row, col)
22582                            .unwrap_or(LiteralValue::Empty);
22583                        Ok(LiteralValue::Array(vec![vec![v]]))
22584                    }
22585                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
22586                    NamedDefinition::Formula { ast, .. } => {
22587                        let context_sheet = match named_range.scope {
22588                            NameScope::Sheet(id) => id,
22589                            NameScope::Workbook => sheet_id,
22590                        };
22591                        let sheet_name = self.graph.sheet_name(context_sheet);
22592                        let cell_ref = self
22593                            .graph
22594                            .get_cell_ref(vertex_id)
22595                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
22596                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22597                        match interpreter.evaluate_ast(ast) {
22598                            Ok(cv) => {
22599                                let v = cv.into_literal();
22600                                match v {
22601                                    LiteralValue::Array(_) => Ok(v),
22602                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
22603                                }
22604                            }
22605                            Err(err) => Ok(LiteralValue::Error(err)),
22606                        }
22607                    }
22608                };
22609            }
22610            VertexKind::InfiniteRange
22611            | VertexKind::Range
22612            | VertexKind::External
22613            | VertexKind::Table => {
22614                // Not directly evaluatable here.
22615                return Ok(LiteralValue::Number(0.0));
22616            }
22617        };
22618
22619        // The interpreter uses a reference to the engine as the context
22620        let sheet_name = self.graph.sheet_name(sheet_id);
22621        let cell_ref = self
22622            .graph
22623            .get_cell_ref(vertex_id)
22624            .expect("cell ref for vertex");
22625        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
22626
22627        interpreter
22628            .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
22629            .map(|cv| {
22630                crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
22631            })
22632    }
22633
22634    /// Get access to the shared thread pool for parallel evaluation
22635    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
22636        self.thread_pool.as_ref()
22637    }
22638}
22639
22640#[derive(Default)]
22641struct RowBoundsCache {
22642    snapshot: u64,
22643    // key: (sheet_id, col_idx)
22644    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
22645}
22646
22647impl RowBoundsCache {
22648    fn new(snapshot: u64) -> Self {
22649        Self {
22650            snapshot,
22651            map: Default::default(),
22652        }
22653    }
22654    fn get_row_bounds(
22655        &self,
22656        sheet_id: SheetId,
22657        col_idx: usize,
22658        snapshot: u64,
22659    ) -> Option<(Option<u32>, Option<u32>)> {
22660        if self.snapshot != snapshot {
22661            return None;
22662        }
22663        self.map.get(&(sheet_id as u32, col_idx)).copied()
22664    }
22665    fn put_row_bounds(
22666        &mut self,
22667        sheet_id: SheetId,
22668        col_idx: usize,
22669        snapshot: u64,
22670        bounds: (Option<u32>, Option<u32>),
22671    ) {
22672        if self.snapshot != snapshot {
22673            self.snapshot = snapshot;
22674            self.map.clear();
22675        }
22676        self.map.insert((sheet_id as u32, col_idx), bounds);
22677    }
22678}
22679
22680struct UsedAxisBoundsCache {
22681    snapshot: u64,
22682    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22683    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
22684    #[cfg(test)]
22685    row_hits: std::sync::atomic::AtomicUsize,
22686    #[cfg(test)]
22687    row_misses: std::sync::atomic::AtomicUsize,
22688    #[cfg(test)]
22689    col_hits: std::sync::atomic::AtomicUsize,
22690    #[cfg(test)]
22691    col_misses: std::sync::atomic::AtomicUsize,
22692}
22693
22694impl UsedAxisBoundsCache {
22695    fn new(snapshot: u64) -> Self {
22696        Self {
22697            snapshot,
22698            row_bounds_by_col_span: Default::default(),
22699            col_bounds_by_row_span: Default::default(),
22700            #[cfg(test)]
22701            row_hits: std::sync::atomic::AtomicUsize::new(0),
22702            #[cfg(test)]
22703            row_misses: std::sync::atomic::AtomicUsize::new(0),
22704            #[cfg(test)]
22705            col_hits: std::sync::atomic::AtomicUsize::new(0),
22706            #[cfg(test)]
22707            col_misses: std::sync::atomic::AtomicUsize::new(0),
22708        }
22709    }
22710
22711    fn reset_for_snapshot(&mut self, snapshot: u64) {
22712        if self.snapshot != snapshot {
22713            self.snapshot = snapshot;
22714            self.row_bounds_by_col_span.clear();
22715            self.col_bounds_by_row_span.clear();
22716        }
22717    }
22718
22719    fn get_row_bounds(
22720        &self,
22721        sheet_id: SheetId,
22722        start_col: u32,
22723        end_col: u32,
22724        snapshot: u64,
22725    ) -> Option<Option<(u32, u32)>> {
22726        if self.snapshot != snapshot {
22727            return None;
22728        }
22729        let cached = self
22730            .row_bounds_by_col_span
22731            .get(&(sheet_id, start_col, end_col))
22732            .copied();
22733        #[cfg(test)]
22734        if cached.is_some() {
22735            self.row_hits.fetch_add(1, Ordering::Relaxed);
22736        }
22737        cached
22738    }
22739
22740    fn put_row_bounds(
22741        &mut self,
22742        sheet_id: SheetId,
22743        start_col: u32,
22744        end_col: u32,
22745        snapshot: u64,
22746        bounds: Option<(u32, u32)>,
22747    ) {
22748        self.reset_for_snapshot(snapshot);
22749        self.row_bounds_by_col_span
22750            .insert((sheet_id, start_col, end_col), bounds);
22751        #[cfg(test)]
22752        self.row_misses.fetch_add(1, Ordering::Relaxed);
22753    }
22754
22755    fn get_col_bounds(
22756        &self,
22757        sheet_id: SheetId,
22758        start_row: u32,
22759        end_row: u32,
22760        snapshot: u64,
22761    ) -> Option<Option<(u32, u32)>> {
22762        if self.snapshot != snapshot {
22763            return None;
22764        }
22765        let cached = self
22766            .col_bounds_by_row_span
22767            .get(&(sheet_id, start_row, end_row))
22768            .copied();
22769        #[cfg(test)]
22770        if cached.is_some() {
22771            self.col_hits.fetch_add(1, Ordering::Relaxed);
22772        }
22773        cached
22774    }
22775
22776    fn put_col_bounds(
22777        &mut self,
22778        sheet_id: SheetId,
22779        start_row: u32,
22780        end_row: u32,
22781        snapshot: u64,
22782        bounds: Option<(u32, u32)>,
22783    ) {
22784        self.reset_for_snapshot(snapshot);
22785        self.col_bounds_by_row_span
22786            .insert((sheet_id, start_row, end_row), bounds);
22787        #[cfg(test)]
22788        self.col_misses.fetch_add(1, Ordering::Relaxed);
22789    }
22790}
22791
22792// Phase 2 shim: in-process spill manager delegating to current graph methods.
22793#[derive(Default)]
22794pub struct ShimSpillManager {
22795    region_locks: RegionLockManager,
22796    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
22797}
22798
22799impl ShimSpillManager {
22800    pub(crate) fn reserve(
22801        &mut self,
22802        owner: VertexId,
22803        anchor_cell: CellRef,
22804        shape: SpillShape,
22805        _meta: SpillMeta,
22806    ) -> Result<(), ExcelError> {
22807        // Derive region from anchor + shape; enforce in-flight exclusivity only.
22808        let region = crate::engine::spill::Region {
22809            sheet_id: anchor_cell.sheet_id as u32,
22810            row_start: anchor_cell.coord.row(),
22811            row_end: anchor_cell
22812                .coord
22813                .row()
22814                .saturating_add(shape.rows)
22815                .saturating_sub(1),
22816            col_start: anchor_cell.coord.col(),
22817            col_end: anchor_cell
22818                .coord
22819                .col()
22820                .saturating_add(shape.cols)
22821                .saturating_sub(1),
22822        };
22823        match self.region_locks.reserve(region, owner) {
22824            Ok(id) => {
22825                if id != 0 {
22826                    self.active_locks.insert(owner, id);
22827                }
22828                Ok(())
22829            }
22830            Err(e) => Err(e),
22831        }
22832    }
22833
22834    /// Release any in-flight region reservation still held for `owner`.
22835    ///
22836    /// Reservations are normally released on commit/rollback, but if an anchor is
22837    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
22838    /// stale reservation could remain. This is a no-op when nothing is held.
22839    pub(crate) fn release_owner(&mut self, owner: VertexId) {
22840        if let Some(id) = self.active_locks.remove(&owner) {
22841            self.region_locks.release(id);
22842        }
22843    }
22844
22845    pub(crate) fn commit_array_with_value_probe<F>(
22846        &mut self,
22847        graph: &mut DependencyGraph,
22848        anchor_vertex: VertexId,
22849        targets: &[CellRef],
22850        rows: Vec<Vec<LiteralValue>>,
22851        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22852        mut value_probe: F,
22853    ) -> Result<(), ExcelError>
22854    where
22855        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
22856    {
22857        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
22858
22859        // Re-run plan on concrete targets before committing to respect blockers.
22860        // This plan checks formula/spill ownership in the graph, but when the graph value cache
22861        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
22862        let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
22863            anchor_vertex,
22864            targets,
22865            overwritable_formulas,
22866        );
22867        if let Err(e) = plan_res {
22868            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22869                self.region_locks.release(id);
22870            }
22871            return Err(e);
22872        }
22873
22874        if !graph.value_cache_enabled() {
22875            // Compute expected spill shape from the target rectangle for diagnostics.
22876            let (expected_rows, expected_cols) = if targets.is_empty() {
22877                (0u32, 0u32)
22878            } else {
22879                let mut min_r = u32::MAX;
22880                let mut max_r = 0u32;
22881                let mut min_c = u32::MAX;
22882                let mut max_c = 0u32;
22883                for cell in targets {
22884                    let r = cell.coord.row();
22885                    let c = cell.coord.col();
22886                    min_r = min_r.min(r);
22887                    max_r = max_r.max(r);
22888                    min_c = min_c.min(c);
22889                    max_c = max_c.max(c);
22890                }
22891                (
22892                    max_r.saturating_sub(min_r).saturating_add(1),
22893                    max_c.saturating_sub(min_c).saturating_add(1),
22894                )
22895            };
22896
22897            let anchor_cell = graph
22898                .get_cell_ref(anchor_vertex)
22899                .expect("anchor cell ref for spill commit");
22900
22901            for cell in targets {
22902                // Never treat the anchor as a blocker.
22903                if *cell == anchor_cell {
22904                    continue;
22905                }
22906                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
22907                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
22908                    continue;
22909                }
22910                // Skip formula vertices in the target region; plan() handled them (or allowed).
22911                if let Some(&vid) = graph.get_vertex_id_for_address(cell)
22912                    && vid != anchor_vertex
22913                {
22914                    match graph.get_vertex_kind(vid) {
22915                        crate::engine::vertex::VertexKind::FormulaScalar
22916                        | crate::engine::vertex::VertexKind::FormulaArray => {
22917                            // plan() already approved allowed overwrites.
22918                            continue;
22919                        }
22920                        _ => {}
22921                    }
22922                }
22923
22924                if let Some(v) = value_probe(graph, cell)
22925                    && !matches!(v, LiteralValue::Empty)
22926                {
22927                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22928                        self.region_locks.release(id);
22929                    }
22930                    return Err(ExcelError::new(ExcelErrorKind::Spill)
22931                        .with_message("BlockedByValue")
22932                        .with_extra(ExcelErrorExtra::Spill {
22933                            expected_rows,
22934                            expected_cols,
22935                        }));
22936                }
22937            }
22938        }
22939
22940        let commit_res = graph.commit_spill_region_atomic_with_fault(
22941            anchor_vertex,
22942            targets.to_vec(),
22943            rows,
22944            None,
22945        );
22946        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22947            self.region_locks.release(id);
22948        }
22949        commit_res.map(|_| ())
22950    }
22951
22952    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
22953    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
22954        &mut self,
22955        engine: &mut Engine<R>,
22956        anchor_vertex: VertexId,
22957        targets: &[CellRef],
22958        rows: Vec<Vec<LiteralValue>>,
22959        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
22960    ) -> Result<(), ExcelError> {
22961        // Re-run plan on concrete targets before committing to respect blockers.
22962        let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
22963            anchor_vertex,
22964            targets,
22965            overwritable_formulas,
22966        );
22967        if let Err(e) = plan_res {
22968            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22969                self.region_locks.release(id);
22970            }
22971            return Err(e);
22972        }
22973
22974        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
22975            anchor_vertex,
22976            targets.to_vec(),
22977            rows.clone(),
22978            None,
22979        );
22980        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
22981            self.region_locks.release(id);
22982        }
22983        commit_res.map(|_| ())?;
22984
22985        // Mirror into Arrow overlay when enabled
22986        if engine.config.arrow_storage_enabled
22987            && engine.config.delta_overlay_enabled
22988            && engine.config.write_formula_overlay_enabled
22989        {
22990            // Expect targets to be a contiguous rectangle row-major starting at some anchor
22991            for (idx, cell) in targets.iter().enumerate() {
22992                let (r_off, c_off) = {
22993                    if rows.is_empty() || rows[0].is_empty() {
22994                        (0usize, 0usize)
22995                    } else {
22996                        let width = rows[0].len();
22997                        (idx / width, idx % width)
22998                    }
22999                };
23000                let v = rows
23001                    .get(r_off)
23002                    .and_then(|r| r.get(c_off))
23003                    .cloned()
23004                    .unwrap_or(LiteralValue::Empty);
23005                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
23006                engine.mirror_value_to_computed_overlay(
23007                    &sheet_name,
23008                    cell.coord.row() + 1,
23009                    cell.coord.col() + 1,
23010                    &v,
23011                );
23012            }
23013        }
23014        Ok(())
23015    }
23016}
23017
23018impl<R> Engine<R>
23019where
23020    R: EvaluationContext,
23021{
23022    fn resolve_shared_ref(
23023        &self,
23024        reference: &ReferenceType,
23025        current_sheet: &str,
23026    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
23027        use formualizer_common::{
23028            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
23029            SheetRef as SharedRef,
23030        };
23031
23032        // Preserve anchor flags from the parsed reference when possible.
23033        let sr = match reference {
23034            ReferenceType::Cell {
23035                sheet,
23036                row,
23037                col,
23038                row_abs,
23039                col_abs,
23040            } => {
23041                let row0 = row
23042                    .checked_sub(1)
23043                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23044                let col0 = col
23045                    .checked_sub(1)
23046                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23047                let sheet_loc = match sheet.as_deref() {
23048                    Some(name) => SheetLocator::from_name(name),
23049                    None => SheetLocator::Current,
23050                };
23051                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
23052                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
23053            }
23054            ReferenceType::Range {
23055                sheet,
23056                start_row,
23057                start_col,
23058                end_row,
23059                end_col,
23060                start_row_abs,
23061                start_col_abs,
23062                end_row_abs,
23063                end_col_abs,
23064            } => {
23065                let sheet_loc = match sheet.as_deref() {
23066                    Some(name) => SheetLocator::from_name(name),
23067                    None => SheetLocator::Current,
23068                };
23069                let sr = start_row
23070                    .map(|r| {
23071                        r.checked_sub(1)
23072                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23073                    })
23074                    .transpose()?;
23075                let sc = start_col
23076                    .map(|c| {
23077                        c.checked_sub(1)
23078                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23079                    })
23080                    .transpose()?;
23081                let er = end_row
23082                    .map(|r| {
23083                        r.checked_sub(1)
23084                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23085                    })
23086                    .transpose()?;
23087                let ec = end_col
23088                    .map(|c| {
23089                        c.checked_sub(1)
23090                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23091                    })
23092                    .transpose()?;
23093                let range = SharedRangeRef::from_parts(
23094                    sheet_loc,
23095                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
23096                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
23097                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
23098                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
23099                )
23100                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
23101                SharedRef::Range(range)
23102            }
23103            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23104        };
23105
23106        let current_id = self
23107            .graph
23108            .sheet_id(current_sheet)
23109            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23110
23111        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
23112            match loc {
23113                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
23114                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
23115                SheetLocator::Name(name) => {
23116                    let n = name.as_ref();
23117                    self.graph
23118                        .sheet_id(n)
23119                        .map(SheetLocator::Id)
23120                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
23121                }
23122            }
23123        };
23124
23125        match sr {
23126            SharedRef::Cell(cell) => {
23127                let owned = cell.into_owned();
23128                let sheet = resolve_loc(owned.sheet)?;
23129                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
23130            }
23131            SharedRef::Range(range) => {
23132                let owned = range.into_owned();
23133                let sheet = resolve_loc(owned.sheet)?;
23134                Ok(SharedRef::Range(SharedRangeRef {
23135                    sheet,
23136                    start_row: owned.start_row,
23137                    start_col: owned.start_col,
23138                    end_row: owned.end_row,
23139                    end_col: owned.end_col,
23140                }))
23141            }
23142        }
23143    }
23144}
23145
23146// Implement the resolver traits for the Engine.
23147// This allows the interpreter to resolve references by querying the engine's graph.
23148impl<R> crate::traits::ReferenceResolver for Engine<R>
23149where
23150    R: EvaluationContext,
23151{
23152    fn resolve_cell_reference(
23153        &self,
23154        sheet: Option<&str>,
23155        row: u32,
23156        col: u32,
23157    ) -> Result<LiteralValue, ExcelError> {
23158        // This context-free trait method has no knowledge of the formula's
23159        // current sheet, so an unqualified (`None`) reference cannot be resolved
23160        // here. Previously this fell back to `default_sheet_name()`, which leaked
23161        // the reference onto an unrelated sheet (issue #110). Interpreter paths
23162        // already qualify references with the current sheet before reaching this
23163        // method (see `Interpreter::implicit_intersection_from_reference`), and
23164        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
23165        // with an explicit `current_sheet`. Returning #REF! for an unqualified
23166        // reference here surfaces the missing context instead of silently
23167        // returning data from the wrong sheet.
23168        let Some(sheet_name) = sheet else {
23169            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
23170                "Unqualified cell reference resolved without sheet context".to_string(),
23171            ));
23172        };
23173        // Prefer engine's unified accessor which consults Arrow store for base values
23174        // and falls back to graph for formulas and stored values.
23175        if let Some(v) = self.get_cell_value(sheet_name, row, col) {
23176            Ok(v)
23177        } else {
23178            // Excel semantics: empty cell coerces to 0 in numeric contexts
23179            Ok(LiteralValue::Number(0.0))
23180        }
23181    }
23182}
23183
23184impl<R> crate::traits::RangeResolver for Engine<R>
23185where
23186    R: EvaluationContext,
23187{
23188    fn resolve_range_reference(
23189        &self,
23190        sheet: Option<&str>,
23191        sr: Option<u32>,
23192        sc: Option<u32>,
23193        er: Option<u32>,
23194        ec: Option<u32>,
23195    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
23196        // For now, delegate range resolution to the external resolver.
23197        // A future optimization could be to handle this within the graph.
23198        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
23199    }
23200}
23201
23202impl<R> crate::traits::NamedRangeResolver for Engine<R>
23203where
23204    R: EvaluationContext,
23205{
23206    fn resolve_named_range_reference(
23207        &self,
23208        name: &str,
23209    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
23210        self.resolver.resolve_named_range_reference(name)
23211    }
23212}
23213
23214impl<R> crate::traits::TableResolver for Engine<R>
23215where
23216    R: EvaluationContext,
23217{
23218    fn resolve_table_reference(
23219        &self,
23220        tref: &formualizer_parse::parser::TableReference,
23221    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23222        self.resolver.resolve_table_reference(tref)
23223    }
23224}
23225
23226impl<R> crate::traits::SourceResolver for Engine<R>
23227where
23228    R: EvaluationContext,
23229{
23230    fn source_scalar_version(&self, name: &str) -> Option<u64> {
23231        self.resolver.source_scalar_version(name)
23232    }
23233
23234    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
23235        self.resolver.resolve_source_scalar(name)
23236    }
23237
23238    fn source_table_version(&self, name: &str) -> Option<u64> {
23239        self.resolver.source_table_version(name)
23240    }
23241
23242    fn resolve_source_table(
23243        &self,
23244        name: &str,
23245    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
23246        self.resolver.resolve_source_table(name)
23247    }
23248}
23249
23250// The Engine is a Resolver because it implements the constituent traits.
23251impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
23252
23253// The Engine provides functions by delegating to its internal resolver.
23254impl<R> crate::traits::FunctionProvider for Engine<R>
23255where
23256    R: EvaluationContext,
23257{
23258    fn planning_semantic_revision(&self) -> Option<u64> {
23259        self.resolver.planning_semantic_revision()
23260    }
23261
23262    fn get_function(
23263        &self,
23264        prefix: &str,
23265        name: &str,
23266    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23267        self.resolver.get_function(prefix, name)
23268    }
23269
23270    fn get_function_for_planning(
23271        &self,
23272        prefix: &str,
23273        name: &str,
23274    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
23275        self.resolver.get_function_for_planning(prefix, name)
23276    }
23277}
23278
23279impl<R> Engine<R>
23280where
23281    R: EvaluationContext,
23282{
23283    /// Semantic used coordinates exclude graph-only dependency placeholders.
23284    ///
23285    /// Non-empty base/overlay/computed cells come from Arrow storage, while
23286    /// scalar and array formulas come from graph formula kinds even before
23287    /// their results are materialized. The legacy graph fallback is omitted:
23288    /// `load_packed_to_vertex` entries are either represented by those sources
23289    /// or are `Empty` dependency placeholders, not a third value authority.
23290    pub(crate) fn semantic_used_rows_for_columns(
23291        &self,
23292        sheet: &str,
23293        start_col: u32,
23294        end_col: u32,
23295    ) -> Option<(u32, u32)> {
23296        let arrow_bounds = self
23297            .sheet_store()
23298            .sheet(sheet)
23299            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23300        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23301        Self::union_used_bounds(arrow_bounds, formula_bounds)
23302    }
23303
23304    pub(crate) fn semantic_used_cols_for_rows(
23305        &self,
23306        sheet: &str,
23307        start_row: u32,
23308        end_row: u32,
23309    ) -> Option<(u32, u32)> {
23310        let arrow_bounds = self
23311            .sheet_store()
23312            .sheet(sheet)
23313            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23314        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23315        Self::union_used_bounds(arrow_bounds, formula_bounds)
23316    }
23317}
23318
23319// Override EvaluationContext to provide thread pool access
23320impl<R> crate::traits::EvaluationContext for Engine<R>
23321where
23322    R: EvaluationContext,
23323{
23324    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
23325        &self.clock
23326    }
23327
23328    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
23329        self.thread_pool.as_ref()
23330    }
23331
23332    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
23333        self.active_cancel_flag.clone()
23334    }
23335
23336    fn chunk_hint(&self) -> Option<usize> {
23337        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
23338        let hint =
23339            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
23340        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
23341    }
23342
23343    fn volatile_level(&self) -> crate::traits::VolatileLevel {
23344        self.config.volatile_level
23345    }
23346
23347    fn workbook_seed(&self) -> u64 {
23348        self.config.workbook_seed
23349    }
23350
23351    fn recalc_epoch(&self) -> u64 {
23352        self.recalc_epoch
23353    }
23354
23355    fn workbook_sheet_count(&self) -> Option<usize> {
23356        Some(self.graph.sheet_reg().active_len())
23357    }
23358
23359    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
23360        self.graph.sheet_reg().active_position(sheet)
23361    }
23362
23363    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
23364        self.sheet_index_by_name(current_sheet)
23365    }
23366
23367    fn inspect_reference(
23368        &self,
23369        reference: &ReferenceType,
23370        current_sheet: &str,
23371    ) -> Result<Option<ReferenceInfo>, ExcelError> {
23372        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
23373            let sheet_id = self
23374                .graph
23375                .sheet_id(sheet_name)
23376                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23377            let sheet_index = self
23378                .graph
23379                .sheet_reg()
23380                .active_position_by_id(sheet_id)
23381                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23382            Ok((sheet_id, sheet_index))
23383        };
23384
23385        let cell_info =
23386            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
23387                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23388                let row0 = row
23389                    .checked_sub(1)
23390                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23391                let col0 = col
23392                    .checked_sub(1)
23393                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23394                Ok(ReferenceInfo {
23395                    first_sheet_index: Some(sheet_index),
23396                    sheet_count: Some(1),
23397                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23398                })
23399            };
23400
23401        let range_info = |sheet_name: &str,
23402                          start_row: Option<u32>,
23403                          start_col: Option<u32>|
23404         -> Result<ReferenceInfo, ExcelError> {
23405            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
23406            let row = start_row.unwrap_or(1);
23407            let col = start_col.unwrap_or(1);
23408            let row0 = row
23409                .checked_sub(1)
23410                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23411            let col0 = col
23412                .checked_sub(1)
23413                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23414            Ok(ReferenceInfo {
23415                first_sheet_index: Some(sheet_index),
23416                sheet_count: Some(1),
23417                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
23418            })
23419        };
23420
23421        let info = match reference {
23422            ReferenceType::Cell {
23423                sheet, row, col, ..
23424            } => {
23425                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23426                cell_info(sheet_name, *row, *col)?
23427            }
23428            ReferenceType::Range {
23429                sheet,
23430                start_row,
23431                start_col,
23432                ..
23433            } => {
23434                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
23435                range_info(sheet_name, *start_row, *start_col)?
23436            }
23437            ReferenceType::Cell3D {
23438                sheet_first,
23439                sheet_last,
23440                row,
23441                col,
23442                ..
23443            } => {
23444                let first = cell_info(sheet_first, *row, *col)?;
23445                ReferenceInfo {
23446                    first_sheet_index: first.first_sheet_index,
23447                    sheet_count: self
23448                        .graph
23449                        .sheet_reg()
23450                        .active_span_len(sheet_first, sheet_last),
23451                    first_cell: first.first_cell,
23452                }
23453            }
23454            ReferenceType::Range3D {
23455                sheet_first,
23456                sheet_last,
23457                start_row,
23458                start_col,
23459                ..
23460            } => {
23461                let first = range_info(sheet_first, *start_row, *start_col)?;
23462                ReferenceInfo {
23463                    first_sheet_index: first.first_sheet_index,
23464                    sheet_count: self
23465                        .graph
23466                        .sheet_reg()
23467                        .active_span_len(sheet_first, sheet_last),
23468                    first_cell: first.first_cell,
23469                }
23470            }
23471            ReferenceType::NamedRange(name) => {
23472                let current_id = self
23473                    .graph
23474                    .sheet_id(current_sheet)
23475                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23476                let named = self
23477                    .graph
23478                    .resolve_name_entry(name, current_id)
23479                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23480                match &named.definition {
23481                    NamedDefinition::Cell(cell) => ReferenceInfo {
23482                        first_sheet_index: self
23483                            .graph
23484                            .sheet_reg()
23485                            .active_position_by_id(cell.sheet_id),
23486                        sheet_count: Some(1),
23487                        first_cell: Some(*cell),
23488                    },
23489                    NamedDefinition::Range(range) => ReferenceInfo {
23490                        first_sheet_index: self
23491                            .graph
23492                            .sheet_reg()
23493                            .active_position_by_id(range.start.sheet_id),
23494                        sheet_count: Some(1),
23495                        first_cell: Some(range.start),
23496                    },
23497                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
23498                        ReferenceInfo {
23499                            first_sheet_index: None,
23500                            sheet_count: None,
23501                            first_cell: None,
23502                        }
23503                    }
23504                }
23505            }
23506            ReferenceType::Table(tref) => {
23507                let table = self
23508                    .graph
23509                    .resolve_table_entry(&tref.name)
23510                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
23511                ReferenceInfo {
23512                    first_sheet_index: self
23513                        .graph
23514                        .sheet_reg()
23515                        .active_position_by_id(table.range.start.sheet_id),
23516                    sheet_count: Some(1),
23517                    first_cell: Some(table.range.start),
23518                }
23519            }
23520            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
23521        };
23522
23523        Ok(Some(info))
23524    }
23525
23526    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
23527        let sheet_name = self.graph.sheet_name(cell.sheet_id);
23528        if sheet_name.is_empty() {
23529            return Err(ExcelError::new(ExcelErrorKind::Ref));
23530        }
23531        let row = cell.coord.row() + 1;
23532        let col = cell.coord.col() + 1;
23533
23534        if let Some(entries) = self.staged_formulas.get(sheet_name)
23535            && let Some(text) = entries.get(row, col)
23536        {
23537            return Ok(Some(if text.starts_with('=') {
23538                text.to_owned()
23539            } else {
23540                format!("={text}")
23541            }));
23542        }
23543
23544        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
23545            return Ok(None);
23546        };
23547        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
23548    }
23549
23550    fn used_rows_for_columns(
23551        &self,
23552        sheet: &str,
23553        start_col: u32,
23554        end_col: u32,
23555    ) -> Option<(u32, u32)> {
23556        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
23557        let sheet_id = self.graph.sheet_id(sheet)?;
23558        let snap = self.data_snapshot_id();
23559        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23560            guard
23561                .as_ref()
23562                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
23563        }) {
23564            return cached;
23565        }
23566
23567        let arrow_bounds = self
23568            .sheet_store()
23569            .sheet(sheet)
23570            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
23571        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
23572        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23573            Some(bounds)
23574        } else {
23575            let sc0 = start_col.saturating_sub(1);
23576            let ec0 = end_col.saturating_sub(1);
23577            self.graph
23578                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
23579                .map(|(a0, b0)| (a0 + 1, b0 + 1))
23580        };
23581
23582        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23583            guard
23584                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23585                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
23586        }
23587
23588        computed
23589    }
23590
23591    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
23592        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
23593        let sheet_id = self.graph.sheet_id(sheet)?;
23594        let snap = self.data_snapshot_id();
23595        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
23596            guard
23597                .as_ref()
23598                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
23599        }) {
23600            return cached;
23601        }
23602
23603        let arrow_bounds = self
23604            .sheet_store()
23605            .sheet(sheet)
23606            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
23607        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
23608        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
23609            Some(bounds)
23610        } else {
23611            let sr0 = start_row.saturating_sub(1);
23612            let er0 = end_row.saturating_sub(1);
23613            self.graph
23614                .used_col_bounds_for_rows(sheet_id, sr0, er0)
23615                .map(|(a0, b0)| (a0 + 1, b0 + 1))
23616        };
23617
23618        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
23619            guard
23620                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
23621                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
23622        }
23623
23624        computed
23625    }
23626
23627    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
23628        let _ = self.graph.sheet_id(sheet)?;
23629        // Excel-like upper bounds; we expose something finite but large.
23630        // Backends may override with real bounds.
23631        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
23632    }
23633
23634    fn data_snapshot_id(&self) -> u64 {
23635        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
23636    }
23637
23638    fn backend_caps(&self) -> crate::traits::BackendCaps {
23639        crate::traits::BackendCaps {
23640            streaming: true,
23641            used_region: true,
23642            write: false,
23643            tables: false,
23644            async_stream: false,
23645        }
23646    }
23647
23648    fn build_lookup_index(
23649        &self,
23650        view: &RangeView<'_>,
23651        axis: LookupAxis,
23652    ) -> Option<Arc<LookupIndex>> {
23653        self.build_lookup_index_impl(view, axis)
23654    }
23655
23656    // Flats removed
23657
23658    fn date_system(&self) -> crate::engine::DateSystem {
23659        self.config.date_system
23660    }
23661    /// New: resolve a reference into a RangeView (Phase 2 API)
23662    fn resolve_range_view<'c>(
23663        &'c self,
23664        reference: &ReferenceType,
23665        current_sheet: &str,
23666    ) -> Result<RangeView<'c>, ExcelError> {
23667        match reference {
23668            ReferenceType::External(ext) => {
23669                let name = ext.raw.as_str();
23670                match ext.kind {
23671                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
23672                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
23673                            return Err(ExcelError::new(ExcelErrorKind::Name)
23674                                .with_message(format!("Undefined name: {name}")));
23675                        };
23676                        let version = source
23677                            .version
23678                            .or_else(|| self.resolver.source_scalar_version(name));
23679                        let v = self.resolve_source_scalar_cached(name, version)?;
23680                        Ok(RangeView::from_owned_rows(
23681                            vec![vec![v]],
23682                            self.config.date_system,
23683                        ))
23684                    }
23685                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
23686                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
23687                            return Err(ExcelError::new(ExcelErrorKind::Name)
23688                                .with_message(format!("Undefined table: {name}")));
23689                        };
23690                        let version = source
23691                            .version
23692                            .or_else(|| self.resolver.source_table_version(name));
23693                        let table = self.resolve_source_table_cached(name, version)?;
23694                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
23695                        self.source_table_to_range_view(table.as_ref(), &spec)
23696                    }
23697                }
23698            }
23699            ReferenceType::Range { .. } => {
23700                let shared = self.resolve_shared_ref(reference, current_sheet)?;
23701                let formualizer_common::SheetRef::Range(range) = shared else {
23702                    return Err(ExcelError::new(ExcelErrorKind::Ref));
23703                };
23704                // No context sheet is available here, so an unresolved locator
23705                // is #REF! rather than a guess (issue #110).
23706                let sheet_id = match range.sheet {
23707                    formualizer_common::SheetLocator::Id(id) => id,
23708                    formualizer_common::SheetLocator::Current
23709                    | formualizer_common::SheetLocator::Name(_) => {
23710                        return Err(ExcelError::new(ExcelErrorKind::Ref));
23711                    }
23712                };
23713                let sheet_name = self.graph.sheet_name(sheet_id);
23714
23715                let bounded_range = if range.start_row.is_some()
23716                    && range.start_col.is_some()
23717                    && range.end_row.is_some()
23718                    && range.end_col.is_some()
23719                {
23720                    Some(RangeRef::try_from_shared(range.as_ref())?)
23721                } else {
23722                    None
23723                };
23724
23725                let sr = bounded_range
23726                    .as_ref()
23727                    .map(|r| r.start.coord.row() + 1)
23728                    .or_else(|| range.start_row.map(|b| b.index + 1));
23729                let sc = bounded_range
23730                    .as_ref()
23731                    .map(|r| r.start.coord.col() + 1)
23732                    .or_else(|| range.start_col.map(|b| b.index + 1));
23733                let er = bounded_range
23734                    .as_ref()
23735                    .map(|r| r.end.coord.row() + 1)
23736                    .or_else(|| range.end_row.map(|b| b.index + 1));
23737                let ec = bounded_range
23738                    .as_ref()
23739                    .map(|r| r.end.coord.col() + 1)
23740                    .or_else(|| range.end_col.map(|b| b.index + 1));
23741
23742                let extent = resolve_used_extent_with_fallback(
23743                    OpenRangeBounds {
23744                        start_row: sr,
23745                        start_column: sc,
23746                        end_row: er,
23747                        end_column: ec,
23748                    },
23749                    ExtentPolicy::EvaluationCompat {
23750                        fallback_row: None,
23751                        fallback_column: None,
23752                    },
23753                    || {
23754                        self.sheet_bounds(sheet_name)
23755                            .map(|_| self.config.max_open_ended_rows)
23756                    },
23757                    || {
23758                        self.sheet_bounds(sheet_name)
23759                            .map(|_| self.config.max_open_ended_cols)
23760                    },
23761                    |first, last| self.used_rows_for_columns(sheet_name, first, last),
23762                    |first, last| self.used_cols_for_rows(sheet_name, first, last),
23763                );
23764                let (sr, sc, er, ec) = extent
23765                    .map(|extent| {
23766                        (
23767                            extent.start_row,
23768                            extent.start_column,
23769                            extent.end_row,
23770                            extent.end_column,
23771                        )
23772                    })
23773                    .unwrap_or((1, 1, 0, 0));
23774
23775                if self.force_materialize_range_views {
23776                    if er < sr || ec < sc {
23777                        return Ok(RangeView::from_owned_rows(
23778                            Vec::new(),
23779                            self.config.date_system,
23780                        ));
23781                    }
23782                    let h = (er - sr + 1) as u64;
23783                    let w = (ec - sc + 1) as u64;
23784                    let cell_count = h.saturating_mul(w);
23785                    if cell_count <= self.config.spill.max_spill_cells as u64 {
23786                        let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
23787                        for r in sr..=er {
23788                            let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
23789                            for c in sc..=ec {
23790                                rowv.push(
23791                                    self.get_cell_value(sheet_name, r, c)
23792                                        .unwrap_or(LiteralValue::Empty),
23793                                );
23794                            }
23795                            rows.push(rowv);
23796                        }
23797                        return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
23798                    }
23799                }
23800
23801                let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
23802                    return Ok(RangeView::from_owned_rows(
23803                        Vec::new(),
23804                        self.config.date_system,
23805                    ));
23806                };
23807
23808                let rv = if er < sr || ec < sc {
23809                    asheet.range_view(1, 1, 0, 0)
23810                } else {
23811                    let sr0 = sr.saturating_sub(1) as usize;
23812                    let sc0 = sc.saturating_sub(1) as usize;
23813                    let er0 = er.saturating_sub(1) as usize;
23814                    let ec0 = ec.saturating_sub(1) as usize;
23815                    asheet.range_view(sr0, sc0, er0, ec0)
23816                };
23817
23818                Ok(rv)
23819            }
23820            ReferenceType::Cell { .. } => {
23821                let shared = self.resolve_shared_ref(reference, current_sheet)?;
23822                let formualizer_common::SheetRef::Cell(cell) = shared else {
23823                    return Err(ExcelError::new(ExcelErrorKind::Ref));
23824                };
23825                let addr = CellRef::try_from_shared(cell)?;
23826                let sheet_id = addr.sheet_id;
23827                let sheet_name = self.graph.sheet_name(sheet_id);
23828                let row = addr.coord.row() + 1;
23829                let col = addr.coord.col() + 1;
23830
23831                if self.force_materialize_range_views {
23832                    let v = self
23833                        .get_cell_value(sheet_name, row, col)
23834                        .unwrap_or(LiteralValue::Empty);
23835                    return Ok(RangeView::from_owned_rows(
23836                        vec![vec![v]],
23837                        self.config.date_system,
23838                    ));
23839                }
23840
23841                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
23842                    let r0 = row.saturating_sub(1) as usize;
23843                    let c0 = col.saturating_sub(1) as usize;
23844                    let rv = asheet.range_view(r0, c0, r0, c0);
23845                    Ok(rv)
23846                } else {
23847                    let v = self
23848                        .get_cell_value(sheet_name, row, col)
23849                        .unwrap_or(LiteralValue::Empty);
23850                    Ok(RangeView::from_owned_rows(
23851                        vec![vec![v]],
23852                        self.config.date_system,
23853                    ))
23854                }
23855            }
23856            ReferenceType::NamedRange(name) => {
23857                if let Some(current_id) = self.graph.sheet_id(current_sheet)
23858                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
23859                {
23860                    match &named.definition {
23861                        NamedDefinition::Cell(cell_ref) => {
23862                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23863                            if self.force_materialize_range_views {
23864                                let v = self
23865                                    .get_cell_value(
23866                                        sheet_name,
23867                                        cell_ref.coord.row() + 1,
23868                                        cell_ref.coord.col() + 1,
23869                                    )
23870                                    .unwrap_or(LiteralValue::Empty);
23871                                return Ok(RangeView::from_owned_rows(
23872                                    vec![vec![v]],
23873                                    self.config.date_system,
23874                                ));
23875                            } else {
23876                                let asheet = self
23877                                    .sheet_store()
23878                                    .sheet(sheet_name)
23879                                    .expect("Arrow sheet missing for named cell");
23880                                let r0 = cell_ref.coord.row() as usize;
23881                                let c0 = cell_ref.coord.col() as usize;
23882                                let rv = asheet.range_view(r0, c0, r0, c0);
23883                                return Ok(rv);
23884                            }
23885                        }
23886                        NamedDefinition::Range(range_ref) => {
23887                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
23888                            let sr = range_ref.start.coord.row() + 1;
23889                            let sc = range_ref.start.coord.col() + 1;
23890                            let er = range_ref.end.coord.row() + 1;
23891                            let ec = range_ref.end.coord.col() + 1;
23892                            if self.force_materialize_range_views {
23893                                let h = (er.saturating_sub(sr) + 1) as u64;
23894                                let w = (ec.saturating_sub(sc) + 1) as u64;
23895                                let cell_count = h.saturating_mul(w);
23896                                if cell_count <= self.config.spill.max_spill_cells as u64 {
23897                                    let mut rows: Vec<Vec<LiteralValue>> =
23898                                        Vec::with_capacity(h as usize);
23899                                    for r in sr..=er {
23900                                        let mut rowv: Vec<LiteralValue> =
23901                                            Vec::with_capacity(w as usize);
23902                                        for c in sc..=ec {
23903                                            rowv.push(
23904                                                self.get_cell_value(sheet_name, r, c)
23905                                                    .unwrap_or(LiteralValue::Empty),
23906                                            );
23907                                        }
23908                                        rows.push(rowv);
23909                                    }
23910                                    return Ok(RangeView::from_owned_rows(
23911                                        rows,
23912                                        self.config.date_system,
23913                                    ));
23914                                }
23915                            }
23916                            let asheet = self
23917                                .sheet_store()
23918                                .sheet(sheet_name)
23919                                .expect("Arrow sheet missing for named range");
23920                            let sr0 = range_ref.start.coord.row() as usize;
23921                            let sc0 = range_ref.start.coord.col() as usize;
23922                            let er0 = range_ref.end.coord.row() as usize;
23923                            let ec0 = range_ref.end.coord.col() as usize;
23924                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
23925                            return Ok(rv);
23926                        }
23927                        NamedDefinition::Literal(v) => {
23928                            return Ok(RangeView::from_owned_rows(
23929                                vec![vec![v.clone()]],
23930                                self.config.date_system,
23931                            ));
23932                        }
23933                        NamedDefinition::Formula { .. } => {
23934                            if let Some(value) = self.graph.get_value(named.vertex) {
23935                                return Ok(RangeView::from_owned_rows(
23936                                    vec![vec![value]],
23937                                    self.config.date_system,
23938                                ));
23939                            }
23940                        }
23941                    }
23942                }
23943
23944                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
23945                    let version = source
23946                        .version
23947                        .or_else(|| self.resolver.source_scalar_version(name));
23948                    let v = self.resolve_source_scalar_cached(name, version)?;
23949                    return Ok(RangeView::from_owned_rows(
23950                        vec![vec![v]],
23951                        self.config.date_system,
23952                    ));
23953                }
23954
23955                let data = self.resolver.resolve_named_range_reference(name)?;
23956                Ok(RangeView::from_owned_rows(data, self.config.date_system))
23957            }
23958            ReferenceType::Table(tref) => {
23959                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
23960                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
23961                    let asheet = self
23962                        .sheet_store()
23963                        .sheet(sheet_name)
23964                        .expect("Arrow sheet missing for table reference");
23965
23966                    let sr0 = table.range.start.coord.row() as usize;
23967                    let sc0 = table.range.start.coord.col() as usize;
23968                    let er0 = table.range.end.coord.row() as usize;
23969                    let ec0 = table.range.end.coord.col() as usize;
23970
23971                    let has_totals = table.totals_row;
23972                    let has_headers = table.header_row;
23973                    let data_sr = if has_headers {
23974                        sr0.saturating_add(1)
23975                    } else {
23976                        sr0
23977                    };
23978                    let data_er = if has_totals {
23979                        er0.saturating_sub(1)
23980                    } else {
23981                        er0
23982                    };
23983
23984                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
23985                        if sr > er || sc > ec {
23986                            asheet.range_view(1, 1, 0, 0)
23987                        } else {
23988                            asheet.range_view(sr, sc, er, ec)
23989                        }
23990                    };
23991
23992                    let av = match &tref.specifier {
23993                        None => {
23994                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
23995                                "Table reference without specifier is unsupported".to_string(),
23996                            ));
23997                        }
23998                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
23999                            let Some(idx) = table.col_index(col) else {
24000                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24001                                    "Column refers to unknown table column".to_string(),
24002                                ));
24003                            };
24004                            let c0 = sc0 + idx;
24005                            select(data_sr, c0, data_er, c0)
24006                        }
24007                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
24008                            start,
24009                            end,
24010                        )) => {
24011                            let Some(si) = table.col_index(start) else {
24012                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24013                                    "Column range refers to unknown column(s)".to_string(),
24014                                ));
24015                            };
24016                            let Some(ei) = table.col_index(end) else {
24017                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
24018                                    "Column range refers to unknown column(s)".to_string(),
24019                                ));
24020                            };
24021                            let (mut a, mut b) = (si, ei);
24022                            if a > b {
24023                                std::mem::swap(&mut a, &mut b);
24024                            }
24025                            let c_start = sc0 + a;
24026                            let c_end = sc0 + b;
24027                            select(data_sr, c_start, data_er, c_end)
24028                        }
24029                        Some(formualizer_parse::parser::TableSpecifier::All)
24030                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24031                            formualizer_parse::parser::SpecialItem::All,
24032                        )) => select(sr0, sc0, er0, ec0),
24033                        Some(formualizer_parse::parser::TableSpecifier::Data)
24034                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24035                            formualizer_parse::parser::SpecialItem::Data,
24036                        )) => select(data_sr, sc0, data_er, ec0),
24037                        Some(formualizer_parse::parser::TableSpecifier::Headers)
24038                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24039                            formualizer_parse::parser::SpecialItem::Headers,
24040                        )) => {
24041                            if !has_headers {
24042                                asheet.range_view(1, 1, 0, 0)
24043                            } else {
24044                                select(sr0, sc0, sr0, ec0)
24045                            }
24046                        }
24047                        Some(formualizer_parse::parser::TableSpecifier::Totals)
24048                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24049                            formualizer_parse::parser::SpecialItem::Totals,
24050                        )) => {
24051                            if !has_totals {
24052                                asheet.range_view(1, 1, 0, 0)
24053                            } else {
24054                                select(er0, sc0, er0, ec0)
24055                            }
24056                        }
24057                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
24058                            formualizer_parse::parser::SpecialItem::ThisRow,
24059                        )) => {
24060                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24061                                "@ (This Row) requires table-aware context; not yet supported"
24062                                    .to_string(),
24063                            ));
24064                        }
24065                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
24066                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
24067                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24068                                "Complex structured references not yet supported".to_string(),
24069                            ));
24070                        }
24071                    };
24072
24073                    return Ok(av);
24074                }
24075
24076                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
24077                    let version = source
24078                        .version
24079                        .or_else(|| self.resolver.source_table_version(&tref.name));
24080                    let table = self.resolve_source_table_cached(&tref.name, version)?;
24081                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
24082                }
24083
24084                // Fallback: materialize via Resolver::resolve_range_like tranche 1
24085                let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
24086                let owned = boxed.materialise().into_owned();
24087                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
24088            }
24089            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
24090                Err(ExcelError::new(ExcelErrorKind::NImpl)
24091                    .with_message("3D references are not yet supported".to_string()))
24092            }
24093        }
24094    }
24095
24096    fn resolve_cell_reference_value(
24097        &self,
24098        sheet: Option<&str>,
24099        row: u32,
24100        col: u32,
24101        current_sheet: &str,
24102    ) -> Result<LiteralValue, ExcelError> {
24103        let sheet_name = sheet.unwrap_or(current_sheet);
24104        if self.graph.sheet_id(sheet_name).is_none() {
24105            return Err(ExcelError::new(ExcelErrorKind::Ref));
24106        }
24107        Ok(self
24108            .get_cell_value(sheet_name, row, col)
24109            .unwrap_or(LiteralValue::Empty))
24110    }
24111
24112    fn build_criteria_mask(
24113        &self,
24114        view: &RangeView<'_>,
24115        col_in_view: usize,
24116        pred: &crate::args::CriteriaPredicate,
24117    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24118        if view.dims().1 == 0 {
24119            return None;
24120        }
24121        // If the view is logically open-ended but the backing sheet has no physical rows,
24122        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
24123        let sheet_rows = view.sheet().nrows as usize;
24124        if sheet_rows == 0 || view.start_row() >= sheet_rows {
24125            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
24126        }
24127        compute_criteria_mask(view, col_in_view, pred)
24128    }
24129
24130    fn build_row_visibility_mask(
24131        &self,
24132        view: &RangeView<'_>,
24133        mode: VisibilityMaskMode,
24134    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
24135        self.build_row_visibility_mask_for_view(view, mode)
24136    }
24137}
24138
24139impl<R> Engine<R>
24140where
24141    R: EvaluationContext,
24142{
24143    fn clear_spill_projection_and_mirror(
24144        &mut self,
24145        anchor_vertex: VertexId,
24146        delta: Option<&mut DeltaCollector>,
24147    ) {
24148        let spill_cells = self
24149            .graph
24150            .spill_cells_for_anchor(anchor_vertex)
24151            .map(|cells| cells.to_vec())
24152            .unwrap_or_default();
24153        if spill_cells.is_empty() {
24154            return;
24155        }
24156
24157        if let Some(delta) = delta
24158            && delta.mode != DeltaMode::Off
24159        {
24160            let empty = LiteralValue::Empty;
24161            for cell in spill_cells.iter() {
24162                let sheet_name = self.graph.sheet_name(cell.sheet_id);
24163                let old = self
24164                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24165                    .unwrap_or(LiteralValue::Empty);
24166                if old != empty {
24167                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24168                }
24169            }
24170        }
24171
24172        self.graph.clear_spill_region(anchor_vertex);
24173        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24174            self.record_formula_plane_structural_change(scope);
24175        }
24176
24177        if self.config.arrow_storage_enabled
24178            && self.config.delta_overlay_enabled
24179            && self.config.write_formula_overlay_enabled
24180        {
24181            let empty = LiteralValue::Empty;
24182            for cell in spill_cells.iter() {
24183                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24184                self.mirror_value_to_computed_overlay(
24185                    &sheet_name,
24186                    cell.coord.row() + 1,
24187                    cell.coord.col() + 1,
24188                    &empty,
24189                );
24190            }
24191        }
24192    }
24193
24194    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
24195    /// (optionally filtered) members via `stamp_cycle_error`.
24196    ///
24197    /// This is the single per-SCC application point used by every schedule
24198    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
24199    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
24200    ///
24201    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
24202    /// only members present in the set are stamped.
24203    ///
24204    /// Returns the number of vertices stamped (0 when a filter excludes every
24205    /// member), so callers can keep their site-specific `cycle_errors`
24206    /// accounting.
24207    fn apply_cycle_outcome(
24208        &mut self,
24209        cycle: &[VertexId],
24210        mut delta: Option<&mut DeltaCollector>,
24211        dirty_filter: Option<&FxHashSet<VertexId>>,
24212    ) -> usize {
24213        let circ_error = LiteralValue::Error(
24214            ExcelError::new(ExcelErrorKind::Circ)
24215                .with_message("Circular dependency detected".to_string()),
24216        );
24217        let mut stamped = 0usize;
24218        for &vertex_id in cycle {
24219            if let Some(filter) = dirty_filter
24220                && !filter.contains(&vertex_id)
24221            {
24222                continue;
24223            }
24224            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
24225            stamped += 1;
24226        }
24227        stamped
24228    }
24229
24230    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
24231    ///
24232    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
24233    /// vertex previously anchored: it clears the spilled cells, releases the graph
24234    /// spill registry, drops any lingering region reservation, and mirrors the
24235    /// cleared cells into the computed overlay — the same teardown a normal scalar/
24236    /// error result performs (see `apply_non_array_result_from_parallel` /
24237    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
24238    /// spill anchor would leave stale spilled values and a reserved region behind
24239    /// (issue #111).
24240    ///
24241    /// When `delta` is provided, the cleared spill cells are recorded (by
24242    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
24243    /// recorded here, matching how other result paths emit deltas.
24244    fn stamp_cycle_error(
24245        &mut self,
24246        vertex_id: VertexId,
24247        circ_error: &LiteralValue,
24248        mut delta: Option<&mut DeltaCollector>,
24249    ) {
24250        // Tear down any previous spill projection/region before overwriting the anchor.
24251        if self.graph.spill_registry_has_anchor(vertex_id) {
24252            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24253        }
24254        // Drop any reservation that was never committed (defensive; normally released
24255        // on the prior successful commit).
24256        self.spill_mgr.release_owner(vertex_id);
24257
24258        // Record the anchor's own #CIRC delta, like other result paths.
24259        if let Some(d) = delta
24260            && d.mode != DeltaMode::Off
24261            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24262        {
24263            let sheet_name = self.graph.sheet_name(cell.sheet_id);
24264            let old = self
24265                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24266                .unwrap_or(LiteralValue::Empty);
24267            if old != *circ_error {
24268                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24269            }
24270        }
24271
24272        self.graph
24273            .update_vertex_value(vertex_id, circ_error.clone());
24274        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
24275    }
24276
24277    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
24278    ///
24279    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
24280    ///   `#CIRC!` on the (optionally dirty-filtered) members.
24281    /// * `CycleDetection::Runtime` — evaluate the SCC via
24282    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
24283    ///   no dirty member → skip the task entirely (values stand); any dirty
24284    ///   member → the whole SCC evaluates (an SCC cannot be partially
24285    ///   evaluated).
24286    ///
24287    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
24288    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
24289    fn handle_cycle_unit(
24290        &mut self,
24291        cycle: &[VertexId],
24292        mut delta: Option<&mut DeltaCollector>,
24293        dirty_filter: Option<&FxHashSet<VertexId>>,
24294        cancel_flag: Option<&AtomicBool>,
24295    ) -> Result<usize, ExcelError> {
24296        self.resource_checkpoint(cycle.len() as u64)?;
24297        match self.config.cycle.detection {
24298            CycleDetection::Static => {
24299                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
24300            }
24301            CycleDetection::Runtime => {
24302                if let Some(filter) = dirty_filter
24303                    && !cycle.iter().any(|v| filter.contains(v))
24304                {
24305                    return Ok(0);
24306                }
24307                // Both policies share `evaluate_scc_unit`; they differ only
24308                // in the settle loop's live-cycle arm (Error stamps,
24309                // Iterate keeps passing — RFC #113).
24310                self.evaluate_scc_unit(cycle, delta, cancel_flag)
24311            }
24312        }
24313    }
24314
24315    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
24316    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
24317    /// spec §3; Iterate policy arm per RFC #113).
24318    ///
24319    /// Phantom SCCs (live-acyclic) produce ordinary values under both
24320    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
24321    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
24322    /// (converge per spec §6 or cap at `max_iterations` passes) under
24323    /// `CyclePolicy::Iterate`. Runs sequentially on the
24324    /// coordinating thread; commits are write-through per member (no
24325    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
24326    /// always flushed before a Cycle unit runs, G1), so later members' scalar
24327    /// *and* range reads observe earlier members' results through the overlay
24328    /// cascade. Deltas are recorded once per member at end of task (G11).
24329    ///
24330    /// Returns the number of vertices stamped `#CIRC!`.
24331    ///
24332    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
24333    /// that ingest-time cycle rejection makes unreachable via public edits.
24334    pub(crate) fn evaluate_scc_unit(
24335        &mut self,
24336        cycle: &[VertexId],
24337        mut delta: Option<&mut DeltaCollector>,
24338        cancel_flag: Option<&AtomicBool>,
24339    ) -> Result<usize, ExcelError> {
24340        struct SccMember {
24341            vertex: VertexId,
24342            cell: Option<CellRef>,
24343        }
24344
24345        let task_start = crate::instant::FzInstant::now();
24346
24347        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
24348        // name vertices after, lexicographic by folded canonical name; any
24349        // other vertex kind (defensive — `get_evaluation_vertices` only emits
24350        // formula/name kinds) last by id, never evaluated.
24351        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
24352        let mut name_members: Vec<(VertexId, String)> = Vec::new();
24353        let mut other_members: Vec<VertexId> = Vec::new();
24354        for &v in cycle {
24355            match self.graph.get_vertex_kind(v) {
24356                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24357                    match self.graph.get_cell_ref(v) {
24358                        Some(cell) => cell_members.push((v, cell)),
24359                        None => other_members.push(v),
24360                    }
24361                }
24362                VertexKind::NamedScalar | VertexKind::NamedArray => {
24363                    match self.graph.name_key_for_vertex(v) {
24364                        Some(key) => name_members.push((v, key)),
24365                        None => other_members.push(v),
24366                    }
24367                }
24368                _ => other_members.push(v),
24369            }
24370        }
24371        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
24372        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
24373        other_members.sort_unstable();
24374
24375        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
24376        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
24377        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
24378        for (v, c) in &cell_members {
24379            members.push(SccMember {
24380                vertex: *v,
24381                cell: Some(*c),
24382            });
24383        }
24384        for (v, _) in &name_members {
24385            members.push(SccMember {
24386                vertex: *v,
24387                cell: None,
24388            });
24389        }
24390        for v in &other_members {
24391            members.push(SccMember {
24392                vertex: *v,
24393                cell: None,
24394            });
24395        }
24396        let n = members.len();
24397        // Indices addressable by the collector (cells + names); `other`
24398        // members can be neither edge sources nor targets.
24399        let recordable = cell_refs.len() + name_keys.len();
24400
24401        let circ_error = LiteralValue::Error(
24402            ExcelError::new(ExcelErrorKind::Circ)
24403                .with_message("Circular dependency detected".to_string()),
24404        );
24405
24406        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
24407        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
24408        // an iterating member's committed value is cycle STATE, not a
24409        // recomputable cache — and in canonical mode the overlay is its ONLY
24410        // home. If the overlay entry vanished since the last recalc, re-seed
24411        // it from the end-of-recalc snapshot (`iterative_state_values`) so
24412        // pass-1 reads (scalar AND range, via the overlay cascade) observe
24413        // the persisted value instead of silently restarting at Empty→0.
24414        // (Found by the iterate edge corpus: inserting/deleting an unrelated
24415        // row reset accumulators, violating spec §4/§7.15.)
24416        if !self.iterative_state_values.is_empty() {
24417            let restore: Vec<(VertexId, LiteralValue)> = members
24418                .iter()
24419                .filter_map(|m| {
24420                    let cell = m.cell?;
24421                    let persisted = self.iterative_state_values.get(&m.vertex)?;
24422                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24423                    let overlay = self
24424                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24425                        .unwrap_or(LiteralValue::Empty);
24426                    if matches!(overlay, LiteralValue::Empty) {
24427                        Some((m.vertex, persisted.clone()))
24428                    } else {
24429                        None
24430                    }
24431                })
24432                .collect();
24433            for (vertex, value) in restore {
24434                self.mirror_vertex_value_to_overlay(vertex, &value);
24435            }
24436        }
24437
24438        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
24439        // graph value map may be evicted in value-cache-disabled mode).
24440        let snapshot: Vec<LiteralValue> = members
24441            .iter()
24442            .map(|m| match m.cell {
24443                Some(cell) => {
24444                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24445                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24446                        .unwrap_or(LiteralValue::Empty)
24447                }
24448                None => self
24449                    .graph
24450                    .get_value(m.vertex)
24451                    .unwrap_or(LiteralValue::Empty),
24452            })
24453            .collect();
24454
24455        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
24456        // with full spill teardown (spec §7.9, #115) and excluded from
24457        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
24458        // and propagate). Non-evaluable defensive members are excluded too.
24459        let mut excluded = vec![false; n];
24460        let mut last_value = snapshot.clone();
24461        let mut stamped = 0usize;
24462        for (i, m) in members.iter().enumerate() {
24463            match self.graph.get_vertex_kind(m.vertex) {
24464                VertexKind::FormulaArray => {
24465                    // Deltas for the cleared spill-region cells (non-members)
24466                    // can only be recorded here; the anchor's own delta is
24467                    // covered by the end-of-task snapshot comparison (dedup).
24468                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
24469                    excluded[i] = true;
24470                    last_value[i] = circ_error.clone();
24471                    stamped += 1;
24472                }
24473                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
24474                _ => excluded[i] = true,
24475            }
24476        }
24477
24478        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
24479
24480        // Per-member live out-edges, refreshed whenever a member re-runs.
24481        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
24482        // Position of each member in the most recent pass (-1 = did not run).
24483        let mut pos: Vec<i64> = vec![-1; n];
24484        // Whether each member's committed value changed in the most recent pass.
24485        let mut changed = vec![false; n];
24486
24487        // Evaluate-and-commit one member; returns Ok(true) when the member was
24488        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
24489        macro_rules! run_member {
24490            ($i:expr) => {{
24491                let i: usize = $i;
24492                let m = &members[i];
24493                if i < recordable {
24494                    collector.set_current(i as u32);
24495                }
24496                let value = {
24497                    let ctx = RecordingContext::new(&*self, &collector);
24498                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
24499                        Ok(v) => v,
24500                        Err(e) => LiteralValue::Error(e),
24501                    }
24502                };
24503                let is_cell_formula = m.cell.is_some();
24504                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
24505                    // A member that *would* spill inside an SCC gets the
24506                    // conservative §7.9 verdict. It has never spilled before
24507                    // (a prior spill would make it FormulaArray, pre-stamped
24508                    // above), so there is no projection to tear down.
24509                    self.stamp_cycle_error(m.vertex, &circ_error, None);
24510                    excluded[i] = true;
24511                    stamped += 1;
24512                    changed[i] = last_value[i] != circ_error;
24513                    last_value[i] = circ_error.clone();
24514                } else {
24515                    self.graph.update_vertex_value(m.vertex, value.clone());
24516                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
24517                    // §7.14 invariant (G2): a formula member must never be
24518                    // shadowed by a user/delta overlay entry, or iteration
24519                    // reads would silently diverge from committed values.
24520                    #[cfg(debug_assertions)]
24521                    if let Some(cell) = m.cell {
24522                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24523                        debug_assert!(
24524                            self.read_delta_overlay_cell(
24525                                &sheet_name,
24526                                cell.coord.row() + 1,
24527                                cell.coord.col() + 1
24528                            )
24529                            .is_none(),
24530                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
24531                            cell.coord.row() + 1,
24532                            cell.coord.col() + 1
24533                        );
24534                    }
24535                    changed[i] = last_value[i] != value;
24536                    last_value[i] = value;
24537                }
24538            }};
24539        }
24540
24541        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
24542            if let Some(flag) = flag
24543                && flag.load(Ordering::Relaxed)
24544            {
24545                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24546                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
24547            }
24548            Ok(())
24549        };
24550
24551        // ── 3. Pass 1: all evaluable members in member order.
24552        check_cancel(cancel_flag)?;
24553        let mut passes = 1usize;
24554        {
24555            let mut p = 0i64;
24556            for i in 0..n {
24557                if excluded[i] {
24558                    continue;
24559                }
24560                run_member!(i);
24561                pos[i] = p;
24562                p += 1;
24563            }
24564        }
24565
24566        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
24567        //
24568        // Acyclic classifications settle stale readers exactly (identical
24569        // under both policies — phantom SCCs never iterate). A witnessed
24570        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
24571        // stops; `Iterate` keeps running full passes over all members in
24572        // member order until converged (spec §6) or capped at
24573        // `max_iterations` total passes. A live cycle that only appears
24574        // mid-settle takes the same arm, and a cycle that dissolves
24575        // mid-iteration falls back to exact acyclic settling.
24576        //
24577        // Defensive acyclic budget: the acyclic settle is monotone, so more
24578        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
24579        // remainder and set telemetry. Tracked via `settle_passes` so
24580        // iteration passes (legitimately many) don't consume the budget.
24581        let policy = self.config.cycle.policy;
24582        let cap = n + 2;
24583        let mut witnessed_cycles = 0usize;
24584        let mut capped = false;
24585        // ── Iterate-policy state ──
24586        let mut iterating = false;
24587        let mut converged = false;
24588        // Values committed by the last *full* pass; `None` until the first
24589        // iteration pass runs (pass 1 has no predecessor to compare against)
24590        // and reset when a settle pass runs (no cross-kind comparisons).
24591        let mut prev_pass: Option<Vec<LiteralValue>> = None;
24592        // Final-round convergence stats (overwritten per round so the values
24593        // reported are the ones observed at stop).
24594        let mut iter_max_delta = 0f64;
24595        let mut iter_nan_converged = 0usize;
24596        // Acyclic stale-reader re-eval passes (defensive budget; under pure
24597        // Error flow `1 + settle_passes == passes`, preserving Stage-2
24598        // behavior exactly).
24599        let mut settle_passes = 0usize;
24600        loop {
24601            // Drain this pass's recordings; members that ran replace their
24602            // out-edge set, members that didn't keep last-known edges.
24603            let drained = collector.take_edges();
24604            for i in 0..n {
24605                if pos[i] >= 0 {
24606                    out_edges[i].clear();
24607                }
24608            }
24609            for (from, to) in drained {
24610                debug_assert!(
24611                    pos[from as usize] >= 0,
24612                    "edge from a member that did not run"
24613                );
24614                out_edges[from as usize].push(to);
24615            }
24616            let mut edges: Vec<(u32, u32)> = Vec::new();
24617            for (i, outs) in out_edges.iter().enumerate() {
24618                if excluded[i] {
24619                    continue;
24620                }
24621                for &t in outs {
24622                    edges.push((i as u32, t));
24623                }
24624            }
24625            edges.sort_unstable();
24626            edges.dedup();
24627
24628            let analysis = analyze_live_graph(n, &edges);
24629
24630            if analysis.cycle_count > 0 {
24631                // Classification repeats every iteration pass under
24632                // `Iterate`; record the widest single witness instead of
24633                // accumulating so the count stays "distinct live cycles".
24634                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
24635                match policy {
24636                    CyclePolicy::Error => {
24637                        // POLICY (Error): stamp every member of a live cycle,
24638                        // then one settling pass over the remaining members in
24639                        // live-topological order so error propagation
24640                        // downstream is consistent (spec §3.4). Blast radius =
24641                        // live cycles only.
24642                        for i in 0..n {
24643                            if analysis.in_cycle[i] && !excluded[i] {
24644                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
24645                                excluded[i] = true;
24646                                last_value[i] = circ_error.clone();
24647                                stamped += 1;
24648                            }
24649                        }
24650                        check_cancel(cancel_flag)?;
24651                        let order: Vec<usize> = analysis
24652                            .topo
24653                            .iter()
24654                            .map(|&i| i as usize)
24655                            .filter(|&i| !excluded[i])
24656                            .collect();
24657                        if !order.is_empty() {
24658                            passes += 1;
24659                            for i in order {
24660                                run_member!(i);
24661                            }
24662                        }
24663                        break;
24664                    }
24665                    CyclePolicy::Iterate {
24666                        max_iterations,
24667                        max_change,
24668                    } => {
24669                        // POLICY (Iterate), spec §3.5/§6.
24670                        iterating = true;
24671
24672                        // Convergence test: the full pass that just completed
24673                        // vs the previous full pass, per the spec-§6 rules.
24674                        // `prev_pass` is `None` until an iteration pass has
24675                        // run — pass 1 has no predecessor, so no convergence
24676                        // test occurs before the second pass (spec §6).
24677                        if let Some(prev) = &prev_pass {
24678                            let mut round_max_delta = 0f64;
24679                            let mut round_nan = 0usize;
24680                            let mut all_converged = true;
24681                            for i in 0..n {
24682                                if excluded[i] {
24683                                    // Stamped mid-iteration (array result,
24684                                    // §7.9): the value is pinned and cannot
24685                                    // change again — trivially settled.
24686                                    continue;
24687                                }
24688                                let out = crate::engine::convergence::values_converged(
24689                                    &prev[i],
24690                                    &last_value[i],
24691                                    max_change,
24692                                    self.config.date_system,
24693                                );
24694                                if out.nan_converged {
24695                                    round_nan += 1;
24696                                }
24697                                if let Some(d) = out.abs_delta {
24698                                    round_max_delta = round_max_delta.max(d);
24699                                }
24700                                if !out.converged {
24701                                    all_converged = false;
24702                                }
24703                            }
24704                            // Overwrite (not max): telemetry reports the
24705                            // round observed at stop.
24706                            iter_max_delta = round_max_delta;
24707                            iter_nan_converged = round_nan;
24708                            if all_converged {
24709                                converged = true;
24710                                break;
24711                            }
24712                        }
24713
24714                        // ── Pass-counting reconciliation (spec §6/§7.6):
24715                        // `max_iterations` counts TOTAL passes, pass 1
24716                        // included, and pass 1 has already run by the time a
24717                        // live cycle is first witnessed here. The budget is
24718                        // therefore checked BEFORE evaluating anything more:
24719                        // with `max_iterations: 1` we stop right here — each
24720                        // member was evaluated exactly once this recalc (the
24721                        // Excel accumulator contract) and no convergence test
24722                        // ran (`prev_pass` is still `None`). Capping keeps
24723                        // the last committed values and is NOT an error
24724                        // (Excel parity); telemetry records it.
24725                        if passes >= max_iterations as usize {
24726                            capped = true;
24727                            break;
24728                        }
24729
24730                        check_cancel(cancel_flag)?;
24731                        // One more full pass over every evaluable member in
24732                        // member order (Gauss–Seidel: each commit is visible
24733                        // to later members within the pass). Live edges
24734                        // re-record — guards can flip near convergence
24735                        // (§7.3) — so classification repeats next time
24736                        // around, and a cycle that dissolves drops back to
24737                        // the exact acyclic settle below.
24738                        prev_pass = Some(last_value.clone());
24739                        for x in pos.iter_mut() {
24740                            *x = -1;
24741                        }
24742                        changed.fill(false);
24743                        passes += 1;
24744                        let mut p = 0i64;
24745                        for i in 0..n {
24746                            if excluded[i] {
24747                                continue;
24748                            }
24749                            run_member!(i);
24750                            pos[i] = p;
24751                            p += 1;
24752                        }
24753                        continue;
24754                    }
24755                }
24756            }
24757
24758            // Acyclic: find stale readers — members whose live read of `to`
24759            // happened before `to`'s value changed in the pass that just ran.
24760            let mut stale: Vec<usize> = Vec::new();
24761            for i in 0..n {
24762                if excluded[i] {
24763                    continue;
24764                }
24765                let is_stale = out_edges[i].iter().any(|&t| {
24766                    let t = t as usize;
24767                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
24768                });
24769                if is_stale {
24770                    stale.push(i);
24771                }
24772            }
24773            if stale.is_empty() {
24774                break; // values exact — phantom SCC (or dissolved live cycle)
24775            }
24776            if 1 + settle_passes >= cap {
24777                // Defensive only; hitting this is a bug (loud telemetry).
24778                capped = true;
24779                for (i, m) in members.iter().enumerate() {
24780                    if !excluded[i] {
24781                        self.stamp_cycle_error(m.vertex, &circ_error, None);
24782                        excluded[i] = true;
24783                        last_value[i] = circ_error.clone();
24784                        stamped += 1;
24785                    }
24786                }
24787                break;
24788            }
24789
24790            check_cancel(cancel_flag)?;
24791            // Re-evaluate stale readers in live-topo order, recording fresh
24792            // edges (branches may flip on re-eval — spec §7.3 — which is why
24793            // classification repeats).
24794            // A settle pass is a partial sweep: drop the full-pass baseline
24795            // so a live cycle (re)appearing afterwards never compares values
24796            // across mixed pass kinds.
24797            prev_pass = None;
24798            let topo_pos = analysis.topo_positions();
24799            stale.sort_unstable_by_key(|&i| topo_pos[i]);
24800            for x in pos.iter_mut() {
24801                *x = -1;
24802            }
24803            changed.fill(false);
24804            passes += 1;
24805            settle_passes += 1;
24806            for (p, i) in stale.into_iter().enumerate() {
24807                run_member!(i);
24808                pos[i] = p as i64;
24809            }
24810        }
24811
24812        // Iteration that ended because the live cycle dissolved and the
24813        // acyclic settle reached exactness counts as converged (values are
24814        // exact, strictly better than threshold-converged). The defensive
24815        // settle cap (`capped` + stamping) is not.
24816        if iterating && !converged && !capped {
24817            converged = true;
24818        }
24819
24820        // ── 5. End of task: one delta per member whose final value differs
24821        // from the pre-task snapshot (spec §3 side-effect rule, G11).
24822        collector.clear_current();
24823        if let Some(d) = delta
24824            && d.mode != DeltaMode::Off
24825        {
24826            for (i, m) in members.iter().enumerate() {
24827                if let Some(cell) = m.cell
24828                    && last_value[i] != snapshot[i]
24829                {
24830                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24831                }
24832            }
24833        }
24834
24835        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
24836        // Excel's circular cells: register them for the end-of-recalc
24837        // volatile-like redirty (see `pending_iterative_redirty`). Marking
24838        // any one member propagates around the (strongly connected) SCC and
24839        // to downstream dependents, but all members are registered so the
24840        // contract survives partial structural edits between recalcs.
24841        if iterating {
24842            self.pending_iterative_redirty
24843                .extend(members.iter().map(|m| m.vertex));
24844        }
24845
24846        {
24847            let t = &mut self.last_cycle_telemetry;
24848            t.static_sccs += 1;
24849            if witnessed_cycles == 0 && stamped == 0 && !capped {
24850                t.phantom_sccs += 1;
24851            }
24852            t.live_cycles_witnessed += witnessed_cycles;
24853            t.circ_cells_stamped += stamped;
24854            t.settle_passes_total += passes;
24855            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
24856            if iterating {
24857                t.iterated_sccs += 1;
24858                if converged {
24859                    t.converged_sccs += 1;
24860                }
24861                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
24862                t.nan_converged += iter_nan_converged;
24863            }
24864            if capped {
24865                t.capped_sccs += 1;
24866            }
24867            t.elapsed_ms += task_start.elapsed().as_millis();
24868        }
24869
24870        Ok(stamped)
24871    }
24872
24873    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
24874    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
24875    /// reads that actually occur are captured as live edges. Value semantics
24876    /// must match `evaluate_vertex_immutable` exactly (including the missing-
24877    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
24878    /// delegate to it after recording the definition region by hand (those
24879    /// reads bypass the context).
24880    fn evaluate_vertex_recorded(
24881        &self,
24882        vertex_id: VertexId,
24883        ctx: &RecordingContext<'_, R>,
24884        collector: &LiveEdgeCollector,
24885    ) -> Result<LiteralValue, ExcelError> {
24886        if !self.graph.vertex_exists(vertex_id) {
24887            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24888                .with_message(format!("Vertex not found: {vertex_id:?}")));
24889        }
24890
24891        let kind = self.graph.get_vertex_kind(vertex_id);
24892        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24893
24894        match kind {
24895            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24896                let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
24897                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
24898                };
24899                let sheet_name = self.graph.sheet_name(sheet_id);
24900                let cell_ref = self
24901                    .graph
24902                    .get_cell_ref(vertex_id)
24903                    .expect("cell ref for vertex");
24904                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24905                interpreter
24906                    .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24907                    .map(|cv| {
24908                        crate::engine::result_finalization::finalize_formula_result(
24909                            cv.into_literal(),
24910                        )
24911                    })
24912            }
24913            VertexKind::NamedScalar | VertexKind::NamedArray => {
24914                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24915                    ExcelError::new(ExcelErrorKind::Name)
24916                        .with_message("Named range metadata missing".to_string())
24917                })?;
24918
24919                match &named_range.definition {
24920                    NamedDefinition::Formula { ast, .. } => {
24921                        let context_sheet = match named_range.scope {
24922                            NameScope::Sheet(id) => id,
24923                            NameScope::Workbook => sheet_id,
24924                        };
24925                        let sheet_name = self.graph.sheet_name(context_sheet);
24926                        let cell_ref = self
24927                            .graph
24928                            .get_cell_ref(vertex_id)
24929                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24930                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
24931                        if kind == VertexKind::NamedScalar {
24932                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24933                        } else {
24934                            match interpreter.evaluate_ast(ast) {
24935                                Ok(cv) => match cv.into_literal() {
24936                                    v @ LiteralValue::Array(_) => Ok(v),
24937                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
24938                                },
24939                                Err(err) => Ok(LiteralValue::Error(err)),
24940                            }
24941                        }
24942                    }
24943                    NamedDefinition::Cell(cell_ref) => {
24944                        // The definition is read via direct grid access in
24945                        // `evaluate_vertex_immutable`; record the live edge
24946                        // by hand before delegating.
24947                        collector.record_scalar(
24948                            cell_ref.sheet_id,
24949                            cell_ref.coord.row(),
24950                            cell_ref.coord.col(),
24951                        );
24952                        self.evaluate_vertex_immutable(vertex_id)
24953                    }
24954                    NamedDefinition::Range(range_ref) => {
24955                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
24956                            collector.record_rect(
24957                                range_ref.start.sheet_id,
24958                                range_ref.start.coord.row(),
24959                                range_ref.start.coord.col(),
24960                                range_ref.end.coord.row(),
24961                                range_ref.end.coord.col(),
24962                            );
24963                        }
24964                        self.evaluate_vertex_immutable(vertex_id)
24965                    }
24966                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
24967                }
24968            }
24969            _ => self.evaluate_vertex_immutable(vertex_id),
24970        }
24971    }
24972
24973    /// Helper: commit spill via shim and mirror resulting cells into Arrow overlay when enabled.
24974    fn commit_spill_and_mirror(
24975        &mut self,
24976        anchor_vertex: VertexId,
24977        targets: &[CellRef],
24978        rows: Vec<Vec<LiteralValue>>,
24979        delta: Option<&mut DeltaCollector>,
24980        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24981    ) -> Result<(), ExcelError> {
24982        let prev_spill_cells = self
24983            .graph
24984            .spill_cells_for_anchor(anchor_vertex)
24985            .map(|cells| cells.to_vec())
24986            .unwrap_or_default();
24987
24988        if let Some(delta) = delta
24989            && delta.mode != DeltaMode::Off
24990        {
24991            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
24992                targets.iter().copied().collect();
24993            let empty = LiteralValue::Empty;
24994
24995            // Clears (prev - targets)
24996            for cell in prev_spill_cells.iter() {
24997                if target_set.contains(cell) {
24998                    continue;
24999                }
25000                let sheet_name = self.graph.sheet_name(cell.sheet_id);
25001                let old = self
25002                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25003                    .unwrap_or(LiteralValue::Empty);
25004                if old != empty {
25005                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25006                }
25007            }
25008
25009            // Writes (targets)
25010            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
25011                let width = rows[0].len();
25012                for (idx, cell) in targets.iter().enumerate() {
25013                    let r_off = idx / width;
25014                    let c_off = idx % width;
25015                    let new = rows
25016                        .get(r_off)
25017                        .and_then(|r| r.get(c_off))
25018                        .cloned()
25019                        .unwrap_or(LiteralValue::Empty);
25020                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
25021                    let old = self
25022                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25023                        .unwrap_or(LiteralValue::Empty);
25024                    if old != new {
25025                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25026                    }
25027                }
25028            } else {
25029                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
25030                for cell in targets.iter() {
25031                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
25032                    let old = self
25033                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25034                        .unwrap_or(LiteralValue::Empty);
25035                    if !matches!(old, LiteralValue::Empty) {
25036                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25037                    }
25038                }
25039            }
25040        }
25041
25042        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
25043        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
25044        let arrow_sheets = &self.arrow_sheets;
25045        self.spill_mgr.commit_array_with_value_probe(
25046            &mut self.graph,
25047            anchor_vertex,
25048            targets,
25049            rows.clone(),
25050            overwritable_formulas,
25051            |g, cell| {
25052                let sheet_name = g.sheet_name(cell.sheet_id);
25053                let asheet = arrow_sheets.sheet(sheet_name)?;
25054                let r0 = cell.coord.row() as usize;
25055                let c0 = cell.coord.col() as usize;
25056                let v = asheet.get_cell_value(r0, c0);
25057                if matches!(v, LiteralValue::Empty) {
25058                    None
25059                } else {
25060                    Some(v)
25061                }
25062            },
25063        )?;
25064
25065        if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
25066            self.record_formula_plane_structural_change(scope);
25067        }
25068        if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
25069            self.record_formula_plane_structural_change(scope);
25070        }
25071
25072        if self.config.arrow_storage_enabled
25073            && self.config.delta_overlay_enabled
25074            && self.config.write_formula_overlay_enabled
25075        {
25076            if !prev_spill_cells.is_empty() {
25077                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
25078                    targets.iter().copied().collect();
25079                let empty = LiteralValue::Empty;
25080                for cell in prev_spill_cells.iter() {
25081                    if !target_set.contains(cell) {
25082                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25083                        self.mirror_value_to_computed_overlay(
25084                            &sheet_name,
25085                            cell.coord.row() + 1,
25086                            cell.coord.col() + 1,
25087                            &empty,
25088                        );
25089                    }
25090                }
25091            }
25092
25093            for (idx, cell) in targets.iter().enumerate() {
25094                if rows.is_empty() || rows[0].is_empty() {
25095                    break;
25096                }
25097                let width = rows[0].len();
25098                let r_off = idx / width;
25099                let c_off = idx % width;
25100                let v = rows[r_off][c_off].clone();
25101                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25102                self.mirror_value_to_computed_overlay(
25103                    &sheet_name,
25104                    cell.coord.row() + 1,
25105                    cell.coord.col() + 1,
25106                    &v,
25107                );
25108            }
25109        }
25110        Ok(())
25111    }
25112}
25113
25114// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
25115//
25116// Compute → Plan → Apply separation for evaluation side-effects.
25117
25118use crate::engine::effects::Effect;
25119use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
25120
25121impl<R> Engine<R>
25122where
25123    R: EvaluationContext,
25124{
25125    /// Plan effects for a single vertex after its value has been computed.
25126    ///
25127    /// This reads graph state but only performs lightweight mutations
25128    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
25129    /// during the planning phase.  Value-changing mutations are deferred to
25130    /// `apply_effect`.
25131    pub(crate) fn plan_vertex_effects(
25132        &mut self,
25133        vertex_id: VertexId,
25134        computed_value: LiteralValue,
25135        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25136    ) -> Result<Vec<Effect>, ExcelError> {
25137        let kind = self.graph.get_vertex_kind(vertex_id);
25138        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
25139
25140        // If this vertex's cell is currently covered by a spill from a different
25141        // anchor, ignore the computed result.  Formula vertices are exempt:
25142        // they must still evaluate so that overlapping spills produce #SPILL!.
25143        if !is_formula {
25144            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
25145                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
25146                && owner != vertex_id
25147            {
25148                return Ok(Vec::new());
25149            }
25150            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
25151            return Ok(vec![Effect::WriteCell {
25152                vertex_id,
25153                value: computed_value,
25154            }]);
25155        }
25156
25157        match computed_value {
25158            LiteralValue::Array(rows) => {
25159                self.plan_array_effects(vertex_id, rows, overwritable_formulas)
25160            }
25161            other => self.plan_scalar_effects(vertex_id, other),
25162        }
25163    }
25164
25165    /// Plan effects for a formula vertex that produced a scalar/error result.
25166    fn plan_scalar_effects(
25167        &self,
25168        vertex_id: VertexId,
25169        value: LiteralValue,
25170    ) -> Result<Vec<Effect>, ExcelError> {
25171        let has_spill = self
25172            .graph
25173            .spill_cells_for_anchor(vertex_id)
25174            .is_some_and(|c| !c.is_empty());
25175
25176        let mut effects = Vec::new();
25177        if has_spill {
25178            effects.push(Effect::SpillClear {
25179                anchor_vertex: vertex_id,
25180            });
25181        }
25182        effects.push(Effect::WriteCell { vertex_id, value });
25183        Ok(effects)
25184    }
25185
25186    /// Plan effects for a formula vertex that produced an array result.
25187    fn plan_array_effects(
25188        &mut self,
25189        vertex_id: VertexId,
25190        rows: Vec<Vec<LiteralValue>>,
25191        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25192    ) -> Result<Vec<Effect>, ExcelError> {
25193        // Lightweight mutation needed for correct spill-blocking checks.
25194        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
25195
25196        let anchor = self
25197            .graph
25198            .get_cell_ref(vertex_id)
25199            .expect("cell ref for vertex");
25200        let sheet_id = anchor.sheet_id;
25201        let h = rows.len() as u32;
25202        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
25203
25204        // Hard cap to avoid vertex explosion from huge dynamic arrays.
25205        let spill_cells = (h as u64).saturating_mul(w as u64);
25206        if spill_cells > self.config.spill.max_spill_cells as u64 {
25207            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
25208        }
25209
25210        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
25211        const PACKED_MAX_ROW: u32 = 1_048_575;
25212        const PACKED_MAX_COL: u32 = 16_383;
25213        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
25214        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
25215        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
25216            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
25217        }
25218
25219        let mut targets = Vec::new();
25220        for r in 0..h {
25221            for c in 0..w {
25222                targets.push(self.graph.make_cell_ref_internal(
25223                    sheet_id,
25224                    anchor.coord.row() + r,
25225                    anchor.coord.col() + c,
25226                ));
25227            }
25228        }
25229
25230        // Region lock via spill manager.
25231        match self.spill_mgr.reserve(
25232            vertex_id,
25233            anchor,
25234            SpillShape { rows: h, cols: w },
25235            SpillMeta {
25236                epoch: self.recalc_epoch,
25237                config: self.config.spill,
25238            },
25239        ) {
25240            Ok(()) => {
25241                // Validate spill region is available.
25242                if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
25243                    vertex_id,
25244                    &targets,
25245                    overwritable_formulas,
25246                ) {
25247                    return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
25248                }
25249
25250                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
25251                // cell values are not cached in the dependency graph. Consult Arrow storage to
25252                // detect occupied cells in the target region.
25253                if !self.graph.value_cache_enabled() {
25254                    let sheet_name = self.graph.sheet_name(sheet_id);
25255                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25256                        for cell in targets.iter() {
25257                            // Allow overwriting the anchor itself.
25258                            if *cell == anchor {
25259                                continue;
25260                            }
25261                            // Allow cells already owned by a spill (plan() validated spill ownership).
25262                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
25263                                continue;
25264                            }
25265                            // Skip formula blockers; plan() handled them (or allowed).
25266                            if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
25267                                && vid != vertex_id
25268                            {
25269                                match self.graph.get_vertex_kind(vid) {
25270                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
25271                                        continue;
25272                                    }
25273                                    _ => {}
25274                                }
25275                            }
25276
25277                            let v = asheet.get_cell_value(
25278                                cell.coord.row() as usize,
25279                                cell.coord.col() as usize,
25280                            );
25281                            if !matches!(v, LiteralValue::Empty) {
25282                                return self.plan_spill_error_effects(
25283                                    vertex_id,
25284                                    "BlockedByValue",
25285                                    h,
25286                                    w,
25287                                );
25288                            }
25289                        }
25290                    }
25291                }
25292
25293                let top_left = rows
25294                    .first()
25295                    .and_then(|r| r.first())
25296                    .cloned()
25297                    .unwrap_or(LiteralValue::Empty);
25298
25299                let mut effects = Vec::new();
25300                // Clear previous spill if any.
25301                let has_prev = self
25302                    .graph
25303                    .spill_cells_for_anchor(vertex_id)
25304                    .is_some_and(|c| !c.is_empty());
25305                if has_prev {
25306                    effects.push(Effect::SpillClear {
25307                        anchor_vertex: vertex_id,
25308                    });
25309                }
25310                effects.push(Effect::SpillCommit {
25311                    anchor_vertex: vertex_id,
25312                    anchor_cell: anchor,
25313                    target_cells: targets,
25314                    values: rows,
25315                });
25316                effects.push(Effect::WriteCell {
25317                    vertex_id,
25318                    value: top_left,
25319                });
25320                Ok(effects)
25321            }
25322            Err(e) => {
25323                let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
25324                self.plan_spill_error_effects(vertex_id, &msg, h, w)
25325            }
25326        }
25327    }
25328
25329    /// Build the effect list for a spill that failed validation.
25330    fn plan_spill_error_effects(
25331        &self,
25332        vertex_id: VertexId,
25333        message: &str,
25334        expected_rows: u32,
25335        expected_cols: u32,
25336    ) -> Result<Vec<Effect>, ExcelError> {
25337        let spill_err = ExcelError::new(ExcelErrorKind::Spill)
25338            .with_message(message)
25339            .with_extra(formualizer_common::ExcelErrorExtra::Spill {
25340                expected_rows,
25341                expected_cols,
25342            });
25343        let spill_val = LiteralValue::Error(spill_err);
25344
25345        let effects = vec![
25346            Effect::SpillClear {
25347                anchor_vertex: vertex_id,
25348            },
25349            Effect::WriteCell {
25350                vertex_id,
25351                value: spill_val,
25352            },
25353        ];
25354        Ok(effects)
25355    }
25356
25357    /// Apply a single effect, performing the actual graph mutations.
25358    pub(crate) fn apply_effect(
25359        &mut self,
25360        effect: &Effect,
25361        delta: Option<&mut DeltaCollector>,
25362        log: Option<&mut ChangeLog>,
25363    ) -> Result<(), ExcelError> {
25364        self.apply_effect_with_computed_writes(effect, delta, log, None)
25365    }
25366
25367    fn apply_effect_with_computed_writes(
25368        &mut self,
25369        effect: &Effect,
25370        delta: Option<&mut DeltaCollector>,
25371        log: Option<&mut ChangeLog>,
25372        computed_writes: Option<&mut ComputedWriteBuffer>,
25373    ) -> Result<(), ExcelError> {
25374        match effect {
25375            Effect::WriteCell { vertex_id, value } => {
25376                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
25377            }
25378            Effect::SpillClear { anchor_vertex } => {
25379                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
25380            }
25381            Effect::SpillCommit {
25382                anchor_vertex,
25383                anchor_cell: _,
25384                target_cells,
25385                values,
25386            } => {
25387                self.apply_spill_commit(
25388                    *anchor_vertex,
25389                    target_cells,
25390                    values.clone(),
25391                    delta,
25392                    log,
25393                    computed_writes,
25394                )?;
25395            }
25396        }
25397        Ok(())
25398    }
25399
25400    /// Apply a WriteCell effect.
25401    fn apply_write_cell(
25402        &mut self,
25403        vertex_id: VertexId,
25404        value: &LiteralValue,
25405        delta: Option<&mut DeltaCollector>,
25406        mut computed_writes: Option<&mut ComputedWriteBuffer>,
25407    ) -> Result<(), ExcelError> {
25408        if let Some(d) = delta
25409            && d.mode != DeltaMode::Off
25410        {
25411            if let Some(buffer) = computed_writes.as_deref_mut() {
25412                self.flush_computed_write_buffer(buffer)?;
25413            }
25414            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
25415                let sheet_name = self.graph.sheet_name(cell.sheet_id);
25416                let old = self
25417                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25418                    .unwrap_or(LiteralValue::Empty);
25419                if old != *value {
25420                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25421                }
25422            }
25423        }
25424        self.graph.update_vertex_value(vertex_id, value.clone());
25425        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
25426        Ok(())
25427    }
25428
25429    /// Apply a SpillClear effect.
25430    fn apply_spill_clear(
25431        &mut self,
25432        anchor_vertex: VertexId,
25433        delta: Option<&mut DeltaCollector>,
25434        log: Option<&mut ChangeLog>,
25435        computed_writes: Option<&mut ComputedWriteBuffer>,
25436    ) -> Result<(), ExcelError> {
25437        if let Some(buffer) = computed_writes {
25438            self.flush_computed_write_buffer(buffer)?;
25439        }
25440
25441        let spill_cells = self
25442            .graph
25443            .spill_cells_for_anchor(anchor_vertex)
25444            .map(|cells| cells.to_vec())
25445            .unwrap_or_default();
25446        if spill_cells.is_empty() {
25447            return Ok(());
25448        }
25449
25450        // Snapshot for ChangeLog before clearing.
25451        let snapshot = if log.is_some() {
25452            self.snapshot_spill_for_anchor(anchor_vertex)
25453        } else {
25454            None
25455        };
25456
25457        // Record delta for cleared cells.
25458        if let Some(d) = delta
25459            && d.mode != DeltaMode::Off
25460        {
25461            let empty = LiteralValue::Empty;
25462            for cell in spill_cells.iter() {
25463                let sheet_name = self.graph.sheet_name(cell.sheet_id);
25464                let old = self
25465                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25466                    .unwrap_or(LiteralValue::Empty);
25467                if old != empty {
25468                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
25469                }
25470            }
25471        }
25472
25473        self.graph.clear_spill_region(anchor_vertex);
25474        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
25475            self.record_formula_plane_structural_change(scope);
25476        }
25477
25478        // Mirror Empty to Arrow overlay for cleared cells.
25479        if self.config.arrow_storage_enabled
25480            && self.config.delta_overlay_enabled
25481            && self.config.write_formula_overlay_enabled
25482        {
25483            let empty = LiteralValue::Empty;
25484            for cell in spill_cells.iter() {
25485                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
25486                self.mirror_value_to_computed_overlay(
25487                    &sheet_name,
25488                    cell.coord.row() + 1,
25489                    cell.coord.col() + 1,
25490                    &empty,
25491                );
25492            }
25493        }
25494
25495        // ChangeLog.
25496        if let Some(log) = log
25497            && let Some(old) = snapshot
25498        {
25499            log.record(ChangeEvent::SpillCleared {
25500                anchor: anchor_vertex,
25501                old,
25502            });
25503        }
25504        Ok(())
25505    }
25506
25507    /// Apply a SpillCommit effect.
25508    fn apply_spill_commit(
25509        &mut self,
25510        anchor_vertex: VertexId,
25511        target_cells: &[CellRef],
25512        values: Vec<Vec<LiteralValue>>,
25513        delta: Option<&mut DeltaCollector>,
25514        log: Option<&mut ChangeLog>,
25515        computed_writes: Option<&mut ComputedWriteBuffer>,
25516    ) -> Result<(), ExcelError> {
25517        if let Some(buffer) = computed_writes {
25518            self.flush_computed_write_buffer(buffer)?;
25519        }
25520
25521        // Snapshot for ChangeLog before commit.
25522        let old_snapshot = if log.is_some() {
25523            self.snapshot_spill_for_anchor(anchor_vertex)
25524        } else {
25525            None
25526        };
25527
25528        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
25529        self.commit_spill_and_mirror(
25530            anchor_vertex,
25531            target_cells,
25532            values.clone(),
25533            delta,
25534            None, // overwritable_formulas already validated in plan phase
25535        )?;
25536
25537        // ChangeLog.
25538        if let Some(log) = log {
25539            log.record(ChangeEvent::SpillCommitted {
25540                anchor: anchor_vertex,
25541                old: old_snapshot,
25542                new: SpillSnapshot {
25543                    target_cells: target_cells.to_vec(),
25544                    values,
25545                },
25546            });
25547        }
25548        Ok(())
25549    }
25550
25551    /// Snapshot a spill region for ChangeLog recording.
25552    ///
25553    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
25554    /// without creating a `VertexEditor`.
25555    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
25556        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
25557        if cells.is_empty() {
25558            return None;
25559        }
25560
25561        let max = self.config.spill.max_spill_cells as usize;
25562        let mut cells = cells;
25563        if cells.len() > max {
25564            cells.truncate(max);
25565        }
25566
25567        let first = *cells.first().expect("non-empty spill cells");
25568        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
25569        let row0 = first.coord.row();
25570        let col0 = first.coord.col();
25571
25572        let mut max_row = row0;
25573        let mut max_col = col0;
25574        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
25575        for cell in &cells {
25576            max_row = max_row.max(cell.coord.row());
25577            max_col = max_col.max(cell.coord.col());
25578            let v = self
25579                .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
25580                .unwrap_or(LiteralValue::Empty);
25581            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
25582        }
25583
25584        let rows = (max_row - row0 + 1) as usize;
25585        let cols = (max_col - col0 + 1) as usize;
25586        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
25587        for r in 0..rows {
25588            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
25589            for c in 0..cols {
25590                row.push(
25591                    by_coord
25592                        .get(&(row0 + r as u32, col0 + c as u32))
25593                        .cloned()
25594                        .unwrap_or(LiteralValue::Empty),
25595                );
25596            }
25597            values.push(row);
25598        }
25599
25600        Some(SpillSnapshot {
25601            target_cells: cells,
25602            values,
25603        })
25604    }
25605
25606    fn flush_before_range_dependent_vertex(
25607        &mut self,
25608        vertex_id: VertexId,
25609        computed_writes: &mut ComputedWriteBuffer,
25610    ) -> Result<(), ExcelError> {
25611        if self.graph.get_range_dependencies(vertex_id).is_some() {
25612            self.flush_computed_write_buffer(computed_writes)?;
25613        }
25614        Ok(())
25615    }
25616
25617    fn plan_vertex_effects_with_computed_flush(
25618        &mut self,
25619        vertex_id: VertexId,
25620        computed_value: LiteralValue,
25621        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25622        computed_writes: &mut ComputedWriteBuffer,
25623    ) -> Result<Vec<Effect>, ExcelError> {
25624        if matches!(&computed_value, LiteralValue::Array(_)) {
25625            self.flush_computed_write_buffer(computed_writes)?;
25626        }
25627        self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
25628    }
25629
25630    // ── Layer evaluation via effects pipeline ──────────────────────────────
25631
25632    fn evaluate_small_layer_direct_effects(
25633        &mut self,
25634        layer: &super::scheduler::Layer,
25635        mut delta: Option<&mut DeltaCollector>,
25636        mut log: Option<&mut ChangeLog>,
25637        cancel_flag: Option<&AtomicBool>,
25638        cancel_check_every: usize,
25639        cancel_message: &'static str,
25640    ) -> Result<usize, ExcelError> {
25641        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25642            if cancel_check_every > 0
25643                && i % cancel_check_every == 0
25644                && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
25645            {
25646                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25647                    .with_message(cancel_message.to_string()));
25648            }
25649            let value = match self.evaluate_vertex_immutable(vertex_id) {
25650                Ok(v) => v,
25651                Err(e) => LiteralValue::Error(e),
25652            };
25653            let effects = self.plan_vertex_effects(vertex_id, value, None)?;
25654            for effect in &effects {
25655                self.apply_effect_with_computed_writes(
25656                    effect,
25657                    delta.as_deref_mut(),
25658                    log.as_deref_mut(),
25659                    None,
25660                )?;
25661            }
25662        }
25663        Ok(layer.vertices.len())
25664    }
25665
25666    /// Evaluate a layer sequentially using the effects pipeline.
25667    fn evaluate_layer_sequential_effects(
25668        &mut self,
25669        layer: &super::scheduler::Layer,
25670    ) -> Result<usize, ExcelError> {
25671        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25672            return self.evaluate_small_layer_direct_effects(
25673                layer,
25674                None,
25675                None,
25676                None,
25677                0,
25678                "Evaluation cancelled within layer",
25679            );
25680        }
25681
25682        let mut computed_writes = ComputedWriteBuffer::default();
25683        for &vertex_id in &layer.vertices {
25684            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25685            let value = match self.evaluate_vertex_immutable(vertex_id) {
25686                Ok(v) => v,
25687                Err(e) => LiteralValue::Error(e),
25688            };
25689            let effects = match self.plan_vertex_effects_with_computed_flush(
25690                vertex_id,
25691                value,
25692                None,
25693                &mut computed_writes,
25694            ) {
25695                Ok(effects) => effects,
25696                Err(e) => {
25697                    self.flush_computed_write_buffer(&mut computed_writes)?;
25698                    return Err(e);
25699                }
25700            };
25701            for effect in &effects {
25702                if let Err(e) = self.apply_effect_with_computed_writes(
25703                    effect,
25704                    None,
25705                    None,
25706                    Some(&mut computed_writes),
25707                ) {
25708                    self.flush_computed_write_buffer(&mut computed_writes)?;
25709                    return Err(e);
25710                }
25711            }
25712        }
25713        self.flush_computed_write_buffer(&mut computed_writes)?;
25714        Ok(layer.vertices.len())
25715    }
25716
25717    /// Evaluate a layer sequentially with delta collection via effects pipeline.
25718    fn evaluate_layer_sequential_with_delta_effects(
25719        &mut self,
25720        layer: &super::scheduler::Layer,
25721        delta: &mut DeltaCollector,
25722    ) -> Result<usize, ExcelError> {
25723        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25724            return self.evaluate_small_layer_direct_effects(
25725                layer,
25726                Some(delta),
25727                None,
25728                None,
25729                0,
25730                "Evaluation cancelled within layer",
25731            );
25732        }
25733
25734        let mut computed_writes = ComputedWriteBuffer::default();
25735        for &vertex_id in &layer.vertices {
25736            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25737            let value = match self.evaluate_vertex_immutable(vertex_id) {
25738                Ok(v) => v,
25739                Err(e) => LiteralValue::Error(e),
25740            };
25741            let effects = match self.plan_vertex_effects_with_computed_flush(
25742                vertex_id,
25743                value,
25744                None,
25745                &mut computed_writes,
25746            ) {
25747                Ok(effects) => effects,
25748                Err(e) => {
25749                    self.flush_computed_write_buffer(&mut computed_writes)?;
25750                    return Err(e);
25751                }
25752            };
25753            for effect in &effects {
25754                if let Err(e) = self.apply_effect_with_computed_writes(
25755                    effect,
25756                    Some(delta),
25757                    None,
25758                    Some(&mut computed_writes),
25759                ) {
25760                    self.flush_computed_write_buffer(&mut computed_writes)?;
25761                    return Err(e);
25762                }
25763            }
25764        }
25765        self.flush_computed_write_buffer(&mut computed_writes)?;
25766        Ok(layer.vertices.len())
25767    }
25768
25769    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
25770    fn evaluate_layer_sequential_cancellable_effects(
25771        &mut self,
25772        layer: &super::scheduler::Layer,
25773        cancel_flag: &AtomicBool,
25774    ) -> Result<usize, ExcelError> {
25775        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25776            return self.evaluate_small_layer_direct_effects(
25777                layer,
25778                None,
25779                None,
25780                Some(cancel_flag),
25781                256,
25782                "Evaluation cancelled within layer",
25783            );
25784        }
25785
25786        let mut computed_writes = ComputedWriteBuffer::default();
25787        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25788            if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
25789                self.flush_computed_write_buffer(&mut computed_writes)?;
25790                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25791                    .with_message("Evaluation cancelled within layer".to_string()));
25792            }
25793            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25794            let value = match self.evaluate_vertex_immutable(vertex_id) {
25795                Ok(v) => v,
25796                Err(e) => LiteralValue::Error(e),
25797            };
25798            let effects = match self.plan_vertex_effects_with_computed_flush(
25799                vertex_id,
25800                value,
25801                None,
25802                &mut computed_writes,
25803            ) {
25804                Ok(effects) => effects,
25805                Err(e) => {
25806                    self.flush_computed_write_buffer(&mut computed_writes)?;
25807                    return Err(e);
25808                }
25809            };
25810            for effect in &effects {
25811                if let Err(e) = self.apply_effect_with_computed_writes(
25812                    effect,
25813                    None,
25814                    None,
25815                    Some(&mut computed_writes),
25816                ) {
25817                    self.flush_computed_write_buffer(&mut computed_writes)?;
25818                    return Err(e);
25819                }
25820            }
25821        }
25822        self.flush_computed_write_buffer(&mut computed_writes)?;
25823        Ok(layer.vertices.len())
25824    }
25825
25826    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
25827    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
25828        &mut self,
25829        layer: &super::scheduler::Layer,
25830        cancel_flag: &AtomicBool,
25831    ) -> Result<usize, ExcelError> {
25832        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
25833            return self.evaluate_small_layer_direct_effects(
25834                layer,
25835                None,
25836                None,
25837                Some(cancel_flag),
25838                128,
25839                "Demand-driven evaluation cancelled within layer",
25840            );
25841        }
25842
25843        let mut computed_writes = ComputedWriteBuffer::default();
25844        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
25845            if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
25846                self.flush_computed_write_buffer(&mut computed_writes)?;
25847                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
25848                    .with_message("Demand-driven evaluation cancelled within layer".to_string()));
25849            }
25850            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
25851            let value = match self.evaluate_vertex_immutable(vertex_id) {
25852                Ok(v) => v,
25853                Err(e) => LiteralValue::Error(e),
25854            };
25855            let effects = match self.plan_vertex_effects_with_computed_flush(
25856                vertex_id,
25857                value,
25858                None,
25859                &mut computed_writes,
25860            ) {
25861                Ok(effects) => effects,
25862                Err(e) => {
25863                    self.flush_computed_write_buffer(&mut computed_writes)?;
25864                    return Err(e);
25865                }
25866            };
25867            for effect in &effects {
25868                if let Err(e) = self.apply_effect_with_computed_writes(
25869                    effect,
25870                    None,
25871                    None,
25872                    Some(&mut computed_writes),
25873                ) {
25874                    self.flush_computed_write_buffer(&mut computed_writes)?;
25875                    return Err(e);
25876                }
25877            }
25878        }
25879        self.flush_computed_write_buffer(&mut computed_writes)?;
25880        Ok(layer.vertices.len())
25881    }
25882
25883    /// Evaluate a layer in parallel, applying via effects pipeline.
25884    fn evaluate_layer_parallel_effects(
25885        &mut self,
25886        layer: &super::scheduler::Layer,
25887    ) -> Result<usize, ExcelError> {
25888        use rayon::prelude::*;
25889
25890        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
25891
25892        let mut phase1: Vec<VertexId> = Vec::new();
25893        let mut phase2: Vec<VertexId> = Vec::new();
25894        for &vid in &layer.vertices {
25895            if self.graph.get_range_dependencies(vid).is_some() {
25896                phase2.push(vid);
25897            } else {
25898                phase1.push(vid);
25899            }
25900        }
25901
25902        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
25903        let mut applied = 0usize;
25904
25905        for group in [&phase1[..], &phase2[..]] {
25906            if group.is_empty() {
25907                continue;
25908            }
25909            let mut computed_writes = ComputedWriteBuffer::default();
25910
25911            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
25912                thread_pool.install(|| {
25913                    group
25914                        .par_iter()
25915                        .map(
25916                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
25917                                Ok(v) => Ok((vertex_id, v)),
25918                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
25919                            },
25920                        )
25921                        .collect()
25922                });
25923
25924            match results {
25925                Ok(vertex_results) => {
25926                    // Arrays first, then scalars — establishes spill regions before
25927                    // scalar results that might land inside a spilled region.
25928                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
25929                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
25930                    for (vertex_id, result) in vertex_results {
25931                        if matches!(result, LiteralValue::Array(_)) {
25932                            arrays.push((vertex_id, result));
25933                        } else {
25934                            others.push((vertex_id, result));
25935                        }
25936                    }
25937                    for (vertex_id, result) in arrays {
25938                        let effects = match self.plan_vertex_effects_with_computed_flush(
25939                            vertex_id,
25940                            result,
25941                            Some(&inflight),
25942                            &mut computed_writes,
25943                        ) {
25944                            Ok(effects) => effects,
25945                            Err(e) => {
25946                                self.flush_computed_write_buffer(&mut computed_writes)?;
25947                                return Err(e);
25948                            }
25949                        };
25950                        for effect in &effects {
25951                            if let Err(e) = self.apply_effect_with_computed_writes(
25952                                effect,
25953                                None,
25954                                None,
25955                                Some(&mut computed_writes),
25956                            ) {
25957                                self.flush_computed_write_buffer(&mut computed_writes)?;
25958                                return Err(e);
25959                            }
25960                        }
25961                        applied = applied.saturating_add(1);
25962                    }
25963                    // Make all array spill/top-left writes visible before scalar effects in this group.
25964                    self.flush_computed_write_buffer(&mut computed_writes)?;
25965                    for (vertex_id, result) in others {
25966                        let effects = match self.plan_vertex_effects_with_computed_flush(
25967                            vertex_id,
25968                            result,
25969                            Some(&inflight),
25970                            &mut computed_writes,
25971                        ) {
25972                            Ok(effects) => effects,
25973                            Err(e) => {
25974                                self.flush_computed_write_buffer(&mut computed_writes)?;
25975                                return Err(e);
25976                            }
25977                        };
25978                        for effect in &effects {
25979                            if let Err(e) = self.apply_effect_with_computed_writes(
25980                                effect,
25981                                None,
25982                                None,
25983                                Some(&mut computed_writes),
25984                            ) {
25985                                self.flush_computed_write_buffer(&mut computed_writes)?;
25986                                return Err(e);
25987                            }
25988                        }
25989                        applied = applied.saturating_add(1);
25990                    }
25991                    // Flush at the group boundary; phase1 must be visible before phase2.
25992                    self.flush_computed_write_buffer(&mut computed_writes)?;
25993                }
25994                Err(e) => {
25995                    self.flush_computed_write_buffer(&mut computed_writes)?;
25996                    return Err(e);
25997                }
25998            }
25999        }
26000
26001        Ok(applied)
26002    }
26003
26004    /// Evaluate a layer in parallel with delta collection via effects pipeline.
26005    fn evaluate_layer_parallel_with_delta_effects(
26006        &mut self,
26007        layer: &super::scheduler::Layer,
26008        delta: &mut DeltaCollector,
26009    ) -> Result<usize, ExcelError> {
26010        use rayon::prelude::*;
26011
26012        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26013
26014        let mut phase1: Vec<VertexId> = Vec::new();
26015        let mut phase2: Vec<VertexId> = Vec::new();
26016        for &vid in &layer.vertices {
26017            if self.graph.get_range_dependencies(vid).is_some() {
26018                phase2.push(vid);
26019            } else {
26020                phase1.push(vid);
26021            }
26022        }
26023
26024        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26025        let mut applied = 0usize;
26026
26027        for group in [&phase1[..], &phase2[..]] {
26028            if group.is_empty() {
26029                continue;
26030            }
26031            let mut computed_writes = ComputedWriteBuffer::default();
26032            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26033                thread_pool.install(|| {
26034                    group
26035                        .par_iter()
26036                        .map(
26037                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
26038                                Ok(v) => Ok((vertex_id, v)),
26039                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26040                            },
26041                        )
26042                        .collect()
26043                });
26044
26045            match results {
26046                Ok(vertex_results) => {
26047                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26048                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26049                    for (vertex_id, result) in vertex_results {
26050                        if matches!(result, LiteralValue::Array(_)) {
26051                            arrays.push((vertex_id, result));
26052                        } else {
26053                            others.push((vertex_id, result));
26054                        }
26055                    }
26056                    for (vertex_id, result) in arrays {
26057                        let effects = match self.plan_vertex_effects_with_computed_flush(
26058                            vertex_id,
26059                            result,
26060                            Some(&inflight),
26061                            &mut computed_writes,
26062                        ) {
26063                            Ok(effects) => effects,
26064                            Err(e) => {
26065                                self.flush_computed_write_buffer(&mut computed_writes)?;
26066                                return Err(e);
26067                            }
26068                        };
26069                        for effect in &effects {
26070                            if let Err(e) = self.apply_effect_with_computed_writes(
26071                                effect,
26072                                Some(delta),
26073                                None,
26074                                Some(&mut computed_writes),
26075                            ) {
26076                                self.flush_computed_write_buffer(&mut computed_writes)?;
26077                                return Err(e);
26078                            }
26079                        }
26080                        applied = applied.saturating_add(1);
26081                    }
26082                    self.flush_computed_write_buffer(&mut computed_writes)?;
26083                    for (vertex_id, result) in others {
26084                        let effects = match self.plan_vertex_effects_with_computed_flush(
26085                            vertex_id,
26086                            result,
26087                            Some(&inflight),
26088                            &mut computed_writes,
26089                        ) {
26090                            Ok(effects) => effects,
26091                            Err(e) => {
26092                                self.flush_computed_write_buffer(&mut computed_writes)?;
26093                                return Err(e);
26094                            }
26095                        };
26096                        for effect in &effects {
26097                            if let Err(e) = self.apply_effect_with_computed_writes(
26098                                effect,
26099                                Some(delta),
26100                                None,
26101                                Some(&mut computed_writes),
26102                            ) {
26103                                self.flush_computed_write_buffer(&mut computed_writes)?;
26104                                return Err(e);
26105                            }
26106                        }
26107                        applied = applied.saturating_add(1);
26108                    }
26109                    self.flush_computed_write_buffer(&mut computed_writes)?;
26110                }
26111                Err(e) => {
26112                    self.flush_computed_write_buffer(&mut computed_writes)?;
26113                    return Err(e);
26114                }
26115            }
26116        }
26117
26118        Ok(applied)
26119    }
26120
26121    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
26122    fn evaluate_layer_parallel_cancellable_effects(
26123        &mut self,
26124        layer: &super::scheduler::Layer,
26125        cancel_flag: &AtomicBool,
26126    ) -> Result<usize, ExcelError> {
26127        use rayon::prelude::*;
26128
26129        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
26130
26131        if cancel_flag.load(Ordering::Relaxed) {
26132            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26133                .with_message("Parallel evaluation cancelled before starting".to_string()));
26134        }
26135
26136        let mut phase1: Vec<VertexId> = Vec::new();
26137        let mut phase2: Vec<VertexId> = Vec::new();
26138        for &vid in &layer.vertices {
26139            if self.graph.get_range_dependencies(vid).is_some() {
26140                phase2.push(vid);
26141            } else {
26142                phase1.push(vid);
26143            }
26144        }
26145
26146        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
26147        let mut applied = 0usize;
26148
26149        for group in [&phase1[..], &phase2[..]] {
26150            if group.is_empty() {
26151                continue;
26152            }
26153            let mut computed_writes = ComputedWriteBuffer::default();
26154
26155            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
26156                thread_pool.install(|| {
26157                    group
26158                        .par_iter()
26159                        .map(|&vertex_id| {
26160                            if cancel_flag.load(Ordering::Relaxed) {
26161                                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26162                                    .with_message(
26163                                        "Parallel evaluation cancelled during execution"
26164                                            .to_string(),
26165                                    ));
26166                            }
26167                            match self.evaluate_vertex_immutable(vertex_id) {
26168                                Ok(v) => Ok((vertex_id, v)),
26169                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
26170                            }
26171                        })
26172                        .collect()
26173                });
26174
26175            match results {
26176                Ok(vertex_results) => {
26177                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
26178                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
26179                    for (vertex_id, result) in vertex_results {
26180                        if matches!(result, LiteralValue::Array(_)) {
26181                            arrays.push((vertex_id, result));
26182                        } else {
26183                            others.push((vertex_id, result));
26184                        }
26185                    }
26186                    for (vertex_id, result) in arrays {
26187                        let effects = match self.plan_vertex_effects_with_computed_flush(
26188                            vertex_id,
26189                            result,
26190                            Some(&inflight),
26191                            &mut computed_writes,
26192                        ) {
26193                            Ok(effects) => effects,
26194                            Err(e) => {
26195                                self.flush_computed_write_buffer(&mut computed_writes)?;
26196                                return Err(e);
26197                            }
26198                        };
26199                        for effect in &effects {
26200                            if let Err(e) = self.apply_effect_with_computed_writes(
26201                                effect,
26202                                None,
26203                                None,
26204                                Some(&mut computed_writes),
26205                            ) {
26206                                self.flush_computed_write_buffer(&mut computed_writes)?;
26207                                return Err(e);
26208                            }
26209                        }
26210                        applied = applied.saturating_add(1);
26211                    }
26212                    self.flush_computed_write_buffer(&mut computed_writes)?;
26213                    for (vertex_id, result) in others {
26214                        let effects = match self.plan_vertex_effects_with_computed_flush(
26215                            vertex_id,
26216                            result,
26217                            Some(&inflight),
26218                            &mut computed_writes,
26219                        ) {
26220                            Ok(effects) => effects,
26221                            Err(e) => {
26222                                self.flush_computed_write_buffer(&mut computed_writes)?;
26223                                return Err(e);
26224                            }
26225                        };
26226                        for effect in &effects {
26227                            if let Err(e) = self.apply_effect_with_computed_writes(
26228                                effect,
26229                                None,
26230                                None,
26231                                Some(&mut computed_writes),
26232                            ) {
26233                                self.flush_computed_write_buffer(&mut computed_writes)?;
26234                                return Err(e);
26235                            }
26236                        }
26237                        applied = applied.saturating_add(1);
26238                    }
26239                    self.flush_computed_write_buffer(&mut computed_writes)?;
26240                }
26241                Err(e) => {
26242                    self.flush_computed_write_buffer(&mut computed_writes)?;
26243                    return Err(e);
26244                }
26245            }
26246        }
26247
26248        Ok(applied)
26249    }
26250
26251    // ── Top-level evaluate_all_logged ───────────────────────────────────────
26252
26253    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
26254    ///
26255    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
26256    /// every effect application so that spill commits/clears are captured.
26257    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
26258        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
26259            engine.evaluate_all_logged_unobserved(log)
26260        })
26261    }
26262
26263    fn evaluate_all_logged_unobserved(
26264        &mut self,
26265        log: &mut ChangeLog,
26266    ) -> Result<EvalResult, ExcelError> {
26267        self.observe_function_semantic_epoch()?;
26268        self.begin_evaluation_request();
26269        let _source_cache = self.source_cache_session();
26270        self.validate_deterministic_mode()?;
26271        if self.config.defer_graph_building {
26272            self.build_graph_all()?;
26273        }
26274        if self.graph.formula_authority().active_span_count() > 0 {
26275            return self.evaluate_authoritative_formula_plane_all();
26276        }
26277        self.reset_virtual_dep_telemetry_if_disabled();
26278        let start = crate::instant::FzInstant::now();
26279        let mut computed_vertices = 0;
26280        let mut cycle_errors = 0;
26281
26282        let mut replan_iterations = 0;
26283        const MAX_REPLAN: usize = 5;
26284        let mut telemetry = self
26285            .config
26286            .enable_virtual_dep_telemetry
26287            .then(|| self.start_virtual_dep_telemetry());
26288
26289        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
26290
26291        let result = (|| -> Result<EvalResult, ExcelError> {
26292            loop {
26293                let to_evaluate = self.graph.get_evaluation_vertices();
26294                if to_evaluate.is_empty() {
26295                    if let Some(t) = telemetry.as_mut()
26296                        && t.bailout_reason.is_none()
26297                    {
26298                        t.bailout_reason = Some("no_work");
26299                    }
26300                    break;
26301                }
26302
26303                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
26304                if let Some(t) = telemetry.as_mut() {
26305                    Self::accumulate_schedule_meta(t, &meta);
26306                }
26307
26308                // Walk units in condensation order: stamp cycles at their
26309                // position, evaluate layers with ChangeLog recording.
26310                for &unit in &schedule.units {
26311                    match unit {
26312                        ScheduleUnit::Cycle(i) => {
26313                            // Journal integration (design doc §4 last row): the
26314                            // ChangeLog in this path only records SpillClear /
26315                            // SpillCommit events; WriteCell effects are never
26316                            // logged (see `apply_write_cell`). Runtime SCC tasks
26317                            // write values directly and never spill (§7.9 stamps
26318                            // would-be anchors), and their spill *teardown* is the
26319                            // same unlogged `stamp_cycle_error` the Static path
26320                            // already uses here — so direct commits coexist with
26321                            // the journal cleanly, with identical semantics to
26322                            // Static. Pinned by `scc_runtime_cycles` tests.
26323                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
26324                            {
26325                                cycle_errors += 1;
26326                            }
26327                        }
26328                        ScheduleUnit::Layer(i) => {
26329                            computed_vertices +=
26330                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
26331                        }
26332                    }
26333                }
26334
26335                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
26336                if let Some(t) = telemetry.as_mut() {
26337                    t.changed_vdeps_total += changed_vertices.len();
26338                }
26339                self.resource_checkpoint(0)?;
26340                self.graph.clear_dirty_flags(&to_evaluate);
26341                for v in &changed_vertices {
26342                    self.graph.set_dirty(*v, true);
26343                }
26344
26345                if changed_vertices.is_empty() {
26346                    if let Some(t) = telemetry.as_mut() {
26347                        t.bailout_reason = Some("converged");
26348                    }
26349                    break;
26350                }
26351                if replan_iterations >= MAX_REPLAN {
26352                    if let Some(mut t) = telemetry.take() {
26353                        t.bailout_reason = Some("max_replan");
26354                        t.replan_iterations = replan_iterations;
26355                        self.last_virtual_dep_telemetry = t;
26356                    }
26357                    return Err(
26358                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
26359                    );
26360                }
26361                replan_iterations += 1;
26362            }
26363
26364            if let Some(mut t) = telemetry {
26365                t.replan_iterations = replan_iterations;
26366                self.last_virtual_dep_telemetry = t;
26367            }
26368
26369            self.redirty_for_next_recalc();
26370            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
26371
26372            Ok(EvalResult {
26373                computed_vertices,
26374                cycle_errors,
26375                elapsed: start.elapsed(),
26376            })
26377        })();
26378        log.end_compound();
26379        result
26380    }
26381
26382    /// Evaluate a single layer with ChangeLog recording.
26383    fn evaluate_layer_logged(
26384        &mut self,
26385        layer: &super::scheduler::Layer,
26386        log: &mut ChangeLog,
26387    ) -> Result<usize, ExcelError> {
26388        self.resource_checkpoint(layer.vertices.len() as u64)?;
26389        let mut computed_writes = ComputedWriteBuffer::default();
26390        for &vertex_id in &layer.vertices {
26391            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
26392            let value = match self.evaluate_vertex_immutable(vertex_id) {
26393                Ok(v) => v,
26394                Err(e) => LiteralValue::Error(e),
26395            };
26396            let effects = match self.plan_vertex_effects_with_computed_flush(
26397                vertex_id,
26398                value,
26399                None,
26400                &mut computed_writes,
26401            ) {
26402                Ok(effects) => effects,
26403                Err(e) => {
26404                    self.flush_computed_write_buffer(&mut computed_writes)?;
26405                    return Err(e);
26406                }
26407            };
26408            for effect in &effects {
26409                if let Err(e) = self.apply_effect_with_computed_writes(
26410                    effect,
26411                    None,
26412                    Some(log),
26413                    Some(&mut computed_writes),
26414                ) {
26415                    self.flush_computed_write_buffer(&mut computed_writes)?;
26416                    return Err(e);
26417                }
26418            }
26419        }
26420        self.flush_computed_write_buffer(&mut computed_writes)?;
26421        Ok(layer.vertices.len())
26422    }
26423}