Skip to main content

formualizer_eval/engine/
eval.rs

1use crate::SheetId;
2use crate::arrow_store::{OverlayFragment, OverlayValue, SheetStore};
3use crate::engine::arena::AstNodeId;
4use crate::engine::eval_delta::{
5    DeltaCollector, DeltaMode, EvalDelta, EvalDeltaCompatibilityPolicy,
6};
7use crate::engine::graph::editor::change_log::MutationCapture;
8use crate::engine::graph::prepared_legacy_graph::{
9    PreparedLegacyGraphError, PreparedLegacyGraphPlan,
10};
11use crate::engine::graph::{FormulaDirtyEventSnapshot, FormulaDirtyLease, WholeSpanDirtyReason};
12use crate::engine::ingest_pipeline::{DependencyPlanRow, FormulaAstInput};
13use crate::engine::live_edges::{LiveEdgeCollector, RecordingContext};
14use crate::engine::live_graph::analyze_live_graph;
15use crate::engine::lookup_index_cache::{
16    BuildOutcome, LookupAxis, LookupIndex, LookupIndexCache, LookupIndexCacheReport,
17    LookupIndexKey, estimate_bytes,
18};
19use crate::engine::named_range::{NameScope, NamedDefinition};
20use crate::engine::range_view::RangeView;
21use crate::engine::row_visibility::RowVisibilityState;
22use crate::engine::spill::{RegionLockManager, SpillMeta, SpillShape};
23use crate::engine::target_preparation::{
24    StagedFormulaIndex, StagedFormulaLease, StagedPackageLease,
25};
26use crate::engine::used_extent::{
27    ExtentPolicy, OpenRangeBounds, resolve_used_extent_with_fallback,
28};
29use crate::engine::virtual_deps::{DynamicRefVirtualDepProvider, VirtualDepBuilder};
30use crate::engine::{
31    ChangeLogger, CycleDetection, CyclePolicy, DependencyGraph, EvalConfig, EvaluationRequestKind,
32    EvaluationRequestOutcome, EvaluationResourceBaselineStats, EvaluationResourceReason,
33    EvaluationResourceRequestStats, FormulaDirtyLeaseOutcome, FormulaIngestBatch,
34    FormulaIngestRecord, FormulaIngestReport, FormulaParseDiagnostic, FormulaParsePolicy,
35    FormulaPlaneMode, FormulaPlaneRoute, FormulaPlaneRouteEvent, FormulaPlaneRoutePhase,
36    FormulaPlaneRouteTransitionReason, FormulaPlaneTopologyCacheOutcome,
37    FormulaPlaneTopologyStrategy, ResourceLedger, RowVisibilitySource, ScheduleUnit, Scheduler,
38    VertexId, VertexKind, VisibilityMaskMode,
39};
40use crate::formula_plane::placement::prepare_anchor_once_fragment;
41use crate::formula_plane::placement::{
42    CandidateAnalysis, FormulaPlacementCandidate, FormulaPlacementResult, PlacementFallbackReason,
43    place_candidate_family_with_analyses, prepare_anchor_once_family,
44    split_candidate_affine_literal_runs, validate_anchor_once_fragment_shadow,
45    validate_anchor_once_shadow_relocation, validate_anchor_once_syntax,
46};
47use crate::formula_plane::producer::{
48    DirtyProjectionRule, FormulaConsumerReadIndex, FormulaProducerId, FormulaProducerResultIndex,
49    FormulaProducerWork, ProducerDirtyDomain, ProjectionResult, SpanReadSummary,
50};
51use crate::formula_plane::region_index::{BoundedRegionQueryResult, DirtyDomain, Region};
52use crate::formula_plane::runtime::{
53    FormulaPlane, FormulaSpanId, FormulaSpanRef, PlacementCoord, PlacementDomain, ResultRegion,
54};
55use crate::formula_plane::scheduler::{
56    MixedLayer, MixedSchedule, MixedScheduleFallback, MixedScheduleFallbackReason, MixedTopology,
57    MixedTopologyCompileResult, MixedTopologyCompileStats, MixedTopologyConfig,
58    build_demand_closure_cached, build_demand_closure_in_memory_runs, build_demand_closure_paged,
59    build_demand_closure_repeated_passes, compile_mixed_topology, schedule_dirty_work,
60    schedule_dirty_work_in_memory_runs, schedule_dirty_work_paged_hybrid,
61    schedule_dirty_work_repeated_passes,
62};
63#[cfg(not(target_arch = "wasm32"))]
64use crate::formula_plane::scheduler::{NativeExactDemandError, build_demand_closure_native};
65#[cfg(not(target_arch = "wasm32"))]
66use crate::formula_plane::scheduler::{NativeExactScheduleError, schedule_dirty_work_native};
67#[cfg(test)]
68use crate::formula_plane::span_eval::SpanEvalReport;
69use crate::formula_plane::span_eval::{SpanComputedWriteSink, SpanEvalTask, SpanEvaluator};
70use crate::formula_plane::structural::relocate_ast_for_template_placement;
71use crate::formula_plane::structural_shift::{SpanShiftPlan, StructuralOp, classify_span_for_op};
72use crate::function::FnCaps;
73use crate::interpreter::Interpreter;
74use crate::reference::{CellRef, Coord, RangeRef};
75use crate::traits::FunctionProvider;
76use crate::traits::{EvaluationContext, ReferenceInfo, Resolver};
77use formualizer_common::{
78    CoordBuildHasher, LiteralValue, col_letters_from_1based, parse_a1_1based,
79};
80use formualizer_parse::parser::ReferenceType;
81use formualizer_parse::{ASTNode, ASTNodeType, ExcelError, ExcelErrorKind};
82use rayon::ThreadPoolBuilder;
83use rustc_hash::{FxHashMap, FxHashSet};
84use std::collections::{BTreeMap, BTreeSet, VecDeque};
85use std::sync::Arc;
86use std::sync::atomic::{AtomicBool, Ordering};
87use std::time::Instant;
88
89type StagedFormulaEntry = (u32, u32, String);
90type StagedSheetParts = (
91    Vec<StagedFormulaEntry>,
92    Option<crate::engine::DeferredFormulaPackage>,
93);
94
95/// Per-sheet staged-formula store (NOTE(#126) follow-up).
96///
97/// Ingest consumers (`build_graph_all`/`build_graph_for_sheets`) walk staged
98/// entries in INSERTION order, so the order-preserving `Vec` stays the
99/// canonical storage; a `(row, col) → index` map removes the linear dup-scan
100/// that made `stage_formula_text`/`get_staged_formula_text` O(staged-on-sheet)
101/// per call (O(n²) for an n-formula deferred load on one sheet — ~570 ms for
102/// 50k stages, release, before the index). `stage`/`get` are O(1);
103/// `remove` keeps the old O(n) `Vec::remove` (rare path, order preserved).
104#[derive(Default)]
105pub(crate) struct StagedSheet {
106    entries: Vec<StagedFormulaEntry>,
107    index: FxHashMap<(u32, u32), usize>,
108    deferred_package: Option<crate::engine::DeferredFormulaPackage>,
109}
110
111impl StagedSheet {
112    fn invalidate_all_deferred_families(package: &mut crate::engine::DeferredFormulaPackage) {
113        package
114            .invalidated
115            .extend(package.families.iter().map(|family| family.source_id));
116        package.invalidated.extend(
117            package
118                .partitioned_families
119                .iter()
120                .map(|family| family.source_id),
121        );
122    }
123
124    fn invalidate_deferred_at(&mut self, row: u32, col: u32) {
125        let Some(package) = self.deferred_package.as_mut() else {
126            return;
127        };
128        let family = package
129            .replay
130            .lock()
131            .ok()
132            .and_then(|mut replay| replay.formula_at(row, col).ok())
133            .flatten()
134            .and_then(|record| record.partition_owner.or(record.family));
135        if let Some(family) = family {
136            package.invalidated.insert(family);
137        } else {
138            // A poisoned lock, replay failure, malformed spool, or missing
139            // lookup result must never let a possibly edited family commit.
140            // Fail closed at package scope before suppressing the edited cell.
141            Self::invalidate_all_deferred_families(package);
142        }
143        package.suppressed.insert((row, col));
144    }
145
146    fn reconcile_attached_deferred_package(&mut self) {
147        let Some(package) = self.deferred_package.as_mut() else {
148            return;
149        };
150        if self.entries.is_empty() {
151            return;
152        }
153
154        let replayed = (|| {
155            let mut disposition = crate::engine::FormulaReplayDisposition::default();
156            for partition in &package.partitioned_families {
157                disposition.register_partition(partition, false)?;
158            }
159            package
160                .replay
161                .lock()
162                .map_err(|_| "deferred formula spool lock poisoned".to_string())?
163                .replay_partitioned(&disposition, &package.partitioned_families)
164        })();
165
166        let Ok(records) = replayed else {
167            Self::invalidate_all_deferred_families(package);
168            package
169                .suppressed
170                .extend(self.entries.iter().map(|(row, col, _)| (*row, *col)));
171            return;
172        };
173        let mut owners = FxHashMap::default();
174        for record in &records {
175            owners
176                .entry((record.row, record.col))
177                .or_insert(record.partition_owner.or(record.family));
178        }
179        for (row, col, _) in &self.entries {
180            if let Some(family) = owners.get(&(*row, *col)).copied().flatten() {
181                package.invalidated.insert(family);
182            } else {
183                Self::invalidate_all_deferred_families(package);
184            }
185            package.suppressed.insert((*row, *col));
186        }
187        package.reconciliation_replay = Some(records);
188    }
189
190    fn stage(&mut self, row: u32, col: u32, text: String) {
191        self.invalidate_deferred_at(row, col);
192        match self.index.entry((row, col)) {
193            std::collections::hash_map::Entry::Occupied(slot) => {
194                self.entries[*slot.get()].2 = text;
195            }
196            std::collections::hash_map::Entry::Vacant(slot) => {
197                slot.insert(self.entries.len());
198                self.entries.push((row, col, text));
199            }
200        }
201    }
202
203    fn remove(&mut self, row: u32, col: u32) -> Option<String> {
204        let deferred_text = self.get(row, col);
205        self.invalidate_deferred_at(row, col);
206        let Some(idx) = self.index.remove(&(row, col)) else {
207            return deferred_text;
208        };
209        let (_, _, text) = self.entries.remove(idx);
210        // `Vec::remove` shifted everything after `idx` left by one.
211        for slot in self.index.values_mut() {
212            if *slot > idx {
213                *slot -= 1;
214            }
215        }
216        Some(text)
217    }
218
219    fn get_ordinary(&self, row: u32, col: u32) -> Option<&str> {
220        self.index
221            .get(&(row, col))
222            .map(|&i| self.entries[i].2.as_str())
223    }
224
225    fn remove_ordinary(&mut self, row: u32, col: u32) -> Option<String> {
226        let idx = self.index.remove(&(row, col))?;
227        let (_, _, text) = self.entries.remove(idx);
228        for slot in self.index.values_mut() {
229            if *slot > idx {
230                *slot -= 1;
231            }
232        }
233        Some(text)
234    }
235
236    fn get(&self, row: u32, col: u32) -> Option<String> {
237        if let Some(text) = self.get_ordinary(row, col) {
238            return Some(text.to_string());
239        }
240        let package = self.deferred_package.as_ref()?;
241        if package.suppressed.contains(&(row, col)) {
242            return None;
243        }
244        package
245            .replay
246            .lock()
247            .ok()?
248            .formula_at(row, col)
249            .ok()?
250            .map(|record| record.text)
251    }
252
253    fn len(&self) -> usize {
254        self.deferred_package
255            .as_ref()
256            .map_or(self.entries.len(), |package| {
257                usize::try_from(package.report.source_formula_records_spooled)
258                    .unwrap_or(usize::MAX)
259                    .saturating_sub(package.suppressed.len())
260                    .saturating_add(self.entries.len())
261            })
262    }
263
264    fn is_empty(&self) -> bool {
265        self.entries.is_empty() && self.deferred_package.is_none()
266    }
267
268    fn into_parts(self) -> StagedSheetParts {
269        (self.entries, self.deferred_package)
270    }
271}
272
273type StagedFormulaMap = std::collections::HashMap<String, StagedSheet>;
274
275#[derive(Clone, Debug, PartialEq, Eq, Hash)]
276struct PreparationRegion {
277    sheet: String,
278    sheet_id: SheetId,
279    start_row: u32,
280    start_col: u32,
281    end_row: u32,
282    end_col: u32,
283}
284
285// Span roots dedupe by exact demanded-region identity. Overlapping, non-identical
286// regions remain ordered entries; downstream demand treats those entries as a union.
287struct OrderedTargetProducers {
288    ordered: Vec<crate::engine::target_preparation::TargetProducer>,
289    seen: FxHashSet<crate::engine::target_preparation::TargetProducer>,
290}
291
292impl OrderedTargetProducers {
293    fn with_capacity(capacity: usize) -> Result<Self, std::collections::TryReserveError> {
294        let mut ordered = Vec::new();
295        ordered.try_reserve(capacity)?;
296        let mut seen = FxHashSet::default();
297        seen.try_reserve(capacity)?;
298        Ok(Self { ordered, seen })
299    }
300
301    fn from_ordered(
302        ordered: Vec<crate::engine::target_preparation::TargetProducer>,
303    ) -> Result<Self, std::collections::TryReserveError> {
304        let mut seen = FxHashSet::default();
305        seen.try_reserve(ordered.len())?;
306        seen.extend(ordered.iter().copied());
307        Ok(Self { ordered, seen })
308    }
309
310    fn push(
311        &mut self,
312        producer: crate::engine::target_preparation::TargetProducer,
313    ) -> Result<bool, std::collections::TryReserveError> {
314        #[cfg(test)]
315        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(probes.get().saturating_add(1)));
316        if self.seen.contains(&producer) {
317            return Ok(false);
318        }
319        self.seen.try_reserve(1)?;
320        self.ordered.try_reserve(1)?;
321        self.seen.insert(producer);
322        self.ordered.push(producer);
323        Ok(true)
324    }
325
326    fn len(&self) -> usize {
327        self.ordered.len()
328    }
329
330    fn into_vec(self) -> Vec<crate::engine::target_preparation::TargetProducer> {
331        self.ordered
332    }
333}
334
335#[cfg(test)]
336thread_local! {
337    static TARGET_ROOT_DEDUP_PROBES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
338}
339
340fn target_root_allocation_error(observed: usize, request_id: Option<u64>) -> ExcelError {
341    crate::engine::ResourceLedgerError::Exhausted(formualizer_common::ResourceExhaustionDetail {
342        reason: formualizer_common::ResourceExhaustionReason::ScratchMemory,
343        limit: u64::MAX,
344        observed: observed as u64,
345        request_id,
346    })
347    .into_excel_error()
348}
349
350#[derive(Debug)]
351struct PreparedOrdinaryStagedFormula {
352    sheet: String,
353    sheet_id: SheetId,
354    lease: StagedFormulaLease,
355    ast_id: Option<AstNodeId>,
356    plan: Option<DependencyPlanRow>,
357}
358
359struct PreparedTargetSourcePackage {
360    sheet: String,
361    sheet_id: SheetId,
362    lease: StagedPackageLease,
363    source_report: crate::engine::FormulaCompressedSourceReport,
364    replay_records: Vec<crate::engine::DeferredReplayFormula>,
365    disposition: crate::engine::FormulaReplayDisposition,
366    placements: Vec<crate::formula_plane::placement::PreparedAnchorOncePlacement>,
367    legacy: Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
368    direct_families: usize,
369    direct_cells: u64,
370    direct_fragments: u64,
371    direct_complete_families: u64,
372    direct_complete_cells: u64,
373    direct_partition_families: u64,
374    direct_partition_cells: u64,
375    anchor_parses: u64,
376    anchor_asts: u64,
377    anchor_analyses: u64,
378}
379
380impl PreparedTargetSourcePackage {
381    fn fallback_records(&self) -> impl Iterator<Item = &crate::engine::DeferredReplayFormula> {
382        self.replay_records.iter().filter(|record| {
383            let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1)) else {
384                return true;
385            };
386            let coord = crate::engine::SourceCoord { row, col };
387            let disposition = match record.family {
388                Some(family) => self.disposition.shared_disposition(family, coord),
389                None => self.disposition.ordinary_disposition(coord).0,
390            };
391            !matches!(
392                disposition,
393                crate::engine::FormulaReplayCoordinateDisposition::Direct
394                    | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
395            )
396        })
397    }
398}
399
400fn producer_dirty_to_span_dirty(
401    dirty: ProducerDirtyDomain,
402    span_ref: FormulaSpanRef,
403) -> DirtyDomain {
404    match dirty {
405        ProducerDirtyDomain::Whole => DirtyDomain::WholeSpan(span_ref),
406        ProducerDirtyDomain::Cells(cells) => DirtyDomain::Cells(cells),
407        ProducerDirtyDomain::Regions(regions) => DirtyDomain::Regions(regions),
408    }
409}
410type PreparedFormulaBatches = Vec<FormulaIngestBatch>;
411type StagedFormulaBatches = Vec<(String, StagedSheet)>;
412#[derive(Debug)]
413pub(crate) struct SourceFamilyPreparationError {
414    reason: String,
415    parse_attempted: bool,
416    ast_created: bool,
417    analysis_created: bool,
418}
419
420impl SourceFamilyPreparationError {
421    fn contract(reason: &'static str) -> Self {
422        Self {
423            reason: reason.to_string(),
424            parse_attempted: false,
425            ast_created: false,
426            analysis_created: false,
427        }
428    }
429
430    fn parse(reason: &'static str) -> Self {
431        Self {
432            reason: reason.to_string(),
433            parse_attempted: true,
434            ast_created: false,
435            analysis_created: false,
436        }
437    }
438
439    fn ast(reason: impl Into<String>) -> Self {
440        Self {
441            reason: reason.into(),
442            parse_attempted: true,
443            ast_created: true,
444            analysis_created: false,
445        }
446    }
447
448    fn analysis(reason: impl Into<String>) -> Self {
449        Self {
450            reason: reason.into(),
451            parse_attempted: true,
452            ast_created: true,
453            analysis_created: true,
454        }
455    }
456}
457
458struct PreparedPartitionShadow {
459    fragment_count: u64,
460    direct_cells: u64,
461    fallback_cells: u64,
462    function_semantics_used: bool,
463}
464
465pub(crate) fn classify_mixed_topology_incomplete(
466    observed: &MixedTopologyCompileStats,
467) -> crate::engine::EvaluationIncompleteReason {
468    if observed.memory_overflow_count > 0 && observed.estimated_memory_bytes == usize::MAX {
469        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyAllocation
470    } else if observed.memory_overflow_count > 0 {
471        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes
472    } else if observed.edge_overflow_count > 0 {
473        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges
474    } else if observed.candidate_overflow_count > 0 {
475        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates
476    } else {
477        crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologySemanticStructural
478    }
479}
480
481#[derive(Clone, Debug, Default)]
482struct LegacyIslandPlan {
483    membership: Vec<VertexId>,
484    dirty_vertices: Vec<VertexId>,
485    sheet_ids: Vec<SheetId>,
486    island_id: u64,
487    retained_bytes: usize,
488    boundary_relationships: usize,
489    omitted_relationships: usize,
490    omitted_direct_relationships: usize,
491}
492
493impl LegacyIslandPlan {
494    fn is_empty(&self) -> bool {
495        self.membership.is_empty()
496    }
497}
498
499type FormulaPlaneMixedScheduleBuild = (
500    MixedSchedule,
501    BTreeMap<crate::formula_plane::runtime::FormulaSpanId, FormulaSpanRef>,
502    u64,
503    Vec<VertexId>,
504    Vec<usize>,
505    LegacyIslandPlan,
506);
507
508#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
509pub(crate) enum FormulaSpanDemotionFault {
510    #[default]
511    None,
512    AstPreparation,
513    LegacyGraphPreparation,
514    FinalLegacyGraphValidation,
515    FinalAuthorityValidation,
516    AllocationReservation,
517    BeforeFirstMutation,
518}
519
520#[derive(Debug)]
521pub(crate) enum FormulaSpanDemotionError {
522    StaleAuthority,
523    InvalidSpan,
524    CountOverflow,
525    Resource(ExcelError),
526    AstPreparation(ExcelError),
527    LegacyGraph(PreparedLegacyGraphError),
528    Injected(FormulaSpanDemotionFault),
529}
530
531impl std::fmt::Display for FormulaSpanDemotionError {
532    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
533        match self {
534            Self::StaleAuthority => write!(f, "prepared FormulaPlane demotion is stale"),
535            Self::InvalidSpan => write!(f, "FormulaPlane demotion referenced an invalid span"),
536            Self::CountOverflow => write!(f, "FormulaPlane demotion count overflow"),
537            Self::Resource(error) => write!(f, "{error}"),
538            Self::AstPreparation(error) => {
539                write!(f, "FormulaPlane demotion analysis failed: {error}")
540            }
541            Self::LegacyGraph(error) => {
542                write!(f, "FormulaPlane demotion graph plan failed: {error}")
543            }
544            Self::Injected(fault) => write!(f, "injected FormulaPlane demotion fault: {fault:?}"),
545        }
546    }
547}
548
549impl std::error::Error for FormulaSpanDemotionError {}
550
551pub(crate) struct PreparedFormulaSpanDemotion {
552    span_refs: Vec<FormulaSpanRef>,
553    expected_plane_epoch: u64,
554    expected_indexes_epoch: u64,
555    expected_indexed_plane_epoch: u64,
556    placement_count: usize,
557    legacy_graph: PreparedLegacyGraphPlan,
558    fault: FormulaSpanDemotionFault,
559}
560
561#[derive(Clone, Copy, Debug, PartialEq, Eq)]
562pub(crate) struct FormulaSpanDemotionReport {
563    pub(crate) spans_demoted: usize,
564    pub(crate) placements_materialized: usize,
565}
566
567type PlannedFormulaMaterialize = BTreeMap<String, Vec<(u32, u32, AstNodeId, DependencyPlanRow)>>;
568type CompressedReplayBatch = (
569    FormulaIngestBatch,
570    crate::engine::FormulaCompressedSourceBatch,
571);
572type PreparedSourceBatch = (
573    FormulaIngestBatch,
574    crate::engine::FormulaCompressedSourceReport,
575    crate::engine::FormulaCompressedPreparation,
576);
577type PreparedStagedFormulaBatches = (
578    PreparedFormulaBatches,
579    Vec<CompressedReplayBatch>,
580    Vec<PreparedSourceBatch>,
581);
582
583/// Backend-neutral source-family ingress. Adapters may prepare candidates and
584/// submit exact replay, but FormulaPlane authority remains engine-owned.
585#[doc(hidden)]
586pub struct SourceFormulaIngress<'a, R> {
587    engine: &'a mut Engine<R>,
588}
589
590impl<R> SourceFormulaIngress<'_, R>
591where
592    R: EvaluationContext,
593{
594    pub fn prepare_families(
595        &mut self,
596        sheet_name: &str,
597        families: &[crate::engine::SourceFormulaFamily],
598    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
599        self.engine.observe_function_semantic_epoch()?;
600        Ok(self
601            .engine
602            .prepare_source_formula_families(sheet_name, families))
603    }
604
605    pub fn prepare_eager_proposals(
606        &mut self,
607        sheet_name: &str,
608        families: &[crate::engine::SourceFormulaFamily],
609        partitions: &[crate::engine::PartitionedSourceFormulaFamily],
610        formula_record_count: u64,
611        replay: Box<dyn crate::engine::DeferredFormulaReplay>,
612    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
613        self.engine.observe_function_semantic_epoch()?;
614        let replay = Arc::new(std::sync::Mutex::new(replay));
615        self.engine.prepare_source_formula_proposals(
616            sheet_name,
617            families,
618            partitions,
619            partitions,
620            formula_record_count,
621            replay,
622            &BTreeSet::new(),
623        )
624    }
625
626    pub fn stage_deferred(&mut self, package: crate::engine::DeferredFormulaPackage) {
627        self.engine.stage_deferred_formula_package(package);
628    }
629
630    pub fn ingest_replay_batches(
631        &mut self,
632        batches: Vec<(
633            FormulaIngestBatch,
634            crate::engine::FormulaCompressedSourceBatch,
635        )>,
636    ) -> Result<FormulaIngestReport, ExcelError> {
637        self.engine
638            .ingest_compressed_formula_source_batches(batches)
639    }
640
641    pub fn finish_prepared(
642        &mut self,
643        batches: Vec<(
644            FormulaIngestBatch,
645            crate::engine::FormulaCompressedSourceReport,
646            crate::engine::FormulaCompressedPreparation,
647        )>,
648    ) -> Result<FormulaIngestReport, ExcelError> {
649        self.engine.finish_compressed_formula_sources(batches)
650    }
651}
652
653// Computed-write coalescing pays a fixed grouping/planning cost. For very narrow
654// layers there is not enough work to amortize it, and the direct point-write path
655// is faster while preserving the same visibility semantics.
656const COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH: usize = 8;
657
658#[derive(Debug, Clone, PartialEq)]
659pub(crate) enum ComputedWrite {
660    Cell {
661        seq: u64,
662        sheet_id: SheetId,
663        row0: u32,
664        col0: u32,
665        value: OverlayValue,
666        format_id: Option<crate::format::FormatId>,
667    },
668    Rect {
669        seq: u64,
670        sheet_id: SheetId,
671        sr0: u32,
672        sc0: u32,
673        values: Vec<Vec<OverlayValue>>,
674    },
675}
676
677impl ComputedWrite {
678    #[inline]
679    pub(crate) fn seq(&self) -> u64 {
680        match self {
681            ComputedWrite::Cell { seq, .. } | ComputedWrite::Rect { seq, .. } => *seq,
682        }
683    }
684}
685
686#[derive(Debug, Default)]
687pub(crate) struct ComputedWriteBuffer {
688    writes: Vec<ComputedWrite>,
689    next_seq: u64,
690    estimated_bytes: usize,
691    formats_present: bool,
692}
693
694impl ComputedWriteBuffer {
695    const ENTRY_BASE_BYTES: usize = 32;
696
697    #[inline]
698    pub(crate) fn is_empty(&self) -> bool {
699        self.writes.is_empty()
700    }
701
702    #[inline]
703    pub(crate) fn len(&self) -> usize {
704        self.writes.len()
705    }
706
707    #[inline]
708    pub(crate) fn estimated_bytes(&self) -> usize {
709        self.estimated_bytes
710    }
711
712    #[inline]
713    pub(crate) fn writes(&self) -> &[ComputedWrite] {
714        &self.writes
715    }
716
717    pub(crate) fn push_cell(
718        &mut self,
719        sheet_id: SheetId,
720        row0: u32,
721        col0: u32,
722        value: OverlayValue,
723    ) {
724        self.push_cell_with_format(sheet_id, row0, col0, value, None);
725    }
726
727    pub(crate) fn push_cell_with_format(
728        &mut self,
729        sheet_id: SheetId,
730        row0: u32,
731        col0: u32,
732        value: OverlayValue,
733        format_id: Option<crate::format::FormatId>,
734    ) {
735        let format_id = format_id.filter(|id| *id != crate::format::FormatId::GENERAL);
736        self.formats_present |= format_id.is_some();
737        let seq = self.next_sequence();
738        self.estimated_bytes = self
739            .estimated_bytes
740            .saturating_add(Self::estimate_value_bytes(&value));
741        self.writes.push(ComputedWrite::Cell {
742            seq,
743            sheet_id,
744            row0,
745            col0,
746            value,
747            format_id,
748        });
749    }
750
751    pub(crate) fn push_rect(
752        &mut self,
753        sheet_id: SheetId,
754        sr0: u32,
755        sc0: u32,
756        values: Vec<Vec<OverlayValue>>,
757    ) {
758        let seq = self.next_sequence();
759        let added = values
760            .iter()
761            .flat_map(|row| row.iter())
762            .map(Self::estimate_value_bytes)
763            .fold(0usize, usize::saturating_add);
764        self.estimated_bytes = self.estimated_bytes.saturating_add(added);
765        self.writes.push(ComputedWrite::Rect {
766            seq,
767            sheet_id,
768            sr0,
769            sc0,
770            values,
771        });
772    }
773
774    pub(crate) fn clear(&mut self) {
775        self.writes.clear();
776        self.estimated_bytes = 0;
777        self.formats_present = false;
778    }
779
780    fn take_writes(&mut self) -> (Vec<ComputedWrite>, bool) {
781        self.estimated_bytes = 0;
782        let formats_present = std::mem::take(&mut self.formats_present);
783        (std::mem::take(&mut self.writes), formats_present)
784    }
785
786    fn next_sequence(&mut self) -> u64 {
787        let seq = self.next_seq;
788        self.next_seq = self.next_seq.wrapping_add(1);
789        seq
790    }
791
792    #[inline]
793    fn estimate_value_bytes(value: &OverlayValue) -> usize {
794        Self::ENTRY_BASE_BYTES.saturating_add(value.estimated_payload_bytes())
795    }
796}
797
798#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
799struct ComputedWriteChunkKey {
800    sheet_id: SheetId,
801    col0: u32,
802    chunk_idx: usize,
803    chunk_start_row0: u32,
804}
805
806#[derive(Debug, Clone, PartialEq)]
807pub(crate) struct ComputedWriteChunkEntryPlan {
808    pub(crate) row_in_chunk: usize,
809    pub(crate) seq: u64,
810    pub(crate) value: OverlayValue,
811    pub(crate) format_id: Option<crate::format::FormatId>,
812}
813
814#[derive(Debug, Clone, PartialEq, Eq)]
815pub(crate) enum ComputedWriteChunkPlanShape {
816    Point,
817    SparseOffsets {
818        entries: usize,
819        span_len: usize,
820    },
821    DenseRange {
822        start: usize,
823        len: usize,
824    },
825    RunRange {
826        start: usize,
827        len: usize,
828        runs: usize,
829    },
830}
831
832#[derive(Debug, Clone, PartialEq, Eq)]
833pub(crate) enum ComputedWriteFormatClear {
834    Range { start: usize, end: usize },
835    Offsets(Vec<usize>),
836}
837
838#[derive(Debug, Clone, PartialEq)]
839pub(crate) enum ComputedWriteChunkFormatEffect {
840    NoFormatWork,
841    ClearStale(ComputedWriteFormatClear),
842    SetExplicit(Vec<(usize, Option<crate::format::FormatId>)>),
843}
844
845struct DerivedFormatPlaneSyncPlan {
846    plane_epoch: u64,
847    active_regions: Vec<crate::formula_plane::region_index::Region>,
848    overlay_regions: Vec<crate::formula_plane::region_index::Region>,
849}
850
851#[derive(Debug, Clone, PartialEq)]
852pub(crate) struct ComputedWriteChunkPlan {
853    pub(crate) sheet_id: SheetId,
854    pub(crate) col0: u32,
855    pub(crate) chunk_idx: usize,
856    pub(crate) chunk_start_row0: u32,
857    pub(crate) entries: Vec<ComputedWriteChunkEntryPlan>,
858    pub(crate) shape: ComputedWriteChunkPlanShape,
859    pub(crate) format_effect: ComputedWriteChunkFormatEffect,
860}
861
862#[derive(Debug, Clone, Default, PartialEq)]
863pub(crate) struct ComputedWriteCoalescingPlan {
864    pub(crate) chunks: Vec<ComputedWriteChunkPlan>,
865    pub(crate) input_cells: usize,
866    pub(crate) coalesced_cells: usize,
867    pub(crate) overwritten_cells: usize,
868}
869
870impl ComputedWriteCoalescingPlan {
871    #[inline]
872    pub(crate) fn is_empty(&self) -> bool {
873        self.chunks.is_empty()
874    }
875}
876
877impl ComputedWriteChunkPlan {
878    fn from_group(
879        key: ComputedWriteChunkKey,
880        mut entries: Vec<ComputedWriteChunkEntryPlan>,
881        formats_present: bool,
882        computed_lane_has_formats: bool,
883    ) -> (Self, usize) {
884        entries.sort_by_key(|entry| (entry.row_in_chunk, entry.seq));
885        let input_len = entries.len();
886        let mut coalesced: Vec<ComputedWriteChunkEntryPlan> = Vec::with_capacity(input_len);
887        for entry in entries {
888            if let Some(prev) = coalesced.last_mut()
889                && prev.row_in_chunk == entry.row_in_chunk
890            {
891                *prev = entry;
892                continue;
893            }
894            coalesced.push(entry);
895        }
896        let overwritten = input_len.saturating_sub(coalesced.len());
897        let shape = Self::classify_shape(&coalesced);
898        let format_effect = Self::classify_format_effect(
899            &coalesced,
900            &shape,
901            formats_present,
902            computed_lane_has_formats,
903        );
904        (
905            Self {
906                sheet_id: key.sheet_id,
907                col0: key.col0,
908                chunk_idx: key.chunk_idx,
909                chunk_start_row0: key.chunk_start_row0,
910                entries: coalesced,
911                shape,
912                format_effect,
913            },
914            overwritten,
915        )
916    }
917
918    fn classify_format_effect(
919        entries: &[ComputedWriteChunkEntryPlan],
920        shape: &ComputedWriteChunkPlanShape,
921        formats_present: bool,
922        computed_lane_has_formats: bool,
923    ) -> ComputedWriteChunkFormatEffect {
924        if !formats_present {
925            return if computed_lane_has_formats {
926                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
927            } else {
928                ComputedWriteChunkFormatEffect::NoFormatWork
929            };
930        }
931
932        if entries.iter().all(|entry| entry.format_id.is_none()) {
933            return if computed_lane_has_formats {
934                ComputedWriteChunkFormatEffect::ClearStale(Self::format_clear_spec(entries, shape))
935            } else {
936                ComputedWriteChunkFormatEffect::NoFormatWork
937            };
938        }
939
940        ComputedWriteChunkFormatEffect::SetExplicit(
941            entries
942                .iter()
943                .map(|entry| (entry.row_in_chunk, entry.format_id))
944                .collect(),
945        )
946    }
947
948    fn format_clear_spec(
949        entries: &[ComputedWriteChunkEntryPlan],
950        shape: &ComputedWriteChunkPlanShape,
951    ) -> ComputedWriteFormatClear {
952        match shape {
953            ComputedWriteChunkPlanShape::DenseRange { start, len }
954            | ComputedWriteChunkPlanShape::RunRange { start, len, .. } => {
955                ComputedWriteFormatClear::Range {
956                    start: *start,
957                    end: start.saturating_add(*len),
958                }
959            }
960            ComputedWriteChunkPlanShape::Point
961            | ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
962                ComputedWriteFormatClear::Offsets(
963                    entries.iter().map(|entry| entry.row_in_chunk).collect(),
964                )
965            }
966        }
967    }
968
969    fn classify_shape(entries: &[ComputedWriteChunkEntryPlan]) -> ComputedWriteChunkPlanShape {
970        debug_assert!(!entries.is_empty());
971        if entries.len() == 1 {
972            return ComputedWriteChunkPlanShape::Point;
973        }
974
975        let start = entries[0].row_in_chunk;
976        let end = entries[entries.len() - 1].row_in_chunk;
977        let span_len = end.saturating_sub(start).saturating_add(1);
978        if span_len != entries.len() {
979            return ComputedWriteChunkPlanShape::SparseOffsets {
980                entries: entries.len(),
981                span_len,
982            };
983        }
984
985        let runs = Self::run_count(entries);
986        if runs < entries.len() {
987            ComputedWriteChunkPlanShape::RunRange {
988                start,
989                len: entries.len(),
990                runs,
991            }
992        } else {
993            ComputedWriteChunkPlanShape::DenseRange {
994                start,
995                len: entries.len(),
996            }
997        }
998    }
999
1000    fn run_count(entries: &[ComputedWriteChunkEntryPlan]) -> usize {
1001        let mut runs = 0usize;
1002        let mut prev: Option<&OverlayValue> = None;
1003        for entry in entries {
1004            if prev != Some(&entry.value) {
1005                runs = runs.saturating_add(1);
1006                prev = Some(&entry.value);
1007            }
1008        }
1009        runs
1010    }
1011}
1012
1013pub struct Engine<R> {
1014    pub(crate) graph: DependencyGraph,
1015    resolver: R,
1016    pub config: EvalConfig,
1017    workbook_load_limits: crate::engine::WorkbookLoadLimits,
1018    /// Clock for volatile date/time builtins, wrapped in a per-recalc
1019    /// snapshot: sampled once at the start of every evaluation request
1020    /// ([`Self::begin_evaluation_request`]) so all `NOW()`/`TODAY()` reads in
1021    /// one recalc — including SCC iteration passes — agree (spec §7.11).
1022    clock: crate::timezone::SnapshotClock,
1023    thread_pool: Option<Arc<rayon::ThreadPool>>,
1024    pub recalc_epoch: u64,
1025    snapshot_id: std::sync::atomic::AtomicU64,
1026    topology_epoch: u64,
1027    /// False after a structural axis operation. While #171 remains open we
1028    /// must not use relocated span read summaries to prove disconnection.
1029    legacy_island_structural_summaries_trusted: bool,
1030    cached_static_schedule: Option<CachedScheduleEntry>,
1031    #[cfg(any(test, feature = "benchmark_internal"))]
1032    recalc_reuse_probe: std::sync::Mutex<RecalcReuseProbe>,
1033    cached_mixed_topology: Option<CachedMixedTopology>,
1034    mixed_topology_cache_builds: u64,
1035    mixed_topology_cache_hits: u64,
1036    mixed_topology_cache_overflows: u64,
1037    mixed_topology_cache_skip_streak: u64,
1038    spill_mgr: ShimSpillManager,
1039    /// Arrow-backed storage for sheet values (Phase A)
1040    arrow_sheets: SheetStore,
1041    /// Workbook-local number-format registry.
1042    format_registry: crate::format::FormatRegistry,
1043    /// Thread-safe handoff from immutable/parallel evaluation to overlay apply.
1044    derived_format_results: std::sync::RwLock<FxHashMap<VertexId, Option<crate::format::FormatId>>>,
1045    /// Derived formula formats keyed by grid position, never graph vertex identity.
1046    derived_formats: std::sync::RwLock<FxHashMap<CellRef, crate::format::FormatId>>,
1047    /// FormulaPlane epoch whose active result regions have been purged from
1048    /// `derived_formats`.
1049    derived_formats_plane_epoch: u64,
1050    #[cfg(test)]
1051    derived_format_operations_for_test: std::sync::atomic::AtomicU64,
1052    #[cfg(test)]
1053    computed_overlay_set_explicit_entry_operations_for_test: u64,
1054    #[cfg(test)]
1055    computed_overlay_stale_clear_range_effects_for_test: u64,
1056    #[cfg(test)]
1057    computed_overlay_stale_clear_offset_attempts_for_test: u64,
1058    #[cfg(test)]
1059    computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64,
1060    /// True if any edit after bulk load; disables Arrow reads for parity
1061    has_edited: bool,
1062    /// Overlay compaction counter (Phase C instrumentation)
1063    overlay_compactions: u64,
1064
1065    // Overlay memory observability / budget (ticket 503)
1066    computed_overlay_bytes_estimate: usize,
1067    computed_overlay_mirroring_disabled: bool,
1068    /// When true, RangeView resolution materializes from graph/Arrow base per-cell.
1069    /// This preserves correctness if we stop mirroring formula/spill outputs into computed overlays.
1070    pub(crate) force_materialize_range_views: bool,
1071    // Pass-scoped cache for Arrow used-row bounds per column
1072    row_bounds_cache: std::sync::RwLock<Option<RowBoundsCache>>,
1073    // Snapshot-scoped final used-axis bounds for open-ended references.
1074    used_axis_bounds_cache: std::sync::RwLock<Option<UsedAxisBoundsCache>>,
1075    lookup_index_cache: LookupIndexCache,
1076    source_cache: Arc<std::sync::RwLock<SourceCache>>,
1077    /// Identity binding for opaque source-family preparations.
1078    source_formula_token: Arc<()>,
1079    /// Dedicated identity binding for reusable recalculation plans.
1080    recalc_plan_token: Arc<()>,
1081    /// Staged formulas by sheet when `defer_graph_building` is enabled.
1082    staged_formulas: StagedFormulaMap,
1083    /// Presence and generation authority for ordinary staged formula discovery.
1084    staged_formula_index: StagedFormulaIndex,
1085    /// Per-sheet row visibility sidecar state.
1086    row_visibility: FxHashMap<SheetId, RowVisibilityState>,
1087    /// Cached row visibility masks keyed by sheet/span/mode/version.
1088    row_visibility_mask_cache: std::sync::RwLock<
1089        FxHashMap<VisibilityMaskCacheKey, std::sync::Arc<arrow_array::BooleanArray>>,
1090    >,
1091    /// Non-fatal malformed formula diagnostics captured during ingest/graph-build.
1092    formula_parse_diagnostics: Vec<FormulaParseDiagnostic>,
1093    /// Last centralized formula ingest report.
1094    last_formula_ingest_report: Option<FormulaIngestReport>,
1095    /// Aggregate centralized formula ingest report for this engine.
1096    formula_ingest_report_total: FormulaIngestReport,
1097    /// Count of FormulaPlane spans demoted to legacy because one or more of
1098    /// their member cells participate in a statically-cyclic SCC. A span member
1099    /// must never be span-evaluated (gotcha G8 of the cycle-architecture track,
1100    /// refs #112): under `CycleDetection::Static` the cycle stamping would race
1101    /// span writes, and under `Runtime` SCC members must be evaluated by the
1102    /// legacy `evaluate_scc_unit` path. Cyclic spans are demoted at
1103    /// schedule-build time (the earliest point cross-cell cycles through span
1104    /// producers become visible) so the cycle members land on the legacy graph
1105    /// path. Observational only.
1106    formula_plane_cycle_member_span_demotions: u64,
1107    /// Spans demoted to legacy because a placement evaluated to an array
1108    /// result (refs #388). Spans publish exactly one value per placement, so an
1109    /// array must be re-evaluated on the legacy path where the spill planner
1110    /// owns it. Observational only.
1111    formula_plane_array_result_span_demotions: u64,
1112    /// Successfully completed non-cycle unsafe mixed requests. Incremented
1113    /// only after every scheduled span has committed demotion and the single
1114    /// legacy completion pass succeeds; failed attempts are not counted.
1115    /// Observational only.
1116    formula_plane_capacity_bailouts: u64,
1117    /// Exact span candidates classified across structural mutations.
1118    formula_plane_structural_span_candidates: u64,
1119    /// Transient cancellation flag used during evaluation
1120    active_cancel_flag: Option<crate::engine::CancelToken>,
1121    /// Transient absolute deadline used by composed target and plan requests.
1122    active_evaluation_deadline: Option<Instant>,
1123
1124    /// Engine-level action depth.
1125    ///
1126    /// Ticket 614 introduces `Engine::action` as a stable, commit-only transaction surface.
1127    /// Nested actions are currently disallowed (deterministic rule) and will return an error.
1128    action_depth: u32,
1129
1130    // Phase 3b virtual-dependency convergence telemetry
1131    last_virtual_dep_telemetry: VirtualDepTelemetry,
1132    virtual_dep_fallback_activations: u64,
1133
1134    // Runtime-cycle SCC evaluation telemetry (RFC #112, Stage 2)
1135    last_cycle_telemetry: CycleTelemetry,
1136
1137    // C0 evaluation-resource observability. IDs are never reset or reused.
1138    next_evaluation_resource_request_id: u64,
1139    evaluation_resource_request_depth: usize,
1140    active_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1141    last_evaluation_resource_request: Option<EvaluationResourceRequestStats>,
1142    evaluation_resource_baseline: EvaluationResourceBaselineStats,
1143    evaluation_resource_request_started_at: Option<crate::instant::FzInstant>,
1144    evaluation_resource_budgets: crate::engine::EvaluationBudgets,
1145    evaluation_resource_config_diagnostic:
1146        Option<crate::engine::EvaluationResourceConfigDiagnostic>,
1147    active_resource_ledger: Option<ResourceLedger>,
1148
1149    /// SCC members that entered iterative calculation (`CyclePolicy::Iterate`
1150    /// with a witnessed live cycle) during the current evaluation request
1151    /// and must re-run on the next one.
1152    ///
1153    /// Excel re-evaluates circular cells on EVERY recalc (the accumulator
1154    /// contract, spec §4/§7.6), but this engine's dirty model marks SCC
1155    /// members clean after a recalc and would otherwise skip them forever.
1156    /// Resolution: members of iterating SCCs are redirtied volatile-like at
1157    /// the end of the same recalc that iterated them
1158    /// ([`Self::redirty_for_next_recalc`], called wherever
1159    /// `redirty_volatiles` runs). The set is per-recalc, never persisted:
1160    /// if an edit breaks the cycle, the next recalc's SCC task either does
1161    /// not exist or settles as phantom, nothing re-registers, and the
1162    /// redirty chain stops by itself.
1163    ///
1164    /// SCCs that landed on an exact fixed point are exempt: they go to
1165    /// [`Self::retained_scc_members`] instead and stay clean until the dirty
1166    /// graph (or a config change) reaches them (#368).
1167    pending_iterative_redirty: Vec<VertexId>,
1168    /// Members of iterating SCCs retained across recalcs (#368), keyed to the
1169    /// id of the retained SCC they belong to (ids come from
1170    /// `next_retained_scc_id`; grouping is only used for telemetry).
1171    ///
1172    /// An SCC is retained when the recalc that iterated it stopped because
1173    /// every member reproduced its previous value exactly (|Δ| = 0, or
1174    /// identity for non-numeric members; never NaN-converged), before the
1175    /// `max_iterations` cap, with no volatile or dynamic-reference member.
1176    /// Such an SCC is a fixed point of its own inputs: running it again with
1177    /// the same inputs cannot change any value, so it is not redirtied. The
1178    /// dirty graph remains the validity authority — any edit that reaches a
1179    /// member dirties it like any other formula and the SCC task re-runs.
1180    /// Membership is dropped when a member runs in an SCC task again (it is
1181    /// then re-retained or re-registered for per-recalc redirty), when the
1182    /// vertex is deleted, or when
1183    /// [`Self::reconcile_retained_sccs_at_request_begin`] invalidates the
1184    /// whole set because a config knob outside the graph changed.
1185    retained_scc_members: FxHashMap<VertexId, u64>,
1186    next_retained_scc_id: u64,
1187    /// [`Self::retained_scc_config_fingerprint`] as of the last retention.
1188    /// `Engine::config` is a public field, so knobs that change a retained
1189    /// SCC's result (cycle policy/tolerance, date system, determinism,
1190    /// volatile seeding) can change between recalcs without touching the
1191    /// graph; a mismatch at request begin dirties every retained member.
1192    /// Only meaningful while `retained_scc_members` is non-empty.
1193    retained_scc_config_fingerprint: u64,
1194    /// Function-registry semantic epoch and runtime-provider revision as of
1195    /// the last time retained SCCs were reconciled against them. A newer
1196    /// epoch dirties only the retained members whose formula calls a changed
1197    /// function (or every member when the change log is incomplete), the
1198    /// same rule FormulaPlane spans use in
1199    /// [`Self::observe_function_semantic_epoch`].
1200    retained_scc_function_epoch_seen: u64,
1201    retained_scc_provider_revision_seen: Option<u64>,
1202    /// Retained members that were already dirty when the current request
1203    /// began, with the SCC id they carried. A member still holding that id
1204    /// at the end of the request was not touched by any SCC task — its
1205    /// cycle dissolved (it evaluated as an ordinary formula) or the request
1206    /// never reached it — so [`Self::redirty_for_next_recalc`] drops it from
1207    /// the retained set instead of letting it linger.
1208    retained_scc_dirty_at_begin: Vec<(VertexId, u64)>,
1209
1210    /// Final committed values of iterating-SCC members (spec §4 persistence).
1211    /// In canonical (value-cache disabled) mode the computed overlay is the
1212    /// ONLY home of a formula's value, and structural edits clear computed
1213    /// overlays wholesale (`clear_computed_overlay_after_row/_col`) —
1214    /// destroying iteration state (accumulators reset to 0; found by the
1215    /// iterate edge corpus). This snapshot lets the next SCC task re-seed
1216    /// members whose overlay entry vanished. Members registered for
1217    /// per-recalc redirty are refreshed by
1218    /// [`Self::redirty_for_next_recalc`]; retained members are written once
1219    /// when retained. Entries are dropped when the member's SCC task ends
1220    /// without iterating or when the vertex is deleted. Empty unless
1221    /// something iterated — zero cost otherwise.
1222    iterative_state_values: FxHashMap<VertexId, LiteralValue>,
1223
1224    /// Global function-registry semantic epoch observed after the latest
1225    /// conservative FormulaPlane invalidation.
1226    function_semantic_epoch_seen: u64,
1227    /// Runtime-provider semantic revision observed after the latest conservative
1228    /// FormulaPlane invalidation.
1229    function_provider_revision_seen: Option<u64>,
1230
1231    #[cfg(test)]
1232    last_formula_plane_span_eval_report: Option<SpanEvalReport>,
1233    #[cfg(test)]
1234    evaluation_request_begin_count_for_test: u64,
1235    #[cfg(any(test, feature = "test-support"))]
1236    before_prepared_span_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1237    #[cfg(test)]
1238    before_target_preparation_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1239    #[cfg(test)]
1240    before_target_planning_snapshot_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1241    #[cfg(test)]
1242    inject_target_semantic_stale_once_for_test: bool,
1243    #[cfg(test)]
1244    force_virtual_dep_changes_remaining_for_test: usize,
1245    #[cfg(test)]
1246    fail_evaluation_commit_preflight_once_for_test: bool,
1247    #[cfg(test)]
1248    cancel_before_formula_plane_layer_commit_once_for_test: bool,
1249    #[cfg(test)]
1250    force_source_family_fallback: bool,
1251    #[cfg(test)]
1252    rerecord_cycle_retry_span_after_lease_extension_for_test: bool,
1253    #[cfg(test)]
1254    fragmented_commit_fault_for_test:
1255        Option<crate::engine::fragmented_transaction::FragmentedCommitFault>,
1256    #[cfg(test)]
1257    formula_span_demotion_fault_for_test: Option<FormulaSpanDemotionFault>,
1258    #[cfg(test)]
1259    target_preparation_fault_for_test:
1260        Option<crate::engine::target_preparation::TargetPreparationFault>,
1261    #[cfg(test)]
1262    force_non_cycle_schedule_fallback_for_test: bool,
1263    #[cfg(test)]
1264    mixed_topology_index_builds_for_test: u64,
1265    #[cfg(test)]
1266    before_legacy_fallback_final_provider_sample_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1267    #[cfg(test)]
1268    after_eager_proposal_commit_hook: Option<Box<dyn FnOnce() + Send + Sync>>,
1269}
1270
1271/// Minimal edit surface used by `Engine::action`.
1272///
1273fn substitute_formula_plane_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
1274    fn visit(ast: &ASTNode, binding: &[LiteralValue], next: &mut usize, in_array: bool) -> ASTNode {
1275        let node_type = match &ast.node_type {
1276            ASTNodeType::Literal(_) if !in_array => {
1277                let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
1278                *next = next.saturating_add(1);
1279                ASTNodeType::Literal(value)
1280            }
1281            ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
1282            ASTNodeType::Omitted => ASTNodeType::Omitted,
1283            ASTNodeType::Reference {
1284                original,
1285                reference,
1286            } => ASTNodeType::Reference {
1287                original: original.clone(),
1288                reference: reference.clone(),
1289            },
1290            ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
1291                op: op.clone(),
1292                expr: Box::new(visit(expr, binding, next, in_array)),
1293            },
1294            ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
1295                op: op.clone(),
1296                left: Box::new(visit(left, binding, next, in_array)),
1297                right: Box::new(visit(right, binding, next, in_array)),
1298            },
1299            ASTNodeType::Function { name, args } => ASTNodeType::Function {
1300                name: name.clone(),
1301                args: args
1302                    .iter()
1303                    .map(|arg| visit(arg, binding, next, in_array))
1304                    .collect(),
1305            },
1306            ASTNodeType::Call { callee, args } => ASTNodeType::Call {
1307                callee: Box::new(visit(callee, binding, next, in_array)),
1308                args: args
1309                    .iter()
1310                    .map(|arg| visit(arg, binding, next, in_array))
1311                    .collect(),
1312            },
1313            ASTNodeType::Array(rows) => ASTNodeType::Array(
1314                rows.iter()
1315                    .map(|row| {
1316                        row.iter()
1317                            .map(|cell| visit(cell, binding, next, true))
1318                            .collect()
1319                    })
1320                    .collect(),
1321            ),
1322        };
1323        ASTNode::new(node_type, ast.source_token.clone())
1324    }
1325
1326    let mut next = 0;
1327    visit(ast, binding, &mut next, false)
1328}
1329
1330/// This wrapper is intentionally thin for ticket 614 (commit-only): it delegates to existing
1331/// `Engine` edit methods and does not create changelog boundaries or implement rollback.
1332impl<R: EvaluationContext> Engine<R> {
1333    pub(crate) fn ingest_pipeline(&mut self) -> crate::engine::ingest_pipeline::IngestPipeline<'_> {
1334        self.graph.ingest_pipeline(&self.resolver)
1335    }
1336}
1337
1338pub struct EngineAction<'a, R>
1339where
1340    R: EvaluationContext,
1341{
1342    engine: &'a mut Engine<R>,
1343    name: String,
1344    // Complete private mutation capture used by atomic actions.
1345    // Stored as a raw pointer to avoid creating aliasing `&mut` borrows alongside `&mut Engine`.
1346    capture: Option<*mut MutationCapture>,
1347    // Optional Arrow undo journal used by `Engine::action_atomic`.
1348    // Stored as a raw pointer to avoid aliasing issues with `&mut Engine`.
1349    arrow_undo: Option<*mut crate::engine::ArrowUndoBatch>,
1350    // True when this EngineAction must enforce conservative atomic transaction policy.
1351    atomic_policy: bool,
1352}
1353
1354impl<'a, R> EngineAction<'a, R>
1355where
1356    R: EvaluationContext,
1357{
1358    #[inline]
1359    fn addr_for(&mut self, sheet: &str, row: u32, col: u32) -> crate::reference::CellRef {
1360        let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1361        let coord = crate::reference::Coord::from_excel(row, col, true, true);
1362        crate::reference::CellRef::new(sheet_id, coord)
1363    }
1364
1365    #[inline]
1366    pub fn name(&self) -> &str {
1367        &self.name
1368    }
1369
1370    #[inline]
1371    pub fn set_cell_value(
1372        &mut self,
1373        sheet: &str,
1374        row: u32,
1375        col: u32,
1376        value: LiteralValue,
1377    ) -> Result<(), crate::engine::EditorError> {
1378        if self.capture.is_some() {
1379            let old_value = self.engine.read_cell_value(sheet, row, col);
1380            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1381            let addr = self.addr_for(sheet, row, col);
1382            let Some(capture_ptr) = self.capture else {
1383                return Err(crate::engine::EditorError::TransactionFailed {
1384                    reason: "action_with_logger: missing mutation capture".to_string(),
1385                });
1386            };
1387
1388            // For atomic journal mode, record computed overlay effects for this cell.
1389            // Delta-overlay undo is recorded semantically based on old_value/old_formula.
1390            let old_comp = if self.arrow_undo.is_some() {
1391                self.engine.read_computed_overlay_cell(sheet, row, col)
1392            } else {
1393                None
1394            };
1395
1396            if self.engine.graph_admission_enabled() {
1397                let admission =
1398                    self.engine
1399                        .graph
1400                        .preview_value_mutation(addr.sheet_id, row, col)?;
1401                self.engine.preflight_graph_admission(admission)?;
1402            }
1403            self.engine.demote_span_containing_cell_for_write(
1404                addr.sheet_id,
1405                addr.coord.row(),
1406                addr.coord.col(),
1407            )?;
1408            if old_formula.is_none() {
1409                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1410            }
1411
1412            let delta_old_sem = if old_formula.is_some() {
1413                None
1414            } else {
1415                Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1416            };
1417
1418            let start_len = unsafe { (&*capture_ptr).len() };
1419
1420            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1421            let capture = unsafe { &mut *capture_ptr };
1422            self.engine.edit_with_capture(capture, |editor| {
1423                editor.set_cell_value_with_old_state(
1424                    addr,
1425                    value.clone(),
1426                    old_value.clone(),
1427                    old_formula.clone(),
1428                );
1429            })?;
1430            self.engine
1431                .record_formula_plane_structural_change(StructuralScope::Cell {
1432                    sheet: addr.sheet_id,
1433                    row: addr.coord.row(),
1434                    col: addr.coord.col(),
1435                });
1436
1437            if let Some(undo_ptr) = self.arrow_undo {
1438                // 1) Spill snapshot operations (computed overlay rect restore).
1439                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1440                let undo = unsafe { &mut *undo_ptr };
1441                self.engine
1442                    .record_spill_ops_into_arrow_undo(undo, new_events);
1443
1444                // 2) Delta/computed overlay single-cell deltas.
1445                let new_comp = self.engine.read_computed_overlay_cell(sheet, row, col);
1446                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1447                let row0 = row.saturating_sub(1);
1448                let col0 = col.saturating_sub(1);
1449                let delta_new_sem = Some(value.clone());
1450                undo.record_delta_cell(sheet_id, row0, col0, delta_old_sem, delta_new_sem);
1451                undo.record_computed_cell(sheet_id, row0, col0, old_comp, new_comp);
1452            }
1453            Ok(())
1454        } else {
1455            self.engine
1456                .set_cell_value(sheet, row, col, value)
1457                .map_err(crate::engine::EditorError::from)
1458        }
1459    }
1460
1461    #[inline]
1462    pub fn set_cell_formula(
1463        &mut self,
1464        sheet: &str,
1465        row: u32,
1466        col: u32,
1467        ast: ASTNode,
1468    ) -> Result<(), crate::engine::EditorError> {
1469        if self.capture.is_some() {
1470            let old_value = self.engine.read_cell_value(sheet, row, col);
1471            let mut old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1472            let addr = self.addr_for(sheet, row, col);
1473            let Some(capture_ptr) = self.capture else {
1474                return Err(crate::engine::EditorError::TransactionFailed {
1475                    reason: "action_with_logger: missing mutation capture".to_string(),
1476                });
1477            };
1478
1479            let admitted_formula = if self.engine.graph_admission_enabled() {
1480                let placement =
1481                    CellRef::new(addr.sheet_id, Coord::from_excel(row, col, true, true));
1482                let ingested = self.engine.ingest_pipeline().ingest_formula(
1483                    FormulaAstInput::Tree(ast.clone()),
1484                    placement,
1485                    None,
1486                )?;
1487                let admission = self.engine.graph.preview_formula_mutations(&[(
1488                    addr.sheet_id,
1489                    row,
1490                    col,
1491                    ingested.dep_plan.clone(),
1492                )])?;
1493                self.engine.preflight_graph_admission(admission)?;
1494                Some((ingested.ast_id, ingested.dep_plan))
1495            } else {
1496                None
1497            };
1498            self.engine.demote_span_containing_cell_for_write(
1499                addr.sheet_id,
1500                addr.coord.row(),
1501                addr.coord.col(),
1502            )?;
1503            if old_formula.is_none() {
1504                old_formula = self.engine.read_cell_formula_ast(sheet, row, col);
1505            }
1506            let delta_old = if self.arrow_undo.is_some() {
1507                if old_formula.is_some() {
1508                    None
1509                } else {
1510                    Some(old_value.clone().unwrap_or(LiteralValue::Empty))
1511                }
1512            } else {
1513                None
1514            };
1515            let start_len = unsafe { (&*capture_ptr).len() };
1516
1517            // Safety: `capture_ptr` comes from a unique operation-local `&mut MutationCapture`.
1518            let capture = unsafe { &mut *capture_ptr };
1519            self.engine.edit_with_capture(capture, |editor| {
1520                if let Some((ast_id, plan)) = admitted_formula {
1521                    editor.set_cell_formula_with_prepared_plan(
1522                        addr,
1523                        ast.clone(),
1524                        old_value,
1525                        old_formula,
1526                        ast_id,
1527                        plan,
1528                    );
1529                } else {
1530                    editor.set_cell_formula_with_old_state(
1531                        addr,
1532                        ast.clone(),
1533                        old_value,
1534                        old_formula,
1535                    );
1536                }
1537            })?;
1538            self.engine
1539                .record_formula_plane_structural_change(StructuralScope::Cell {
1540                    sheet: addr.sheet_id,
1541                    row: addr.coord.row(),
1542                    col: addr.coord.col(),
1543                });
1544
1545            if let Some(undo_ptr) = self.arrow_undo {
1546                let new_events = &unsafe { (&*capture_ptr).events() }[start_len..];
1547                let undo = unsafe { &mut *undo_ptr };
1548                self.engine
1549                    .record_spill_ops_into_arrow_undo(undo, new_events);
1550                let delta_new: Option<LiteralValue> = None;
1551                let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1552                let row0 = row.saturating_sub(1);
1553                let col0 = col.saturating_sub(1);
1554                undo.record_delta_cell(sheet_id, row0, col0, delta_old, delta_new);
1555            }
1556            Ok(())
1557        } else {
1558            self.engine
1559                .set_cell_formula(sheet, row, col, ast)
1560                .map_err(crate::engine::EditorError::from)
1561        }
1562    }
1563
1564    #[inline]
1565    pub fn set_row_hidden(
1566        &mut self,
1567        sheet: &str,
1568        row_1based: u32,
1569        hidden: bool,
1570        source: RowVisibilitySource,
1571    ) -> Result<(), crate::engine::EditorError> {
1572        if self.capture.is_some() {
1573            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1574            let row0 = Engine::<R>::normalize_row_1based(row_1based)?;
1575            let old_hidden = self
1576                .engine
1577                .row_visibility
1578                .get(&sheet_id)
1579                .map(|state| state.is_row_hidden(row0, Some(source)))
1580                .unwrap_or(false);
1581            if old_hidden == hidden {
1582                return Ok(());
1583            }
1584
1585            let _ = self
1586                .engine
1587                .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1588
1589            let Some(capture_ptr) = self.capture else {
1590                return Err(crate::engine::EditorError::TransactionFailed {
1591                    reason: "action_with_logger: missing mutation capture".to_string(),
1592                });
1593            };
1594            unsafe { &mut *capture_ptr }.record(crate::engine::ChangeEvent::SetRowVisibility {
1595                sheet_id,
1596                row0,
1597                source,
1598                old_hidden,
1599                new_hidden: hidden,
1600            });
1601
1602            Ok(())
1603        } else {
1604            self.engine
1605                .set_row_hidden(sheet, row_1based, hidden, source)
1606        }
1607    }
1608
1609    #[inline]
1610    pub fn set_rows_hidden(
1611        &mut self,
1612        sheet: &str,
1613        start_row_1based: u32,
1614        end_row_1based: u32,
1615        hidden: bool,
1616        source: RowVisibilitySource,
1617    ) -> Result<(), crate::engine::EditorError> {
1618        if self.capture.is_some() {
1619            let sheet_id = self.engine.ensure_known_sheet_id(sheet)?;
1620            let (start_row0, end_row0) =
1621                Engine::<R>::normalize_row_range_1based(start_row_1based, end_row_1based)?;
1622
1623            let Some(capture_ptr) = self.capture else {
1624                return Err(crate::engine::EditorError::TransactionFailed {
1625                    reason: "action_with_logger: missing mutation capture".to_string(),
1626                });
1627            };
1628            let capture = unsafe { &mut *capture_ptr };
1629
1630            for row0 in start_row0..=end_row0 {
1631                let old_hidden = self
1632                    .engine
1633                    .row_visibility
1634                    .get(&sheet_id)
1635                    .map(|state| state.is_row_hidden(row0, Some(source)))
1636                    .unwrap_or(false);
1637                if old_hidden == hidden {
1638                    continue;
1639                }
1640
1641                let _ = self
1642                    .engine
1643                    .set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source);
1644
1645                capture.record(crate::engine::ChangeEvent::SetRowVisibility {
1646                    sheet_id,
1647                    row0,
1648                    source,
1649                    old_hidden,
1650                    new_hidden: hidden,
1651                });
1652            }
1653
1654            Ok(())
1655        } else {
1656            self.engine
1657                .set_rows_hidden(sheet, start_row_1based, end_row_1based, hidden, source)
1658        }
1659    }
1660
1661    #[inline]
1662    pub fn insert_rows(
1663        &mut self,
1664        sheet: &str,
1665        before: u32,
1666        count: u32,
1667    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1668        if count == 0 {
1669            return Ok(crate::engine::ShiftSummary::default());
1670        }
1671        if self.capture.is_some() {
1672            let Some(capture_ptr) = self.capture else {
1673                return Err(crate::engine::EditorError::TransactionFailed {
1674                    reason: "action_atomic: missing mutation capture".to_string(),
1675                });
1676            };
1677
1678            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1679            let before0 = before.saturating_sub(1);
1680            let occupancy = self.engine.structural_row_occupancy(sheet, sheet_id);
1681            let affected_region = Engine::<R>::structural_row_region(sheet_id, before0);
1682            // Authority geometry is not journaled. Materialize affected spans
1683            // before the logged graph shift so undo/redo remains exact instead
1684            // of leaving an unlogged shifted span behind.
1685            self.engine
1686                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1687
1688            // Graph structural insert (logged) - no snapshot bump.
1689            let summary = {
1690                let capture = unsafe { &mut *capture_ptr };
1691                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1692                    Ok(crate::engine::ShiftSummary::default());
1693                self.engine.edit_with_capture(capture, |editor| {
1694                    editor.set_structural_occupancy(occupancy);
1695                    out = editor.insert_rows(sheet_id, before0, count);
1696                })?;
1697                out?
1698            };
1699
1700            // Arrow insert (truth) + undo op.
1701            self.engine.ensure_arrow_sheet(sheet);
1702            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1703                asheet.insert_rows(before0 as usize, count as usize);
1704            }
1705            self.engine
1706                .purge_derived_formats_after_row(sheet_id, before0);
1707            self.engine
1708                .shift_row_visibility_insert(sheet_id, before0, count);
1709            self.engine.mark_moved_formula_vertices_dirty(&summary);
1710            self.engine
1711                .clear_computed_overlay_after_row(sheet, before0 as usize);
1712            self.engine
1713                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1714            if let Some(undo_ptr) = self.arrow_undo {
1715                unsafe { &mut *undo_ptr }.record_insert_rows(sheet_id, before0, count);
1716            }
1717            Ok(summary)
1718        } else {
1719            self.engine.insert_rows(sheet, before, count)
1720        }
1721    }
1722
1723    #[inline]
1724    pub fn delete_rows(
1725        &mut self,
1726        sheet: &str,
1727        start: u32,
1728        count: u32,
1729    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1730        if count == 0 {
1731            return Ok(crate::engine::ShiftSummary::default());
1732        }
1733        if self.atomic_policy {
1734            return Err(crate::engine::EditorError::TransactionUnsupported {
1735                reason:
1736                    "delete_rows is not supported inside atomic actions (conservative rollback policy)"
1737                        .to_string(),
1738            });
1739        }
1740        self.engine.delete_rows(sheet, start, count)
1741    }
1742
1743    #[inline]
1744    pub fn insert_columns(
1745        &mut self,
1746        sheet: &str,
1747        before: u32,
1748        count: u32,
1749    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1750        if count == 0 {
1751            return Ok(crate::engine::ShiftSummary::default());
1752        }
1753        if self.capture.is_some() {
1754            let Some(capture_ptr) = self.capture else {
1755                return Err(crate::engine::EditorError::TransactionFailed {
1756                    reason: "action_atomic: missing mutation capture".to_string(),
1757                });
1758            };
1759
1760            let sheet_id = self.engine.graph.sheet_id_mut(sheet);
1761            let before0 = before.saturating_sub(1);
1762            let occupancy = self.engine.structural_column_occupancy();
1763            let affected_region = Engine::<R>::structural_col_region(sheet_id, before0);
1764            self.engine
1765                .demote_spans_preserving_computed_overlays(sheet_id, affected_region)?;
1766
1767            let summary = {
1768                let capture = unsafe { &mut *capture_ptr };
1769                let mut out: Result<crate::engine::ShiftSummary, crate::engine::EditorError> =
1770                    Ok(crate::engine::ShiftSummary::default());
1771                self.engine.edit_with_capture(capture, |editor| {
1772                    editor.set_structural_occupancy(occupancy);
1773                    out = editor.insert_columns(sheet_id, before0, count);
1774                })?;
1775                out?
1776            };
1777
1778            self.engine.ensure_arrow_sheet(sheet);
1779            if let Some(asheet) = self.engine.arrow_sheets.sheet_mut(sheet) {
1780                asheet.insert_columns(before0 as usize, count as usize);
1781            }
1782            self.engine
1783                .purge_derived_formats_after_col(sheet_id, before0);
1784            self.engine.mark_moved_formula_vertices_dirty(&summary);
1785            self.engine
1786                .clear_computed_overlay_after_col(sheet, before0 as usize);
1787            self.engine
1788                .record_formula_plane_structural_change(StructuralScope::Region(affected_region));
1789            if let Some(undo_ptr) = self.arrow_undo {
1790                unsafe { &mut *undo_ptr }.record_insert_cols(sheet_id, before0, count);
1791            }
1792            Ok(summary)
1793        } else {
1794            self.engine.insert_columns(sheet, before, count)
1795        }
1796    }
1797
1798    #[inline]
1799    pub fn delete_columns(
1800        &mut self,
1801        sheet: &str,
1802        start: u32,
1803        count: u32,
1804    ) -> Result<crate::engine::ShiftSummary, crate::engine::EditorError> {
1805        if count == 0 {
1806            return Ok(crate::engine::ShiftSummary::default());
1807        }
1808        if self.atomic_policy {
1809            return Err(crate::engine::EditorError::TransactionUnsupported {
1810                reason:
1811                    "delete_columns is not supported inside atomic actions (conservative rollback policy)"
1812                        .to_string(),
1813            });
1814        }
1815        self.engine.delete_columns(sheet, start, count)
1816    }
1817
1818    /// Start an action from within an action.
1819    ///
1820    /// Nested actions are currently disallowed (ticket 614), so this will return a
1821    /// `EditorError::TransactionFailed` while an outer action is active.
1822    #[inline]
1823    pub fn action<T>(
1824        &mut self,
1825        name: impl AsRef<str>,
1826        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
1827    ) -> Result<T, crate::engine::EditorError> {
1828        self.engine.action(name, f)
1829    }
1830}
1831
1832struct ActionDepthGuard<'a, R> {
1833    engine: *mut Engine<R>,
1834    _marker: std::marker::PhantomData<&'a mut Engine<R>>,
1835}
1836
1837impl<'a, R> Drop for ActionDepthGuard<'a, R> {
1838    fn drop(&mut self) {
1839        // Safety: the guard is created from a unique `&mut Engine` borrow and lives no longer
1840        // than the surrounding `Engine::action` call.
1841        unsafe {
1842            let e = &mut *self.engine;
1843            e.action_depth = e.action_depth.saturating_sub(1);
1844        }
1845    }
1846}
1847
1848#[derive(Default)]
1849struct SourceCache {
1850    scalars: FxHashMap<(String, Option<u64>), LiteralValue>,
1851    tables: FxHashMap<(String, Option<u64>), Arc<dyn crate::traits::Table>>,
1852}
1853
1854#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
1855struct VisibilityMaskCacheKey {
1856    sheet_id: SheetId,
1857    start_row0: u32,
1858    end_row0: u32,
1859    mode: VisibilityMaskMode,
1860    version: u64,
1861}
1862
1863#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1864enum StructuralScope {
1865    Cell { sheet: SheetId, row: u32, col: u32 },
1866    Region(Region),
1867    Sheet(SheetId),
1868    RemovedSheet(SheetId),
1869    OpaqueGlobal,
1870    AllSheets,
1871}
1872
1873#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
1874enum LoggedEditImpact {
1875    NoOp,
1876    DataOnly,
1877    Topology,
1878}
1879
1880#[derive(Clone, Copy, Debug, PartialEq, Eq)]
1881enum LoggedEditDirection {
1882    Original,
1883    InverseReplay,
1884    ForwardReplay,
1885}
1886
1887#[derive(Clone, Copy)]
1888struct InvalidationBaseline {
1889    snapshot_id: u64,
1890    topology_epoch: u64,
1891}
1892
1893struct SourceCacheSession {
1894    cache: Arc<std::sync::RwLock<SourceCache>>,
1895}
1896
1897impl Drop for SourceCacheSession {
1898    fn drop(&mut self) {
1899        if let Ok(mut g) = self.cache.write() {
1900            *g = SourceCache::default();
1901        }
1902    }
1903}
1904
1905#[derive(Debug)]
1906#[non_exhaustive]
1907pub struct EvalResult {
1908    pub computed_vertices: usize,
1909    pub cycle_errors: usize,
1910    pub elapsed: std::time::Duration,
1911}
1912
1913#[derive(Clone, Debug, PartialEq, Eq)]
1914#[non_exhaustive]
1915pub struct TableMetadata {
1916    pub name: String,
1917    pub sheet: String,
1918    pub start_row: u32,
1919    pub start_col: u32,
1920    pub end_row: u32,
1921    pub end_col: u32,
1922    pub header_row: bool,
1923    pub headers: Vec<String>,
1924    pub totals_row: bool,
1925}
1926
1927/// Read-only engine counters used by benchmark/instrumentation tooling.
1928///
1929/// These counters are deliberately observational: collecting them must not mutate engine state or
1930/// alter formula evaluation semantics.
1931#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
1932#[non_exhaustive]
1933pub struct EngineBaselineStats {
1934    pub graph_vertex_count: usize,
1935    pub graph_formula_vertex_count: usize,
1936    pub graph_edge_count: usize,
1937    pub dirty_vertex_count: usize,
1938    pub evaluation_vertex_count: usize,
1939    pub formula_ast_root_count: usize,
1940    pub formula_ast_node_count: usize,
1941    pub staged_formula_count: usize,
1942    pub formula_plane_active_span_count: usize,
1943    pub formula_plane_producer_result_entries: usize,
1944    pub formula_plane_consumer_read_entries: usize,
1945    pub formula_plane_mixed_topology_cache_builds: u64,
1946    pub formula_plane_mixed_topology_cache_hits: u64,
1947    pub formula_plane_mixed_topology_cache_overflows: u64,
1948    pub formula_plane_dirty_pending_events: usize,
1949    pub formula_plane_dirty_region_events_recorded: u64,
1950    pub formula_plane_dirty_span_region_events_recorded: u64,
1951    pub formula_plane_dirty_whole_span_seeds_recorded: u64,
1952    pub formula_plane_dirty_global_invalidations: u64,
1953    pub formula_plane_structural_span_candidates: u64,
1954    /// Number of spans demoted to legacy because a member participated in a
1955    /// statically-cyclic SCC (gotcha G8, refs #112).
1956    pub formula_plane_cycle_member_span_demotions: u64,
1957    /// Number of spans demoted to legacy because span evaluation produced an
1958    /// array result that legacy would route through the spill planner
1959    /// (refs #388).
1960    pub formula_plane_array_result_span_demotions: u64,
1961    /// Members of exactly converged iterative SCCs currently retained across
1962    /// recalcs (#368).
1963    pub retained_scc_members: usize,
1964}
1965
1966#[derive(Debug, Clone, Default)]
1967#[non_exhaustive]
1968pub struct VirtualDepTelemetry {
1969    pub candidate_vertices_total: usize,
1970    pub vdeps_vertices_total: usize,
1971    pub vdeps_edges_total: usize,
1972    pub builder_elapsed_ms_total: u128,
1973    pub schedule_virtual_passes: usize,
1974    pub schedule_static_passes: usize,
1975    pub schedule_cache_hits: usize,
1976    pub schedule_cache_misses: usize,
1977    pub reused_schedule_vertices_total: usize,
1978    pub replan_iterations: usize,
1979    pub changed_vdeps_total: usize,
1980    pub bailout_reason: Option<&'static str>,
1981    pub fallback_mode_activations: u64,
1982}
1983
1984/// Per-recalc telemetry for SCC evaluation under `CycleDetection::Runtime`
1985/// (spec `formualizer-cycle-semantics-spec.md` §10).
1986///
1987/// Collection is unconditional: SCC tasks are rare relative to ordinary
1988/// vertex evaluation and the counters are a handful of integer adds per
1989/// task, so no config flag gates them (unlike [`VirtualDepTelemetry`],
1990/// which pays per-schedule costs). Counters reset at the start of every
1991/// evaluation request.
1992#[derive(Debug, Clone, Default, PartialEq)]
1993#[non_exhaustive]
1994pub struct CycleTelemetry {
1995    /// SCC tasks executed (static SCCs that reached Runtime evaluation).
1996    pub static_sccs: usize,
1997    /// SCC tasks whose live subgraph was acyclic — values produced.
1998    pub phantom_sccs: usize,
1999    /// Distinct live cycles witnessed across all SCC tasks.
2000    pub live_cycles_witnessed: usize,
2001    /// Cells stamped `#CIRC!` by Runtime SCC tasks.
2002    pub circ_cells_stamped: usize,
2003    /// Evaluation sweeps over (subsets of) SCC members, totalled across tasks
2004    /// (pass 1 included).
2005    pub settle_passes_total: usize,
2006    /// Largest pass count any single SCC task needed.
2007    pub max_passes_single_scc: usize,
2008    /// SCC tasks that entered iterative calculation (`CyclePolicy::Iterate`
2009    /// with a witnessed live cycle). RFC #113, Stage 3.
2010    pub iterated_sccs: usize,
2011    /// Iterating SCC tasks that stopped because every member passed the
2012    /// spec-§6 convergence test.
2013    pub converged_sccs: usize,
2014    /// SCC tasks that stopped at a pass cap. Under `CyclePolicy::Iterate`
2015    /// this is the Excel `max_iterations` cap (NOT an error — last values
2016    /// are kept; includes the no-convergence-test `max_iterations: 1`
2017    /// contract). Under `CyclePolicy::Error` it is the defensive acyclic
2018    /// settle cap (|SCC| + 2), which only a bug can hit.
2019    pub capped_sccs: usize,
2020    /// Largest `|Δ|` observed in any member's final-pass convergence
2021    /// comparison across iterating SCC tasks (numeric-class members only).
2022    /// `0.0` when no comparison ran (e.g. `max_iterations: 1`).
2023    pub max_abs_delta_at_stop: f64,
2024    /// Identical-bit NaN vs NaN member comparisons that were treated as
2025    /// converged (spec §6 NaN rule).
2026    pub nan_converged: usize,
2027    /// Retained iterative SCCs (#368) that had no dirty member at request begin and were therefore not
2028    /// re-run: their last exact fixed point is served as-is. Counted at
2029    /// request begin, so a demand-driven request that never reaches a
2030    /// retained SCC still reports it as reused.
2031    pub reused_sccs: usize,
2032    /// Members of the SCCs counted in `reused_sccs`.
2033    pub reused_scc_members: usize,
2034    /// Total wall-clock time spent inside Runtime SCC tasks.
2035    pub elapsed_ms: u128,
2036}
2037
2038#[derive(Debug, Clone, Copy)]
2039struct ScheduleBuildMeta {
2040    candidate_vertices: usize,
2041    vdeps_vertices: usize,
2042    vdeps_edges: usize,
2043    builder_elapsed_ms: u128,
2044    used_virtual_schedule: bool,
2045    schedule_cache_hit: bool,
2046    schedule_cache_eligible: bool,
2047}
2048
2049#[cfg(any(test, feature = "benchmark_internal"))]
2050#[doc(hidden)]
2051#[derive(Debug, Clone, Default)]
2052pub struct RecalcReuseProbe {
2053    pub schedule_requests: usize,
2054    pub schedule_cache_hits: usize,
2055    pub schedule_cache_misses: usize,
2056    pub schedule_cache_ineligible: usize,
2057    pub schedule_builds: usize,
2058    pub schedule_shared_handles: usize,
2059    pub schedule_retained_bytes: usize,
2060    pub legacy_target_requests: usize,
2061    pub target_schedule_builds: usize,
2062    pub demand_builds: usize,
2063    pub demand_vertices: usize,
2064    pub demand_clean_formulas: usize,
2065    pub demand_explicit_edges: usize,
2066    pub demand_virtual_builder_calls: usize,
2067}
2068
2069#[cfg(any(test, feature = "benchmark_internal"))]
2070fn schedule_probe_retained_bytes(schedule: &crate::engine::Schedule) -> usize {
2071    fn vector_bytes<T>(values: &Vec<T>) -> usize {
2072        values.capacity() * std::mem::size_of::<T>()
2073    }
2074
2075    [
2076        vector_bytes(&schedule.units),
2077        vector_bytes(&schedule.layers),
2078        vector_bytes(&schedule.cycles),
2079    ]
2080    .into_iter()
2081    .chain(
2082        schedule
2083            .layers
2084            .iter()
2085            .map(|layer| vector_bytes(&layer.vertices)),
2086    )
2087    .chain(schedule.cycles.iter().map(vector_bytes))
2088    .sum()
2089}
2090
2091#[derive(Debug, Clone)]
2092struct CachedScheduleEntry {
2093    topology_epoch: u64,
2094    candidate_vertices: Vec<VertexId>,
2095    schedule: Arc<crate::engine::scheduler::Schedule>,
2096}
2097
2098/// Uncacheable requests keep their schedule inline without a shared allocation.
2099enum EvaluationSchedule {
2100    Owned(crate::engine::scheduler::Schedule),
2101    Shared(Arc<crate::engine::scheduler::Schedule>),
2102}
2103
2104impl std::ops::Deref for EvaluationSchedule {
2105    type Target = crate::engine::scheduler::Schedule;
2106
2107    fn deref(&self) -> &Self::Target {
2108        match self {
2109            Self::Owned(schedule) => schedule,
2110            Self::Shared(schedule) => schedule,
2111        }
2112    }
2113}
2114
2115#[derive(Clone, Debug, PartialEq, Eq)]
2116struct MixedTopologyCacheKey {
2117    engine_topology_epoch: u64,
2118    graph_topology_revision: u64,
2119    authority_indexes_epoch: u64,
2120    /// Explicit revisions for retained island membership and boundary views.
2121    /// They currently share the authoritative graph/index clocks but remain
2122    /// distinct key fields so future independent indexes cannot omit them.
2123    legacy_island_revision: u64,
2124    boundary_index_revision: u64,
2125    function_semantic_epoch: u64,
2126    function_provider_revision: Option<u64>,
2127    max_candidates: usize,
2128    max_edges: usize,
2129    max_memory_bytes: usize,
2130}
2131
2132#[derive(Debug)]
2133struct CachedMixedTopology {
2134    key: MixedTopologyCacheKey,
2135    topology: MixedTopology,
2136    producer_results: FormulaProducerResultIndex,
2137    consumer_reads: FormulaConsumerReadIndex,
2138    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
2139    plane_epoch: u64,
2140    island: LegacyIslandPlan,
2141}
2142
2143#[derive(Debug)]
2144struct SkippedMixedTopology {
2145    reason: MixedScheduleFallbackReason,
2146    observed: MixedTopologyCompileStats,
2147    producer_results: FormulaProducerResultIndex,
2148    consumer_reads: FormulaConsumerReadIndex,
2149    span_refs_by_id: BTreeMap<FormulaSpanId, FormulaSpanRef>,
2150    plane_epoch: u64,
2151    island: LegacyIslandPlan,
2152}
2153
2154#[derive(Debug)]
2155enum FormulaPlaneTopologyCompileResult {
2156    Cached(CachedMixedTopology),
2157    CacheSkipped(SkippedMixedTopology),
2158}
2159
2160#[derive(Clone, Copy, Debug)]
2161struct ExactDemandScratchEstimates {
2162    paged: u64,
2163    runs: u64,
2164    native: u64,
2165    repeated: u64,
2166}
2167
2168fn exact_demand_scratch_estimates(
2169    read_count: usize,
2170    closure_base: u64,
2171) -> ExactDemandScratchEstimates {
2172    let read_count = read_count as u64;
2173    let page_count = read_count.div_ceil(128);
2174    let run_count = read_count.div_ceil(256);
2175    let pointer_bytes = std::mem::size_of::<usize>() as u64;
2176    let keyed_entry_bytes = (std::mem::size_of::<FormulaProducerId>()
2177        + std::mem::size_of::<Vec<usize>>()
2178        + 12 * std::mem::size_of::<usize>()) as u64;
2179    let paged_memory = read_count.saturating_mul(keyed_entry_bytes).saturating_add(
2180        page_count.saturating_mul((2 * std::mem::size_of::<BTreeMap<(), ()>>()) as u64),
2181    );
2182    let runs_memory = read_count
2183        .saturating_mul(pointer_bytes)
2184        .saturating_add(run_count.saturating_mul((2 * std::mem::size_of::<Vec<usize>>()) as u64));
2185    ExactDemandScratchEstimates {
2186        paged: closure_base.saturating_add(paged_memory),
2187        runs: closure_base.saturating_add(runs_memory),
2188        native: closure_base
2189            .saturating_add(crate::formula_plane::scheduler::NATIVE_EXACT_DEMAND_SCRATCH_BYTES),
2190        repeated: closure_base.saturating_add(512),
2191    }
2192}
2193
2194fn native_exact_demand_allowed(
2195    policy: Option<crate::engine::DiskScratchPolicy>,
2196    target_supports_native: bool,
2197) -> bool {
2198    target_supports_native && policy == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
2199}
2200
2201fn select_exact_demand_strategy(
2202    estimates: ExactDemandScratchEstimates,
2203    native_allowed: bool,
2204    mut can_reserve: impl FnMut(u64) -> bool,
2205) -> (FormulaPlaneTopologyStrategy, u64) {
2206    if can_reserve(estimates.paged) {
2207        (
2208            FormulaPlaneTopologyStrategy::ExactPagedIndexed,
2209            estimates.paged,
2210        )
2211    } else if can_reserve(estimates.runs) {
2212        (
2213            FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
2214            estimates.runs,
2215        )
2216    } else if native_allowed && can_reserve(estimates.native) {
2217        (
2218            FormulaPlaneTopologyStrategy::ExactNativeScratch,
2219            estimates.native,
2220        )
2221    } else {
2222        (
2223            FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
2224            estimates.repeated,
2225        )
2226    }
2227}
2228
2229#[cfg(test)]
2230mod exact_demand_strategy_tests {
2231    use super::*;
2232
2233    #[test]
2234    fn every_exact_demand_strategy_is_selected_by_its_scratch_rung() {
2235        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2236        assert!(estimates.paged > estimates.runs);
2237        assert!(estimates.runs > estimates.native);
2238        assert!(estimates.native > estimates.repeated);
2239        for (limit, native_allowed, expected) in [
2240            (
2241                estimates.paged,
2242                true,
2243                FormulaPlaneTopologyStrategy::ExactPagedIndexed,
2244            ),
2245            (
2246                estimates.runs,
2247                true,
2248                FormulaPlaneTopologyStrategy::ExactInMemoryRuns,
2249            ),
2250            (
2251                estimates.native,
2252                true,
2253                FormulaPlaneTopologyStrategy::ExactNativeScratch,
2254            ),
2255            (
2256                estimates.repeated,
2257                true,
2258                FormulaPlaneTopologyStrategy::ExactRepeatedPasses,
2259            ),
2260        ] {
2261            assert_eq!(
2262                select_exact_demand_strategy(estimates, native_allowed, |bytes| bytes <= limit).0,
2263                expected
2264            );
2265        }
2266    }
2267
2268    #[cfg(not(target_arch = "wasm32"))]
2269    #[test]
2270    fn native_topology_scratch_drop_removes_primary_and_auxiliary_files() {
2271        let scratch = NativeTopologyScratch::create(u64::MAX).unwrap();
2272        let path = scratch.path.clone();
2273        let auxiliary_path = scratch.auxiliary_path.clone();
2274        assert!(path.exists());
2275        assert!(auxiliary_path.exists());
2276        drop(scratch);
2277        assert!(!path.exists());
2278        assert!(!auxiliary_path.exists());
2279    }
2280
2281    #[test]
2282    fn native_policy_and_wasm_target_support_gate_native_rung() {
2283        let estimates = exact_demand_scratch_estimates(4_096, 4_096);
2284        assert!(!native_exact_demand_allowed(
2285            Some(crate::engine::DiskScratchPolicy::MemoryOnly),
2286            true,
2287        ));
2288        assert!(!native_exact_demand_allowed(
2289            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2290            false,
2291        ));
2292        let native_allowed = native_exact_demand_allowed(
2293            Some(crate::engine::DiskScratchPolicy::NativeTemporary),
2294            !cfg!(target_arch = "wasm32"),
2295        );
2296        let expected = if cfg!(target_arch = "wasm32") {
2297            FormulaPlaneTopologyStrategy::ExactRepeatedPasses
2298        } else {
2299            FormulaPlaneTopologyStrategy::ExactNativeScratch
2300        };
2301        assert_eq!(
2302            select_exact_demand_strategy(estimates, native_allowed, |bytes| {
2303                bytes <= estimates.native
2304            })
2305            .0,
2306            expected
2307        );
2308    }
2309}
2310
2311#[cfg(not(target_arch = "wasm32"))]
2312struct NativeTopologyScratch {
2313    path: std::path::PathBuf,
2314    file: Option<std::fs::File>,
2315    auxiliary_path: std::path::PathBuf,
2316    auxiliary_file: Option<std::fs::File>,
2317}
2318
2319#[cfg(not(target_arch = "wasm32"))]
2320impl NativeTopologyScratch {
2321    fn create(request_id: u64) -> std::io::Result<Self> {
2322        use std::fs::OpenOptions;
2323        for attempt in 0..32_u64 {
2324            let path = std::env::temp_dir().join(format!(
2325                "formualizer-topology-{request_id}-{attempt}-{}.tmp",
2326                std::process::id()
2327            ));
2328            match OpenOptions::new()
2329                .read(true)
2330                .write(true)
2331                .create_new(true)
2332                .open(&path)
2333            {
2334                Ok(file) => {
2335                    let auxiliary_path = path.with_extension("aux.tmp");
2336                    match OpenOptions::new()
2337                        .read(true)
2338                        .write(true)
2339                        .create_new(true)
2340                        .open(&auxiliary_path)
2341                    {
2342                        Ok(auxiliary_file) => {
2343                            return Ok(Self {
2344                                path,
2345                                file: Some(file),
2346                                auxiliary_path,
2347                                auxiliary_file: Some(auxiliary_file),
2348                            });
2349                        }
2350                        Err(error) => {
2351                            drop(file);
2352                            let _ = std::fs::remove_file(&path);
2353                            if error.kind() == std::io::ErrorKind::AlreadyExists {
2354                                continue;
2355                            }
2356                            return Err(error);
2357                        }
2358                    }
2359                }
2360                Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => continue,
2361                Err(error) => return Err(error),
2362            }
2363        }
2364        Err(std::io::Error::new(
2365            std::io::ErrorKind::AlreadyExists,
2366            "could not allocate unique topology scratch file",
2367        ))
2368    }
2369}
2370
2371#[cfg(not(target_arch = "wasm32"))]
2372impl Drop for NativeTopologyScratch {
2373    fn drop(&mut self) {
2374        drop(self.file.take());
2375        drop(self.auxiliary_file.take());
2376        let _ = std::fs::remove_file(&self.path);
2377        let _ = std::fs::remove_file(&self.auxiliary_path);
2378    }
2379}
2380
2381fn estimated_span_bindings_bytes(
2382    bindings: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
2383) -> Option<usize> {
2384    const TREE_ENTRY_OVERHEAD: usize = 4 * std::mem::size_of::<usize>();
2385    bindings.len().checked_mul(
2386        std::mem::size_of::<FormulaSpanId>()
2387            .checked_add(std::mem::size_of::<FormulaSpanRef>())?
2388            .checked_add(TREE_ENTRY_OVERHEAD)?,
2389    )
2390}
2391
2392type ScheduleBuildOutput = (
2393    crate::engine::scheduler::Schedule,
2394    FxHashMap<VertexId, Vec<VertexId>>,
2395    ScheduleBuildMeta,
2396);
2397
2398type EvaluationScheduleBuildOutput = (
2399    EvaluationSchedule,
2400    FxHashMap<VertexId, Vec<VertexId>>,
2401    ScheduleBuildMeta,
2402);
2403
2404/// Opaque, revision-bound recalculation recipe.
2405#[derive(Debug)]
2406pub struct RecalcPlan {
2407    key: RecalcPlanKey,
2408    kind: RecalcPlanKind,
2409}
2410
2411#[derive(Debug)]
2412struct RecalcPlanKey {
2413    engine_token: Arc<()>,
2414    revisions: PlanningRevisionSnapshot,
2415}
2416
2417#[derive(Clone, Debug, PartialEq, Eq)]
2418struct PlanningRevisionSnapshot {
2419    engine_topology_epoch: u64,
2420    graph_topology_revision: u64,
2421    authority: u64,
2422    authority_indexes: u64,
2423    authority_indexed_plane: u64,
2424    staged: u64,
2425    symbols: u64,
2426    semantic: u64,
2427    provider: Option<u64>,
2428    formula_plane_mode: FormulaPlaneMode,
2429    deterministic_mode: crate::engine::DeterministicMode,
2430    budgets: crate::engine::EvaluationBudgets,
2431    span_refs: Vec<FormulaSpanRef>,
2432}
2433
2434#[derive(Debug)]
2435enum RecalcPlanKind {
2436    CompatibilityFull {
2437        schedule: crate::engine::Schedule,
2438        has_dynamic_refs: bool,
2439    },
2440    Target {
2441        targets: Vec<crate::engine::EvaluationTarget>,
2442        scope: crate::engine::PrepareScope,
2443        topology: RecalcTopology,
2444        dynamic_policy: DynamicPlanPolicy,
2445    },
2446}
2447
2448#[derive(Debug)]
2449enum RecalcTopology {
2450    RunLocalRecipe,
2451    Workbook,
2452}
2453
2454#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2455enum DynamicPlanPolicy {
2456    BoundedTargetReplan,
2457}
2458
2459impl RecalcPlan {
2460    /// Returns the retained compatibility schedule depth. Target plans retain a
2461    /// run-local recipe rather than a schedule, so their layer count is zero.
2462    pub fn layer_count(&self) -> usize {
2463        match &self.kind {
2464            RecalcPlanKind::CompatibilityFull { schedule, .. } => schedule.layers.len(),
2465            RecalcPlanKind::Target { .. } => 0,
2466        }
2467    }
2468
2469    pub fn has_dynamic_refs(&self) -> bool {
2470        match &self.kind {
2471            RecalcPlanKind::CompatibilityFull {
2472                has_dynamic_refs, ..
2473            } => *has_dynamic_refs,
2474            RecalcPlanKind::Target { .. } => false,
2475        }
2476    }
2477
2478    #[cfg(test)]
2479    pub(crate) fn force_stale_reasons_for_test(
2480        &mut self,
2481        reasons: &[formualizer_common::PlanStaleReason],
2482    ) {
2483        use formualizer_common::PlanStaleReason;
2484        for reason in reasons {
2485            match reason {
2486                PlanStaleReason::Engine => {
2487                    self.key.engine_token = Arc::new(());
2488                }
2489                PlanStaleReason::Provider => {
2490                    self.key.revisions.provider = Some(
2491                        self.key
2492                            .revisions
2493                            .provider
2494                            .unwrap_or_default()
2495                            .wrapping_add(1),
2496                    );
2497                }
2498                PlanStaleReason::Semantic => {
2499                    self.key.revisions.semantic = self.key.revisions.semantic.wrapping_add(1);
2500                }
2501                PlanStaleReason::Budget => {
2502                    let current = self.key.revisions.budgets.work.max_work_units;
2503                    self.key.revisions.budgets.work.max_work_units =
2504                        Some(current.unwrap_or_default().wrapping_add(1));
2505                }
2506                PlanStaleReason::Staged => {
2507                    self.key.revisions.staged = self.key.revisions.staged.wrapping_add(1);
2508                }
2509                PlanStaleReason::Symbols => {
2510                    self.key.revisions.symbols = self.key.revisions.symbols.wrapping_add(1);
2511                }
2512                PlanStaleReason::Authority => {
2513                    self.key.revisions.authority = self.key.revisions.authority.wrapping_add(1);
2514                }
2515                PlanStaleReason::SpanGeneration => {
2516                    self.key.revisions.span_refs.push(FormulaSpanRef {
2517                        id: crate::formula_plane::runtime::FormulaSpanId(u32::MAX),
2518                        generation: u32::MAX,
2519                        version: u32::MAX,
2520                    });
2521                }
2522                PlanStaleReason::Graph => {
2523                    self.key.revisions.graph_topology_revision =
2524                        self.key.revisions.graph_topology_revision.wrapping_add(1);
2525                }
2526                _ => {}
2527            }
2528        }
2529    }
2530}
2531
2532#[cfg(any(test, feature = "test-support"))]
2533pub(crate) mod criteria_mask_test_hooks {
2534    use std::cell::Cell;
2535
2536    thread_local! {
2537        static MASK_CALLS_ROWS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
2538        static TEXT_SEGMENTS_TOTAL: Cell<usize> = const { Cell::new(0) };
2539        static TEXT_SEGMENTS_ALL_NULL: Cell<usize> = const { Cell::new(0) };
2540    }
2541
2542    pub(crate) fn take_mask_work() -> (usize, usize) {
2543        MASK_CALLS_ROWS.with(|c| c.replace((0, 0)))
2544    }
2545
2546    pub(crate) fn note_mask(rows: usize) {
2547        MASK_CALLS_ROWS.with(|c| {
2548            let (calls, work) = c.get();
2549            c.set((calls + 1, work + rows));
2550        });
2551    }
2552
2553    pub fn reset_text_segment_counters() {
2554        TEXT_SEGMENTS_TOTAL.with(|c| c.set(0));
2555        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(0));
2556    }
2557
2558    pub fn text_segment_counters() -> (usize, usize) {
2559        let a = TEXT_SEGMENTS_TOTAL.with(|c| c.get());
2560        let b = TEXT_SEGMENTS_ALL_NULL.with(|c| c.get());
2561        (a, b)
2562    }
2563
2564    pub(crate) fn inc_total() {
2565        TEXT_SEGMENTS_TOTAL.with(|c| c.set(c.get() + 1));
2566    }
2567    pub(crate) fn inc_all_null() {
2568        TEXT_SEGMENTS_ALL_NULL.with(|c| c.set(c.get() + 1));
2569    }
2570}
2571
2572#[cfg(test)]
2573pub(crate) mod visibility_mask_test_hooks {
2574    use std::cell::Cell;
2575
2576    thread_local! {
2577        static HITS: Cell<usize> = const { Cell::new(0) };
2578        static MISSES: Cell<usize> = const { Cell::new(0) };
2579        static EVICTIONS: Cell<usize> = const { Cell::new(0) };
2580    }
2581
2582    pub fn reset() {
2583        HITS.with(|c| c.set(0));
2584        MISSES.with(|c| c.set(0));
2585        EVICTIONS.with(|c| c.set(0));
2586    }
2587
2588    pub fn counters() -> (usize, usize, usize) {
2589        let hits = HITS.with(|c| c.get());
2590        let misses = MISSES.with(|c| c.get());
2591        let evictions = EVICTIONS.with(|c| c.get());
2592        (hits, misses, evictions)
2593    }
2594
2595    pub(crate) fn inc_hit() {
2596        HITS.with(|c| c.set(c.get() + 1));
2597    }
2598
2599    pub(crate) fn inc_miss() {
2600        MISSES.with(|c| c.set(c.get() + 1));
2601    }
2602
2603    pub(crate) fn inc_eviction() {
2604        EVICTIONS.with(|c| c.set(c.get() + 1));
2605    }
2606}
2607
2608fn compute_criteria_mask(
2609    view: &RangeView<'_>,
2610    col_in_view: usize,
2611    pred: &crate::args::CriteriaPredicate,
2612) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
2613    use crate::compute_prelude::{boolean, cmp, concat_arrays};
2614    use arrow::compute::kernels::comparison::{ilike, nilike};
2615    use arrow_array::{
2616        Array as _, ArrayRef, BooleanArray, Float64Array, StringArray, builder::BooleanBuilder,
2617    };
2618
2619    // Helper: apply a numeric predicate to a single Float64Array chunk
2620    fn apply_numeric_pred(
2621        chunk: &Float64Array,
2622        pred: &crate::args::CriteriaPredicate,
2623    ) -> Option<BooleanArray> {
2624        match pred {
2625            crate::args::CriteriaPredicate::Gt(n) => {
2626                cmp::gt(chunk, &Float64Array::new_scalar(*n)).ok()
2627            }
2628            crate::args::CriteriaPredicate::Ge(n) => {
2629                cmp::gt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2630            }
2631            crate::args::CriteriaPredicate::Lt(n) => {
2632                cmp::lt(chunk, &Float64Array::new_scalar(*n)).ok()
2633            }
2634            crate::args::CriteriaPredicate::Le(n) => {
2635                cmp::lt_eq(chunk, &Float64Array::new_scalar(*n)).ok()
2636            }
2637            crate::args::CriteriaPredicate::Eq(v) => match v {
2638                formualizer_common::LiteralValue::Number(x) => {
2639                    cmp::eq(chunk, &Float64Array::new_scalar(*x)).ok()
2640                }
2641                formualizer_common::LiteralValue::Int(i) => {
2642                    cmp::eq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2643                }
2644                _ => None,
2645            },
2646            crate::args::CriteriaPredicate::Ne(v) => match v {
2647                formualizer_common::LiteralValue::Number(x) => {
2648                    cmp::neq(chunk, &Float64Array::new_scalar(*x)).ok()
2649                }
2650                formualizer_common::LiteralValue::Int(i) => {
2651                    cmp::neq(chunk, &Float64Array::new_scalar(*i as f64)).ok()
2652                }
2653                _ => None,
2654            },
2655            _ => None,
2656        }
2657    }
2658
2659    // Check if this is a numeric predicate that can be applied per-chunk
2660    let is_numeric_pred = matches!(
2661        pred,
2662        crate::args::CriteriaPredicate::Gt(_)
2663            | crate::args::CriteriaPredicate::Ge(_)
2664            | crate::args::CriteriaPredicate::Lt(_)
2665            | crate::args::CriteriaPredicate::Le(_)
2666            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Number(_))
2667            | crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Int(_))
2668            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Number(_))
2669            | crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Int(_))
2670    );
2671
2672    // OPTIMIZED PATH: For numeric predicates, apply per-chunk and concatenate boolean masks.
2673    // This avoids materializing the full numeric column (64-bit per element) and instead
2674    // concatenates boolean masks (1-bit per element) - a 64x memory reduction.
2675    if is_numeric_pred {
2676        let mut bool_parts: Vec<BooleanArray> = Vec::new();
2677        for res in view.numbers_slices() {
2678            let (_rs, _rl, cols_seg) = res.ok()?;
2679            if col_in_view < cols_seg.len() {
2680                let chunk = cols_seg[col_in_view].as_ref();
2681                let mask = apply_numeric_pred(chunk, pred)?;
2682                bool_parts.push(mask);
2683            }
2684        }
2685
2686        if bool_parts.is_empty() {
2687            return None;
2688        } else if bool_parts.len() == 1 {
2689            return Some(std::sync::Arc::new(bool_parts.remove(0)));
2690        } else {
2691            // Concatenate boolean masks (much cheaper than concatenating Float64 arrays)
2692            let anys: Vec<&dyn arrow_array::Array> = bool_parts
2693                .iter()
2694                .map(|a| a as &dyn arrow_array::Array)
2695                .collect();
2696            let conc: ArrayRef = concat_arrays(&anys).ok()?;
2697            let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2698            return Some(std::sync::Arc::new(ba));
2699        }
2700    }
2701
2702    // TEXT PATH: build masks per row-chunk using lowered text slices.
2703    // This avoids concatenating full-string columns just to compute a boolean mask.
2704    let (text_kind, text_pat, empty_special) = match pred {
2705        crate::args::CriteriaPredicate::Eq(formualizer_common::LiteralValue::Text(t)) => {
2706            (0u8, t.to_lowercase(), t.is_empty())
2707        }
2708        crate::args::CriteriaPredicate::Ne(formualizer_common::LiteralValue::Text(t)) => {
2709            (1u8, t.to_lowercase(), false)
2710        }
2711        crate::args::CriteriaPredicate::TextLike {
2712            pattern,
2713            case_insensitive,
2714        } => {
2715            let p = if *case_insensitive {
2716                pattern.to_lowercase()
2717            } else {
2718                pattern.clone()
2719            };
2720            (2u8, p.replace('*', "%").replace('?', "_"), false)
2721        }
2722        _ => return None,
2723    };
2724
2725    let ne_matches_blank = text_kind == 1 && !text_pat.is_empty();
2726    let pat = StringArray::new_scalar(text_pat);
2727    let mut bool_parts: Vec<BooleanArray> = Vec::new();
2728
2729    for res in view.iter_row_chunks() {
2730        let cs = res.ok()?;
2731        if cs.row_len == 0 {
2732            continue;
2733        }
2734        #[cfg(test)]
2735        criteria_mask_test_hooks::inc_total();
2736
2737        let slices = view.slice_lowered_text(cs.row_start, cs.row_len);
2738        if col_in_view >= slices.len() {
2739            return None;
2740        }
2741
2742        let seg_opt = slices[col_in_view].as_ref().map(|a| a.as_ref());
2743        let seg = match seg_opt {
2744            Some(s) => s,
2745            None => {
2746                #[cfg(test)]
2747                criteria_mask_test_hooks::inc_all_null();
2748                if (text_kind == 0 && empty_special) || ne_matches_blank {
2749                    // Eq("") treats nulls (Empty) as equal.
2750                    let mut bb = BooleanBuilder::with_capacity(cs.row_len);
2751                    bb.append_n(cs.row_len, true);
2752                    bool_parts.push(bb.finish());
2753                } else {
2754                    // For non-empty patterns, ilike/nilike return null on null inputs.
2755                    bool_parts.push(BooleanArray::new_null(cs.row_len));
2756                }
2757                continue;
2758            }
2759        };
2760
2761        let seg_sa = seg.as_any().downcast_ref::<StringArray>()?;
2762        let mut m = match text_kind {
2763            0 => ilike(seg_sa, &pat).ok()?,
2764            1 => nilike(seg_sa, &pat).ok()?,
2765            2 => ilike(seg_sa, &pat).ok()?,
2766            _ => return None,
2767        };
2768
2769        // Only fold blank/Empty (null) cells into the mask when the segment
2770        // actually contains any. The null-fill loop + or_kleene are pure
2771        // overhead on blank-free chunks, so a `<>text` (or `=""`) aggregation
2772        // over a column with no blanks stays fully vectorized on the ilike/
2773        // nilike result.
2774        if ((text_kind == 0 && empty_special) || ne_matches_blank) && seg_sa.null_count() > 0 {
2775            // Treat nulls as equal to empty string
2776            let mut bb = BooleanBuilder::with_capacity(seg_sa.len());
2777            for i in 0..seg_sa.len() {
2778                bb.append_value(seg_sa.is_null(i));
2779            }
2780            let nulls = bb.finish();
2781            m = boolean::or_kleene(&m, &nulls).ok()?;
2782        }
2783
2784        bool_parts.push(m);
2785    }
2786
2787    if bool_parts.is_empty() {
2788        None
2789    } else if bool_parts.len() == 1 {
2790        Some(std::sync::Arc::new(bool_parts.remove(0)))
2791    } else {
2792        let anys: Vec<&dyn arrow_array::Array> = bool_parts
2793            .iter()
2794            .map(|a| a as &dyn arrow_array::Array)
2795            .collect();
2796        let conc: ArrayRef = concat_arrays(&anys).ok()?;
2797        let ba = conc.as_any().downcast_ref::<BooleanArray>()?.clone();
2798        Some(std::sync::Arc::new(ba))
2799    }
2800}
2801
2802#[derive(Debug, Clone)]
2803pub struct LayerInfo {
2804    pub vertex_count: usize,
2805    pub parallel_eligible: bool,
2806    pub sample_cells: Vec<String>, // Sample of up to 5 cell addresses
2807}
2808
2809#[derive(Debug, Clone)]
2810pub struct EvalPlan {
2811    pub total_vertices_to_evaluate: usize,
2812    pub layers: Vec<LayerInfo>,
2813    pub cycles_detected: usize,
2814    pub dirty_count: usize,
2815    pub volatile_count: usize,
2816    pub parallel_enabled: bool,
2817    pub estimated_parallel_layers: usize,
2818    pub target_cells: Vec<String>,
2819}
2820
2821impl<R> Engine<R>
2822where
2823    R: EvaluationContext,
2824{
2825    /// # Panics
2826    /// Panics when `config.cycle` is invalid ([`CycleConfig::validate`],
2827    /// spec §2): `Iterate` with `detection: Static`, `max_iterations == 0`,
2828    /// or a negative/non-finite `max_change`. `EvalConfig::with_cycle`
2829    /// rejects these at build; this re-validates configs assembled via
2830    /// struct literals.
2831    pub fn new(resolver: R, config: EvalConfig) -> Self {
2832        if let Err(msg) = config.cycle.validate() {
2833            panic!("invalid CycleConfig: {msg}");
2834        }
2835        crate::builtins::load_builtins();
2836        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
2837            &config.evaluation_budgets,
2838            config.max_vertices,
2839            config.max_memory_mb,
2840            config.max_eval_time,
2841        );
2842
2843        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
2844            #[cfg(feature = "system-clock")]
2845            {
2846                Arc::new(crate::timezone::SystemClock::new(
2847                    crate::timezone::TimeZoneSpec::default(),
2848                ))
2849            }
2850            #[cfg(not(feature = "system-clock"))]
2851            {
2852                Arc::new(crate::timezone::FixedClock::new(
2853                    chrono::DateTime::UNIX_EPOCH,
2854                    crate::timezone::TimeZoneSpec::Utc,
2855                ))
2856            }
2857        });
2858
2859        // Initialize thread pool based on config
2860        let thread_pool = if config.enable_parallel {
2861            let mut builder = ThreadPoolBuilder::new();
2862            if let Some(max_threads) = config.max_threads {
2863                builder = builder.num_threads(max_threads);
2864            }
2865
2866            match builder.build() {
2867                Ok(pool) => Some(Arc::new(pool)),
2868                Err(_) => {
2869                    // Fall back to sequential evaluation if thread pool creation fails
2870                    None
2871                }
2872            }
2873        } else {
2874            None
2875        };
2876
2877        // C1a retained/cache budgets are observational; cache defaults stay explicit.
2878        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
2879        let function_provider_revision_seen = resolver.planning_semantic_revision();
2880        let mut engine = Self {
2881            graph: DependencyGraph::new_with_config(config.clone()),
2882            resolver,
2883            config,
2884            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
2885            clock: crate::timezone::SnapshotClock::new(clock),
2886            thread_pool,
2887            recalc_epoch: 0,
2888            snapshot_id: std::sync::atomic::AtomicU64::new(1),
2889            topology_epoch: 0,
2890            legacy_island_structural_summaries_trusted: true,
2891            cached_static_schedule: None,
2892            #[cfg(any(test, feature = "benchmark_internal"))]
2893            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
2894            cached_mixed_topology: None,
2895            mixed_topology_cache_builds: 0,
2896            mixed_topology_cache_hits: 0,
2897            mixed_topology_cache_overflows: 0,
2898            mixed_topology_cache_skip_streak: 0,
2899            spill_mgr: ShimSpillManager::default(),
2900            arrow_sheets: SheetStore::default(),
2901            format_registry: crate::format::FormatRegistry::default(),
2902            derived_format_results: std::sync::RwLock::new(FxHashMap::default()),
2903            derived_formats: std::sync::RwLock::new(FxHashMap::default()),
2904            derived_formats_plane_epoch: 0,
2905            #[cfg(test)]
2906            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
2907            #[cfg(test)]
2908            computed_overlay_set_explicit_entry_operations_for_test: 0,
2909            #[cfg(test)]
2910            computed_overlay_stale_clear_range_effects_for_test: 0,
2911            #[cfg(test)]
2912            computed_overlay_stale_clear_offset_attempts_for_test: 0,
2913            #[cfg(test)]
2914            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
2915            has_edited: false,
2916            overlay_compactions: 0,
2917            computed_overlay_bytes_estimate: 0,
2918            computed_overlay_mirroring_disabled: false,
2919            force_materialize_range_views: false,
2920            row_bounds_cache: std::sync::RwLock::new(None),
2921            used_axis_bounds_cache: std::sync::RwLock::new(None),
2922            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
2923            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
2924            source_formula_token: Arc::new(()),
2925            recalc_plan_token: Arc::new(()),
2926            staged_formulas: std::collections::HashMap::new(),
2927            staged_formula_index: StagedFormulaIndex::default(),
2928            row_visibility: FxHashMap::default(),
2929            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
2930            formula_parse_diagnostics: Vec::new(),
2931            last_formula_ingest_report: None,
2932            formula_ingest_report_total: FormulaIngestReport::default(),
2933            formula_plane_cycle_member_span_demotions: 0,
2934            formula_plane_array_result_span_demotions: 0,
2935            formula_plane_capacity_bailouts: 0,
2936            formula_plane_structural_span_candidates: 0,
2937            active_cancel_flag: None,
2938            active_evaluation_deadline: None,
2939            action_depth: 0,
2940            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
2941            virtual_dep_fallback_activations: 0,
2942            last_cycle_telemetry: CycleTelemetry::default(),
2943            next_evaluation_resource_request_id: 1,
2944            evaluation_resource_request_depth: 0,
2945            active_evaluation_resource_request: None,
2946            last_evaluation_resource_request: None,
2947            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
2948            evaluation_resource_request_started_at: None,
2949            evaluation_resource_budgets: resolved_resources.budgets,
2950            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
2951            active_resource_ledger: None,
2952            pending_iterative_redirty: Vec::new(),
2953            retained_scc_members: FxHashMap::default(),
2954            next_retained_scc_id: 0,
2955            retained_scc_config_fingerprint: 0,
2956            retained_scc_function_epoch_seen: 0,
2957            retained_scc_provider_revision_seen: None,
2958            retained_scc_dirty_at_begin: Vec::new(),
2959            iterative_state_values: FxHashMap::default(),
2960            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
2961            function_provider_revision_seen,
2962            #[cfg(test)]
2963            last_formula_plane_span_eval_report: None,
2964            #[cfg(test)]
2965            evaluation_request_begin_count_for_test: 0,
2966            #[cfg(any(test, feature = "test-support"))]
2967            before_prepared_span_commit_hook: None,
2968            #[cfg(test)]
2969            before_target_preparation_commit_hook: None,
2970            #[cfg(test)]
2971            before_target_planning_snapshot_hook: None,
2972            #[cfg(test)]
2973            inject_target_semantic_stale_once_for_test: false,
2974            #[cfg(test)]
2975            force_virtual_dep_changes_remaining_for_test: 0,
2976            #[cfg(test)]
2977            fail_evaluation_commit_preflight_once_for_test: false,
2978            #[cfg(test)]
2979            cancel_before_formula_plane_layer_commit_once_for_test: false,
2980            #[cfg(test)]
2981            force_source_family_fallback: false,
2982            #[cfg(test)]
2983            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
2984            #[cfg(test)]
2985            fragmented_commit_fault_for_test: None,
2986            #[cfg(test)]
2987            formula_span_demotion_fault_for_test: None,
2988            #[cfg(test)]
2989            target_preparation_fault_for_test: None,
2990            #[cfg(test)]
2991            force_non_cycle_schedule_fallback_for_test: false,
2992            #[cfg(test)]
2993            mixed_topology_index_builds_for_test: 0,
2994            #[cfg(test)]
2995            before_legacy_fallback_final_provider_sample_hook: None,
2996            #[cfg(test)]
2997            after_eager_proposal_commit_hook: None,
2998        };
2999        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3000        engine.config.arrow_storage_enabled = true;
3001        engine.config.delta_overlay_enabled = true;
3002        engine.config.write_formula_overlay_enabled = true;
3003        let default_sheet = engine.graph.default_sheet_name().to_string();
3004        engine.ensure_arrow_sheet(&default_sheet);
3005        engine
3006    }
3007
3008    /// Create an Engine with a custom thread pool (for shared thread pool scenarios)
3009    ///
3010    /// # Panics
3011    /// Panics when `config.cycle` is invalid, exactly like [`Engine::new`].
3012    pub fn with_thread_pool(
3013        resolver: R,
3014        config: EvalConfig,
3015        thread_pool: Arc<rayon::ThreadPool>,
3016    ) -> Self {
3017        if let Err(msg) = config.cycle.validate() {
3018            panic!("invalid CycleConfig: {msg}");
3019        }
3020        crate::builtins::load_builtins();
3021        let resolved_resources = crate::engine::resource_ledger::resolve_evaluation_budgets(
3022            &config.evaluation_budgets,
3023            config.max_vertices,
3024            config.max_memory_mb,
3025            config.max_eval_time,
3026        );
3027        let clock = config.deterministic_mode.build_clock().unwrap_or_else(|_| {
3028            #[cfg(feature = "system-clock")]
3029            {
3030                Arc::new(crate::timezone::SystemClock::new(
3031                    crate::timezone::TimeZoneSpec::default(),
3032                ))
3033            }
3034            #[cfg(not(feature = "system-clock"))]
3035            {
3036                Arc::new(crate::timezone::FixedClock::new(
3037                    chrono::DateTime::UNIX_EPOCH,
3038                    crate::timezone::TimeZoneSpec::Utc,
3039                ))
3040            }
3041        });
3042        // C1a retained/cache budgets are observational; cache defaults stay explicit.
3043        let lookup_cache_max_bytes = config.lookup_index_cache_max_bytes;
3044        let function_provider_revision_seen = resolver.planning_semantic_revision();
3045        let mut engine = Self {
3046            graph: DependencyGraph::new_with_config(config.clone()),
3047            resolver,
3048            config,
3049            workbook_load_limits: crate::engine::WorkbookLoadLimits::default(),
3050            clock: crate::timezone::SnapshotClock::new(clock),
3051            thread_pool: Some(thread_pool),
3052            recalc_epoch: 0,
3053            snapshot_id: std::sync::atomic::AtomicU64::new(1),
3054            topology_epoch: 0,
3055            legacy_island_structural_summaries_trusted: true,
3056            cached_static_schedule: None,
3057            #[cfg(any(test, feature = "benchmark_internal"))]
3058            recalc_reuse_probe: std::sync::Mutex::new(RecalcReuseProbe::default()),
3059            cached_mixed_topology: None,
3060            mixed_topology_cache_builds: 0,
3061            mixed_topology_cache_hits: 0,
3062            mixed_topology_cache_overflows: 0,
3063            mixed_topology_cache_skip_streak: 0,
3064            spill_mgr: ShimSpillManager::default(),
3065            arrow_sheets: SheetStore::default(),
3066            format_registry: crate::format::FormatRegistry::default(),
3067            derived_format_results: std::sync::RwLock::new(FxHashMap::default()),
3068            derived_formats: std::sync::RwLock::new(FxHashMap::default()),
3069            derived_formats_plane_epoch: 0,
3070            #[cfg(test)]
3071            derived_format_operations_for_test: std::sync::atomic::AtomicU64::new(0),
3072            #[cfg(test)]
3073            computed_overlay_set_explicit_entry_operations_for_test: 0,
3074            #[cfg(test)]
3075            computed_overlay_stale_clear_range_effects_for_test: 0,
3076            #[cfg(test)]
3077            computed_overlay_stale_clear_offset_attempts_for_test: 0,
3078            #[cfg(test)]
3079            computed_format_vector_allocations_for_test: std::sync::atomic::AtomicU64::new(0),
3080            has_edited: false,
3081            overlay_compactions: 0,
3082            computed_overlay_bytes_estimate: 0,
3083            computed_overlay_mirroring_disabled: false,
3084            force_materialize_range_views: false,
3085            row_bounds_cache: std::sync::RwLock::new(None),
3086            used_axis_bounds_cache: std::sync::RwLock::new(None),
3087            lookup_index_cache: LookupIndexCache::new(lookup_cache_max_bytes),
3088            source_cache: Arc::new(std::sync::RwLock::new(SourceCache::default())),
3089            source_formula_token: Arc::new(()),
3090            recalc_plan_token: Arc::new(()),
3091            staged_formulas: std::collections::HashMap::new(),
3092            staged_formula_index: StagedFormulaIndex::default(),
3093            row_visibility: FxHashMap::default(),
3094            row_visibility_mask_cache: std::sync::RwLock::new(FxHashMap::default()),
3095            formula_parse_diagnostics: Vec::new(),
3096            last_formula_ingest_report: None,
3097            formula_ingest_report_total: FormulaIngestReport::default(),
3098            formula_plane_cycle_member_span_demotions: 0,
3099            formula_plane_array_result_span_demotions: 0,
3100            formula_plane_capacity_bailouts: 0,
3101            formula_plane_structural_span_candidates: 0,
3102            active_cancel_flag: None,
3103            active_evaluation_deadline: None,
3104            action_depth: 0,
3105            last_virtual_dep_telemetry: VirtualDepTelemetry::default(),
3106            virtual_dep_fallback_activations: 0,
3107            last_cycle_telemetry: CycleTelemetry::default(),
3108            next_evaluation_resource_request_id: 1,
3109            evaluation_resource_request_depth: 0,
3110            active_evaluation_resource_request: None,
3111            last_evaluation_resource_request: None,
3112            evaluation_resource_baseline: EvaluationResourceBaselineStats::default(),
3113            evaluation_resource_request_started_at: None,
3114            evaluation_resource_budgets: resolved_resources.budgets,
3115            evaluation_resource_config_diagnostic: resolved_resources.diagnostic,
3116            active_resource_ledger: None,
3117            pending_iterative_redirty: Vec::new(),
3118            retained_scc_members: FxHashMap::default(),
3119            next_retained_scc_id: 0,
3120            retained_scc_config_fingerprint: 0,
3121            retained_scc_function_epoch_seen: 0,
3122            retained_scc_provider_revision_seen: None,
3123            retained_scc_dirty_at_begin: Vec::new(),
3124            iterative_state_values: FxHashMap::default(),
3125            function_semantic_epoch_seen: crate::function_registry::semantic_epoch(),
3126            function_provider_revision_seen,
3127            #[cfg(test)]
3128            last_formula_plane_span_eval_report: None,
3129            #[cfg(test)]
3130            evaluation_request_begin_count_for_test: 0,
3131            #[cfg(any(test, feature = "test-support"))]
3132            before_prepared_span_commit_hook: None,
3133            #[cfg(test)]
3134            before_target_preparation_commit_hook: None,
3135            #[cfg(test)]
3136            before_target_planning_snapshot_hook: None,
3137            #[cfg(test)]
3138            inject_target_semantic_stale_once_for_test: false,
3139            #[cfg(test)]
3140            force_virtual_dep_changes_remaining_for_test: 0,
3141            #[cfg(test)]
3142            fail_evaluation_commit_preflight_once_for_test: false,
3143            #[cfg(test)]
3144            cancel_before_formula_plane_layer_commit_once_for_test: false,
3145            #[cfg(test)]
3146            force_source_family_fallback: false,
3147            #[cfg(test)]
3148            rerecord_cycle_retry_span_after_lease_extension_for_test: false,
3149            #[cfg(test)]
3150            fragmented_commit_fault_for_test: None,
3151            #[cfg(test)]
3152            formula_span_demotion_fault_for_test: None,
3153            #[cfg(test)]
3154            target_preparation_fault_for_test: None,
3155            #[cfg(test)]
3156            force_non_cycle_schedule_fallback_for_test: false,
3157            #[cfg(test)]
3158            mixed_topology_index_builds_for_test: 0,
3159            #[cfg(test)]
3160            before_legacy_fallback_final_provider_sample_hook: None,
3161            #[cfg(test)]
3162            after_eager_proposal_commit_hook: None,
3163        };
3164        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
3165        engine.config.arrow_storage_enabled = true;
3166        engine.config.delta_overlay_enabled = true;
3167        engine.config.write_formula_overlay_enabled = true;
3168        let default_sheet = engine.graph.default_sheet_name().to_string();
3169        engine.ensure_arrow_sheet(&default_sheet);
3170        engine
3171    }
3172
3173    pub fn workbook_load_limits(&self) -> &crate::engine::WorkbookLoadLimits {
3174        &self.workbook_load_limits
3175    }
3176
3177    pub fn set_workbook_load_limits(&mut self, limits: crate::engine::WorkbookLoadLimits) {
3178        self.workbook_load_limits = limits;
3179    }
3180
3181    fn clear_source_cache(&self) {
3182        if let Ok(mut g) = self.source_cache.write() {
3183            *g = SourceCache::default();
3184        }
3185    }
3186
3187    pub fn last_virtual_dep_telemetry(&self) -> &VirtualDepTelemetry {
3188        &self.last_virtual_dep_telemetry
3189    }
3190
3191    /// Telemetry from Runtime SCC evaluation during the most recent
3192    /// evaluation request (always default-zero under `CycleDetection::Static`
3193    /// or when `enable_virtual_dep_telemetry` is off).
3194    pub fn last_cycle_telemetry(&self) -> &CycleTelemetry {
3195        &self.last_cycle_telemetry
3196    }
3197
3198    /// Resource observations for the most recently completed public evaluation request.
3199    pub fn last_evaluation_resource_request_stats(
3200        &self,
3201    ) -> Option<&EvaluationResourceRequestStats> {
3202        self.last_evaluation_resource_request.as_ref()
3203    }
3204
3205    /// Cumulative resource observations since engine creation or the last telemetry reset.
3206    pub fn evaluation_resource_baseline_stats(&self) -> EvaluationResourceBaselineStats {
3207        self.evaluation_resource_baseline
3208    }
3209
3210    pub fn evaluation_resource_budgets(&self) -> &crate::engine::EvaluationBudgets {
3211        &self.evaluation_resource_budgets
3212    }
3213
3214    /// At most one diagnostic is emitted for deprecated resource fields.
3215    pub fn evaluation_resource_config_diagnostic(
3216        &self,
3217    ) -> Option<&crate::engine::EvaluationResourceConfigDiagnostic> {
3218        self.evaluation_resource_config_diagnostic.as_ref()
3219    }
3220
3221    /// Reset accumulated and last-request observations without reusing request IDs.
3222    pub fn reset_evaluation_resource_telemetry(&mut self) {
3223        self.evaluation_resource_baseline = EvaluationResourceBaselineStats::default();
3224        self.last_evaluation_resource_request = None;
3225    }
3226
3227    fn duration_ns(duration: std::time::Duration) -> u64 {
3228        u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
3229    }
3230
3231    fn observe_evaluation_resource_request<T>(
3232        &mut self,
3233        kind: EvaluationRequestKind,
3234        evaluate: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3235    ) -> Result<T, ExcelError> {
3236        let outermost = self.evaluation_resource_request_depth == 0;
3237        if outermost {
3238            let request_id = self.next_evaluation_resource_request_id;
3239            self.next_evaluation_resource_request_id = request_id
3240                .checked_add(1)
3241                .expect("evaluation resource request ID exhausted");
3242            self.active_evaluation_resource_request = Some(EvaluationResourceRequestStats::new(
3243                request_id,
3244                kind,
3245                self.config.formula_plane_mode,
3246                self.staged_formula_count(),
3247            ));
3248            self.evaluation_resource_baseline.record_started(request_id);
3249            self.evaluation_resource_request_started_at = Some(crate::instant::FzInstant::now());
3250            self.active_resource_ledger = Some(ResourceLedger::new(
3251                Some(request_id),
3252                self.evaluation_resource_budgets.clone(),
3253            ));
3254        }
3255        self.evaluation_resource_request_depth =
3256            self.evaluation_resource_request_depth.saturating_add(1);
3257        let result = if outermost {
3258            self.resource_checkpoint(0).and_then(|()| evaluate(self))
3259        } else {
3260            evaluate(self)
3261        };
3262        self.evaluation_resource_request_depth =
3263            self.evaluation_resource_request_depth.saturating_sub(1);
3264
3265        if outermost {
3266            let total_ns = self
3267                .evaluation_resource_request_started_at
3268                .take()
3269                .map(|start| Self::duration_ns(start.elapsed()))
3270                .unwrap_or(0);
3271            let mut stats = self
3272                .active_evaluation_resource_request
3273                .take()
3274                .expect("outer evaluation resource request has active stats");
3275            let mut ledger = self
3276                .active_resource_ledger
3277                .take()
3278                .expect("outer evaluation resource request has active ledger");
3279            ledger.release_all_scratch();
3280            stats.ledger.update(ledger.snapshot());
3281            stats.outcome = match &result {
3282                Ok(_) => EvaluationRequestOutcome::Success,
3283                Err(error) if error.kind == ExcelErrorKind::Cancelled => {
3284                    EvaluationRequestOutcome::Cancelled
3285                }
3286                Err(_) => EvaluationRequestOutcome::Error,
3287            };
3288            if stats.dirty_lease == FormulaDirtyLeaseOutcome::Acquired {
3289                stats.dirty_lease = if stats.outcome == EvaluationRequestOutcome::Cancelled {
3290                    FormulaDirtyLeaseOutcome::RetainedOnCancellation
3291                } else {
3292                    FormulaDirtyLeaseOutcome::RetainedOnError
3293                };
3294            }
3295            stats.phases.total_ns = total_ns;
3296            let attributed = stats
3297                .phases
3298                .staged_prepare_ns
3299                .saturating_add(stats.phases.topology_ns)
3300                .saturating_add(stats.phases.materialization_ns);
3301            stats.phases.evaluation_ns = total_ns.saturating_sub(attributed);
3302            self.evaluation_resource_baseline.record_finished(&stats);
3303            self.last_evaluation_resource_request = Some(stats);
3304        }
3305        result
3306    }
3307
3308    pub fn set_evaluation_resource_budgets(&mut self, budgets: crate::engine::EvaluationBudgets) {
3309        self.evaluation_resource_budgets = budgets.clone();
3310        self.config.evaluation_budgets = budgets.clone();
3311        self.graph.set_evaluation_budgets(budgets);
3312    }
3313
3314    #[cfg(test)]
3315    pub(crate) fn set_evaluation_budgets_for_test(
3316        &mut self,
3317        budgets: crate::engine::EvaluationBudgets,
3318    ) {
3319        self.set_evaluation_resource_budgets(budgets);
3320    }
3321
3322    fn resource_loop_checkpoint(
3323        ledger: &mut Option<ResourceLedger>,
3324        work_units: u64,
3325    ) -> Result<(), ExcelError> {
3326        ledger
3327            .as_mut()
3328            .map_or(Ok(()), |ledger| {
3329                ledger
3330                    .charge_work(work_units)
3331                    .and_then(|()| ledger.checkpoint_deadline())
3332            })
3333            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3334    }
3335
3336    fn preflight_evaluation_commit_window(
3337        &mut self,
3338        bounded_writes: usize,
3339    ) -> Result<crate::instant::FzInstant, ExcelError> {
3340        #[cfg(test)]
3341        if std::mem::take(&mut self.fail_evaluation_commit_preflight_once_for_test) {
3342            return Err(crate::engine::ResourceLedgerError::Exhausted(
3343                formualizer_common::ResourceExhaustionDetail {
3344                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3345                    limit: 0,
3346                    observed: 1,
3347                    request_id: self
3348                        .active_evaluation_resource_request
3349                        .as_ref()
3350                        .map(|stats| stats.request_id),
3351                },
3352            )
3353            .into_excel_error());
3354        }
3355        let estimate = std::time::Duration::from_nanos(
3356            u64::try_from(bounded_writes)
3357                .unwrap_or(u64::MAX)
3358                .saturating_mul(100),
3359        );
3360        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3361            ledger
3362                .preflight_commit_window(estimate)
3363                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
3364        }
3365        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3366            stats.evaluation_commit_preflight_count =
3367                stats.evaluation_commit_preflight_count.saturating_add(1);
3368            stats.evaluation_commit_estimated_ns = stats
3369                .evaluation_commit_estimated_ns
3370                .saturating_add(Self::duration_ns(estimate));
3371        }
3372        Ok(crate::instant::FzInstant::now())
3373    }
3374
3375    fn observe_evaluation_commit_window(&mut self, started: crate::instant::FzInstant) {
3376        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3377            stats.evaluation_commit_actual_ns = stats
3378                .evaluation_commit_actual_ns
3379                .saturating_add(Self::duration_ns(started.elapsed()));
3380        }
3381    }
3382
3383    fn cancellation_checkpoint(&self, message: &'static str) -> Result<(), ExcelError> {
3384        if self
3385            .active_cancel_flag
3386            .as_ref()
3387            .is_some_and(|cancel| cancel.is_cancelled())
3388        {
3389            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(message));
3390        }
3391        if self
3392            .active_evaluation_deadline
3393            .is_some_and(|deadline| Instant::now() >= deadline)
3394        {
3395            return Err(crate::engine::ResourceLedgerError::Exhausted(
3396                formualizer_common::ResourceExhaustionDetail {
3397                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
3398                    limit: 0,
3399                    observed: 1,
3400                    request_id: self
3401                        .active_evaluation_resource_request
3402                        .as_ref()
3403                        .map(|request| request.request_id),
3404                },
3405            )
3406            .into_excel_error()
3407            .with_message(message));
3408        }
3409        Ok(())
3410    }
3411
3412    fn resource_checkpoint(&mut self, work_units: u64) -> Result<(), ExcelError> {
3413        let Some(ledger) = self.active_resource_ledger.as_mut() else {
3414            return Ok(());
3415        };
3416        ledger
3417            .charge_work(work_units)
3418            .and_then(|()| ledger.checkpoint_deadline())
3419            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3420    }
3421
3422    fn charge_bounded_work(&mut self, mut work_units: u64) -> Result<(), ExcelError> {
3423        if work_units == 0 {
3424            return self.resource_checkpoint(0);
3425        }
3426        while work_units > 0 {
3427            let chunk = work_units.min(256);
3428            self.resource_checkpoint(chunk)?;
3429            work_units -= chunk;
3430        }
3431        Ok(())
3432    }
3433
3434    fn reserve_request_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3435        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3436            // Exact request topology and activation of the scratch cap are C1b. C1a records
3437            // scoped ownership but must not introduce a new skip or terminal path.
3438            ledger.observe_scratch(bytes);
3439        }
3440        Ok(())
3441    }
3442
3443    fn release_request_scratch(&mut self, bytes: u64) {
3444        if let Some(ledger) = self.active_resource_ledger.as_mut() {
3445            let released = ledger.release_scratch(bytes);
3446            debug_assert!(
3447                released.is_ok(),
3448                "request scratch release exceeded the outstanding reservation"
3449            );
3450        }
3451    }
3452
3453    fn can_reserve_topology_scratch(&self, bytes: u64) -> bool {
3454        self.active_resource_ledger
3455            .as_ref()
3456            .is_none_or(|ledger| ledger.can_reserve_schedule_discovery(bytes))
3457    }
3458
3459    fn reserve_topology_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3460        self.active_resource_ledger
3461            .as_mut()
3462            .map_or(Ok(()), |ledger| ledger.reserve_schedule_discovery(bytes))
3463            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3464    }
3465
3466    fn reserve_graph_source_scratch(&mut self, bytes: u64) -> Result<(), ExcelError> {
3467        self.active_resource_ledger
3468            .as_mut()
3469            .map_or(Ok(()), |ledger| ledger.reserve_graph_source(bytes))
3470            .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3471    }
3472
3473    fn with_request_scratch<T>(
3474        &mut self,
3475        bytes: u64,
3476        work: impl FnOnce(&mut Self) -> Result<T, ExcelError>,
3477    ) -> Result<T, ExcelError> {
3478        self.reserve_request_scratch(bytes)?;
3479        let result = work(self);
3480        self.release_request_scratch(bytes);
3481        result
3482    }
3483
3484    fn observe_staged_preparation(
3485        &mut self,
3486        selected: usize,
3487        retained: usize,
3488        elapsed: std::time::Duration,
3489    ) {
3490        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3491            stats.staged_selected = stats.staged_selected.saturating_add(selected as u64);
3492            stats.staged_retained = retained as u64;
3493            stats.phases.staged_prepare_ns = stats
3494                .phases
3495                .staged_prepare_ns
3496                .saturating_add(Self::duration_ns(elapsed));
3497        }
3498    }
3499
3500    fn graph_admission_enabled(&self) -> bool {
3501        crate::engine::resource_ledger::graph_admission_enabled(&self.evaluation_resource_budgets)
3502    }
3503
3504    fn preflight_graph_admission(
3505        &mut self,
3506        usage: crate::engine::resource_ledger::GraphAdmission,
3507    ) -> Result<(), ExcelError> {
3508        let request_id = self
3509            .active_evaluation_resource_request
3510            .as_ref()
3511            .map(|stats| stats.request_id);
3512        crate::engine::resource_ledger::preflight_graph_admission(
3513            &self.evaluation_resource_budgets,
3514            usage,
3515            request_id,
3516        )
3517        .map_err(crate::engine::ResourceLedgerError::into_excel_error)
3518    }
3519
3520    fn prepared_legacy_admission(
3521        &mut self,
3522        plan: &PreparedLegacyGraphPlan,
3523        materialization_cells: u64,
3524    ) -> Result<(), ExcelError> {
3525        if !self.graph_admission_enabled() {
3526            return Ok(());
3527        }
3528        let stats = self.graph.baseline_stats();
3529        let added_edges = plan.planned_edge_count().ok_or_else(|| {
3530            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3531        })?;
3532        let removed_edges = plan.removed_edge_count().ok_or_else(|| {
3533            ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3534        })?;
3535        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
3536            final_vertices: stats
3537                .graph_vertex_count
3538                .checked_add(plan.new_vertex_count())
3539                .ok_or_else(|| {
3540                    ExcelError::new(ExcelErrorKind::NImpl)
3541                        .with_message("graph vertex count overflow")
3542                })?,
3543            final_edges: stats
3544                .graph_edge_count
3545                .checked_sub(removed_edges)
3546                .and_then(|count| count.checked_add(added_edges))
3547                .ok_or_else(|| {
3548                    ExcelError::new(ExcelErrorKind::NImpl).with_message("graph edge count overflow")
3549                })?,
3550            materialization_cells,
3551            added_vertices: plan.new_vertex_count(),
3552            added_edges,
3553        })
3554    }
3555
3556    fn observe_target_admission_failure(
3557        &mut self,
3558        reason: formualizer_common::ResourceExhaustionReason,
3559    ) {
3560        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3561            stats.target_admission_failure = Some(reason);
3562        }
3563    }
3564
3565    fn observe_target_preparation_report(
3566        &mut self,
3567        report: &crate::engine::PreparedTargetGraphReport,
3568    ) {
3569        let reason_bit = |reason: crate::engine::OpaqueReason| -> u64 {
3570            let index = match reason {
3571                crate::engine::OpaqueReason::DynamicReference => 0,
3572                crate::engine::OpaqueReason::RuntimeTextReference => 1,
3573                crate::engine::OpaqueReason::UnknownFunction => 2,
3574                crate::engine::OpaqueReason::UnknownCustomFunction => 3,
3575                crate::engine::OpaqueReason::UnresolvedCrossSheetBinding => 4,
3576                crate::engine::OpaqueReason::UnresolvedName => 5,
3577                crate::engine::OpaqueReason::UnresolvedTable => 6,
3578                crate::engine::OpaqueReason::FormulaName => 7,
3579                crate::engine::OpaqueReason::DeferredSourcePackage => 8,
3580                crate::engine::OpaqueReason::UnsupportedSourceSemantics => 9,
3581                crate::engine::OpaqueReason::UncertainDefaultSheetBinding => 10,
3582            };
3583            1u64 << index
3584        };
3585        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3586            stats.staged_selected = report.selected_staged_cells as u64;
3587            stats.staged_retained = report.retained_staged_cells as u64;
3588            stats.target_requested = report.requested_targets as u64;
3589            stats.target_normalized_regions = report.normalized_regions as u64;
3590            stats.target_scope_level = match &report.widened_scope {
3591                crate::engine::PrepareScope::Exact => 0,
3592                crate::engine::PrepareScope::Sheets(_) => 1,
3593                crate::engine::PrepareScope::Workbook => 2,
3594            };
3595            stats.target_widening_reason_bits = report
3596                .widening_reasons
3597                .iter()
3598                .copied()
3599                .fold(0, |bits, reason| bits | reason_bit(reason));
3600            stats.graph_source_scratch_estimated = report.estimated_scratch_bytes;
3601            stats.graph_source_scratch_observed = report.observed_scratch_bytes;
3602            stats.target_commit_estimated_work = report.estimated_commit_work;
3603            stats.target_commit_actual_work = report.actual_commit_work;
3604            stats.target_commit_window_ns = Self::duration_ns(report.commit_window);
3605            stats.phases.staged_prepare_ns = stats
3606                .phases
3607                .staged_prepare_ns
3608                .saturating_add(Self::duration_ns(report.commit_window));
3609        }
3610    }
3611
3612    fn observe_topology(
3613        &mut self,
3614        outcome: FormulaPlaneTopologyCacheOutcome,
3615        strategy: FormulaPlaneTopologyStrategy,
3616        observed: &MixedTopologyCompileStats,
3617        elapsed: std::time::Duration,
3618    ) {
3619        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3620            if outcome.severity() > stats.topology.cache_outcome.severity() {
3621                stats.topology.cache_outcome = outcome;
3622            }
3623            if strategy.severity() > stats.topology.strategy.severity() {
3624                stats.topology.strategy = strategy;
3625            }
3626            match outcome {
3627                FormulaPlaneTopologyCacheOutcome::Hit => {
3628                    stats.topology.cache_hit_events =
3629                        stats.topology.cache_hit_events.saturating_add(1);
3630                }
3631                FormulaPlaneTopologyCacheOutcome::Built => {
3632                    stats.topology.cache_build_events =
3633                        stats.topology.cache_build_events.saturating_add(1);
3634                }
3635                FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3636                | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy => {
3637                    stats.topology.cache_build_events =
3638                        stats.topology.cache_build_events.saturating_add(1);
3639                    stats.topology.cache_skip_events =
3640                        stats.topology.cache_skip_events.saturating_add(1);
3641                }
3642                FormulaPlaneTopologyCacheOutcome::NotUsed => {}
3643            }
3644            if matches!(
3645                outcome,
3646                FormulaPlaneTopologyCacheOutcome::Built
3647                    | FormulaPlaneTopologyCacheOutcome::SkippedOverflow
3648                    | FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy
3649            ) {
3650                stats.topology.producers_observed = stats
3651                    .topology
3652                    .producers_observed
3653                    .saturating_add(observed.producers as u64);
3654                stats.topology.candidates_observed = stats
3655                    .topology
3656                    .candidates_observed
3657                    .saturating_add(observed.candidates as u64);
3658                stats.topology.edges_observed = stats
3659                    .topology
3660                    .edges_observed
3661                    .saturating_add(observed.relationships as u64);
3662                stats.topology.candidate_cap_hits = stats
3663                    .topology
3664                    .candidate_cap_hits
3665                    .saturating_add(observed.candidate_overflow_count as u64);
3666                stats.topology.edge_cap_hits = stats
3667                    .topology
3668                    .edge_cap_hits
3669                    .saturating_add(observed.edge_overflow_count as u64);
3670                stats.topology.byte_cap_hits = stats
3671                    .topology
3672                    .byte_cap_hits
3673                    .saturating_add(observed.memory_overflow_count as u64);
3674                stats.topology.overflow_reason = if stats.topology.byte_cap_hits > 0 {
3675                    Some(EvaluationResourceReason::FormulaPlaneTopologyRetainedBytes)
3676                } else if stats.topology.edge_cap_hits > 0 {
3677                    Some(EvaluationResourceReason::FormulaPlaneTopologyEdges)
3678                } else if stats.topology.candidate_cap_hits > 0 {
3679                    Some(EvaluationResourceReason::FormulaPlaneTopologyCandidates)
3680                } else {
3681                    None
3682                };
3683                stats.topology.incomplete_reason = if stats.topology.byte_cap_hits > 0 {
3684                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyRetainedBytes)
3685                } else if stats.topology.edge_cap_hits > 0 {
3686                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyEdges)
3687                } else if stats.topology.candidate_cap_hits > 0 {
3688                    Some(crate::engine::EvaluationIncompleteReason::FormulaPlaneTopologyCandidates)
3689                } else {
3690                    None
3691                };
3692            }
3693            stats.topology.retained_bytes_observed = stats
3694                .topology
3695                .retained_bytes_observed
3696                .max(observed.estimated_memory_bytes as u64);
3697            stats.phases.topology_ns = stats
3698                .phases
3699                .topology_ns
3700                .saturating_add(Self::duration_ns(elapsed));
3701        }
3702    }
3703
3704    fn observe_topology_strategy(&mut self, strategy: FormulaPlaneTopologyStrategy) {
3705        if let Some(stats) = self.active_evaluation_resource_request.as_mut()
3706            && strategy.severity() > stats.topology.strategy.severity()
3707        {
3708            stats.topology.strategy = strategy;
3709        }
3710    }
3711
3712    fn observe_formula_plane_route(
3713        &mut self,
3714        plan: &LegacyIslandPlan,
3715        phase: FormulaPlaneRoutePhase,
3716        reason: FormulaPlaneRouteTransitionReason,
3717        demotion_generation: usize,
3718        replan_generation: usize,
3719    ) {
3720        if plan.is_empty() {
3721            return;
3722        }
3723        let mut sheet_ids = [0_u16;
3724            crate::engine::resource_observability::FORMULA_PLANE_ROUTE_EVENT_SHEET_CAPACITY];
3725        let sheet_count = plan.sheet_ids.len().min(sheet_ids.len());
3726        sheet_ids[..sheet_count].copy_from_slice(&plan.sheet_ids[..sheet_count]);
3727        let authority = self.graph.formula_authority();
3728        let event = FormulaPlaneRouteEvent {
3729            island_id: plan.island_id,
3730            phase,
3731            route: FormulaPlaneRoute::ContractedLegacyIsland,
3732            sheet_ids,
3733            sheet_count: sheet_count as u8,
3734            authority_epoch: authority.plane.epoch().0,
3735            index_epoch: authority.indexes_epoch(),
3736            transition_reason: reason,
3737            demotion_generation: demotion_generation.min(u32::MAX as usize) as u32,
3738            replan_generation: replan_generation.min(u32::MAX as usize) as u32,
3739        };
3740        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3741            let index = usize::from(stats.topology.route_event_count);
3742            if index < stats.topology.route_events.len() {
3743                stats.topology.route_events[index] = event;
3744                stats.topology.route_event_count =
3745                    stats.topology.route_event_count.saturating_add(1);
3746            } else {
3747                stats.topology.route_events_dropped =
3748                    stats.topology.route_events_dropped.saturating_add(1);
3749            }
3750            stats.topology.route_event_bytes_observed = stats
3751                .topology
3752                .route_event_bytes_observed
3753                .saturating_add(std::mem::size_of::<FormulaPlaneRouteEvent>() as u64);
3754            stats.topology.island_membership_vertices = plan.membership.len() as u64;
3755            stats.topology.island_membership_retained_bytes = plan.retained_bytes as u64;
3756            stats.topology.legacy_relationships_omitted = plan.omitted_relationships as u64;
3757            stats.topology.boundary_relationships_retained = plan.boundary_relationships as u64;
3758        }
3759    }
3760
3761    fn observe_materialization(
3762        &mut self,
3763        placements: usize,
3764        cycle: bool,
3765        elapsed: std::time::Duration,
3766    ) {
3767        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3768            if cycle {
3769                stats.cycle_materialized_cells = stats
3770                    .cycle_materialized_cells
3771                    .saturating_add(placements as u64);
3772            } else {
3773                stats.fallback_materialized_cells = stats
3774                    .fallback_materialized_cells
3775                    .saturating_add(placements as u64);
3776                stats.topology.strategy =
3777                    FormulaPlaneTopologyStrategy::CapacityFallbackMaterialization;
3778            }
3779            stats.phases.materialization_ns = stats
3780                .phases
3781                .materialization_ns
3782                .saturating_add(Self::duration_ns(elapsed));
3783        }
3784    }
3785
3786    fn observe_dirty_lease_acquired(&mut self, empty: bool) {
3787        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3788            stats.dirty_lease = if empty {
3789                FormulaDirtyLeaseOutcome::Empty
3790            } else {
3791                FormulaDirtyLeaseOutcome::Acquired
3792            };
3793        }
3794    }
3795
3796    fn ack_formula_dirty_observed(&mut self, lease: FormulaDirtyLease) {
3797        let empty = lease.is_empty();
3798        let _ = self.graph.ack_formula_dirty(lease);
3799        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3800            stats.dirty_lease = if empty {
3801                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3802            } else {
3803                FormulaDirtyLeaseOutcome::Acknowledged
3804            };
3805        }
3806    }
3807
3808    fn checked_ack_formula_dirty_observed(
3809        &mut self,
3810        lease: FormulaDirtyLease,
3811    ) -> Result<(), ExcelError> {
3812        self.resource_checkpoint(0)?;
3813        self.ack_formula_dirty_observed(lease);
3814        Ok(())
3815    }
3816
3817    fn checked_ack_formula_dirty_sublease_observed(
3818        &mut self,
3819        lease: FormulaDirtyLease,
3820        owned_events: &[usize],
3821    ) -> Result<(), ExcelError> {
3822        self.resource_checkpoint(owned_events.len() as u64)?;
3823        let empty = lease.is_empty();
3824        let partial = owned_events.len() < lease.len();
3825        let sublease = lease.sublease(owned_events.iter().copied());
3826        if !self.graph.ack_formula_dirty_sublease(sublease) {
3827            return Err(ExcelError::new(ExcelErrorKind::NImpl)
3828                .with_message("FormulaPlane dirty sublease became stale"));
3829        }
3830        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
3831            stats.dirty_lease = if empty {
3832                FormulaDirtyLeaseOutcome::AcknowledgedEmpty
3833            } else if partial {
3834                FormulaDirtyLeaseOutcome::AcknowledgedPartial
3835            } else {
3836                FormulaDirtyLeaseOutcome::Acknowledged
3837            };
3838        }
3839        Ok(())
3840    }
3841
3842    /// Begin a new evaluation request: reset per-recalc cycle telemetry and
3843    /// take the per-recalc volatile clock sample. Called at the start of
3844    /// every evaluation request that walks schedule units.
3845    fn begin_evaluation_request(&mut self) {
3846        #[cfg(test)]
3847        {
3848            self.evaluation_request_begin_count_for_test = self
3849                .evaluation_request_begin_count_for_test
3850                .saturating_add(1);
3851        }
3852        self.last_cycle_telemetry = CycleTelemetry::default();
3853        // Defensive: consumed at the end of the previous request; a request
3854        // that errored out mid-walk must not leak its members into this one.
3855        self.pending_iterative_redirty.clear();
3856        self.reconcile_retained_sccs_at_request_begin();
3857        // Spec §7.11: NOW()/TODAY() sample the clock ONCE per recalc; every
3858        // read within this request (including SCC iteration passes) observes
3859        // this sample.
3860        self.clock.refresh();
3861    }
3862
3863    /// End-of-recalc redirty: volatile vertices (as always) plus members of
3864    /// SCCs that iterated this recalc without reaching a retainable fixed
3865    /// point (`CyclePolicy::Iterate`), so circular cells re-evaluate on every
3866    /// recalc exactly like Excel's iterative calculation (spec §4
3867    /// persistence / §7.6 accumulator / §7.11 volatile redirty). Retained
3868    /// SCCs (`retained_scc_members`, #368) are left clean. Replaces the bare
3869    /// `graph.redirty_volatiles()` call at every evaluation-flow exit; must
3870    /// run AFTER the flow's `clear_dirty_flags`.
3871    fn redirty_for_next_recalc(&mut self) {
3872        self.graph.redirty_volatiles();
3873        let pending = std::mem::take(&mut self.pending_iterative_redirty);
3874        let dirty_at_begin = std::mem::take(&mut self.retained_scc_dirty_at_begin);
3875        for (vertex, scc) in dirty_at_begin {
3876            if self.retained_scc_members.get(&vertex) == Some(&scc) {
3877                // No SCC task claimed this member during the request: the
3878                // cycle dissolved and it evaluated as an ordinary formula,
3879                // or the request never reached it. Either way it is no
3880                // longer a retained fixed point. Its persisted value is
3881                // obsolete only if it actually re-evaluated (clean now).
3882                self.retained_scc_members.remove(&vertex);
3883                if !self.graph.is_dirty(vertex) {
3884                    self.iterative_state_values.remove(&vertex);
3885                }
3886            }
3887        }
3888        if !self.iterative_state_values.is_empty() || !self.retained_scc_members.is_empty() {
3889            let graph = &self.graph;
3890            self.iterative_state_values
3891                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3892            self.retained_scc_members
3893                .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3894        }
3895        // Refresh the §4-persistence snapshot for members that re-run each
3896        // recalc: these final values survive structural edits that clear the
3897        // computed overlay (the only value home in canonical mode) so the
3898        // next SCC task can re-seed from them (see `iterative_state_values`).
3899        for &vertex in &pending {
3900            if !self.graph.is_live_formula_vertex(vertex) {
3901                continue;
3902            }
3903            if let Some(cell) = self.graph.get_cell_ref(vertex) {
3904                let sheet_name = self.graph.sheet_name(cell.sheet_id);
3905                match self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1) {
3906                    Some(value) if !matches!(value, LiteralValue::Empty) => {
3907                        self.iterative_state_values.insert(vertex, value);
3908                    }
3909                    _ => {
3910                        self.iterative_state_values.remove(&vertex);
3911                    }
3912                }
3913            }
3914        }
3915        if !pending.is_empty() {
3916            self.graph.redirty_iterative_members(&pending);
3917        }
3918    }
3919
3920    /// Hash of every `EvalConfig` knob that can change the result of a
3921    /// retained SCC without any edit reaching the dependency graph. The
3922    /// function registry is tracked separately and precisely (see
3923    /// `retained_scc_function_epoch_seen`).
3924    fn retained_scc_config_fingerprint(&self) -> u64 {
3925        use std::hash::{Hash, Hasher};
3926        let mut hasher = rustc_hash::FxHasher::default();
3927        let config = &self.config;
3928        std::mem::discriminant(&config.cycle.detection).hash(&mut hasher);
3929        match config.cycle.policy {
3930            CyclePolicy::Error => 0u8.hash(&mut hasher),
3931            CyclePolicy::Iterate {
3932                max_iterations,
3933                max_change,
3934            } => {
3935                1u8.hash(&mut hasher);
3936                max_iterations.hash(&mut hasher);
3937                max_change.to_bits().hash(&mut hasher);
3938            }
3939        }
3940        config.date_system.hash(&mut hasher);
3941        config.workbook_seed.hash(&mut hasher);
3942        std::mem::discriminant(&config.volatile_level).hash(&mut hasher);
3943        // `DeterministicMode` carries a timestamp and a timezone spec; hash
3944        // its Debug rendering rather than growing its derive set for this.
3945        format!("{:?}", config.deterministic_mode).hash(&mut hasher);
3946        config.range_expansion_limit.hash(&mut hasher);
3947        config.max_open_ended_rows.hash(&mut hasher);
3948        config.max_open_ended_cols.hash(&mut hasher);
3949        hasher.finish()
3950    }
3951
3952    /// Request-begin bookkeeping for retained SCCs (#368): drop deleted
3953    /// vertices, invalidate everything when the config fingerprint moved
3954    /// (marking retained members dirty so their SCC tasks run in this
3955    /// request), and record how many retained SCCs are being reused, i.e.
3956    /// have no dirty member at request begin.
3957    fn reconcile_retained_sccs_at_request_begin(&mut self) {
3958        self.retained_scc_dirty_at_begin.clear();
3959        if self.retained_scc_members.is_empty() {
3960            return;
3961        }
3962        // Deleted vertices and members overwritten with a literal (the
3963        // vertex survives as a value cell) are no longer retained formulas.
3964        let graph = &self.graph;
3965        self.retained_scc_members
3966            .retain(|vertex, _| graph.is_live_formula_vertex(*vertex));
3967        if self.retained_scc_config_fingerprint() != self.retained_scc_config_fingerprint {
3968            let members: Vec<VertexId> = self.retained_scc_members.keys().copied().collect();
3969            self.retained_scc_members.clear();
3970            self.graph.mark_dirty_many(&members);
3971            return;
3972        }
3973        let changes =
3974            crate::function_registry::semantic_changes_since(self.retained_scc_function_epoch_seen);
3975        let global_changed = changes.epoch != self.retained_scc_function_epoch_seen;
3976        let provider_revision = self.resolver.planning_semantic_revision();
3977        let provider_changed = provider_revision != self.retained_scc_provider_revision_seen;
3978        if global_changed || provider_changed {
3979            let changed: BTreeSet<(String, String)> = changes.keys.into_iter().collect();
3980            let affected: Vec<VertexId> = self
3981                .retained_scc_members
3982                .keys()
3983                .copied()
3984                .filter(|&vertex| {
3985                    let Some(ast) = self.graph.get_formula(vertex) else {
3986                        return true;
3987                    };
3988                    (global_changed
3989                        && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed)))
3990                        || (provider_changed && Self::ast_contains_function(&ast))
3991                })
3992                .collect();
3993            for vertex in &affected {
3994                self.retained_scc_members.remove(vertex);
3995            }
3996            if !affected.is_empty() {
3997                self.graph.mark_dirty_many(&affected);
3998            }
3999            self.retained_scc_function_epoch_seen = changes.epoch;
4000            self.retained_scc_provider_revision_seen = provider_revision;
4001        }
4002        let mut dirty_sccs: FxHashSet<u64> = FxHashSet::default();
4003        let mut all_sccs: FxHashSet<u64> = FxHashSet::default();
4004        for (&vertex, &scc) in &self.retained_scc_members {
4005            all_sccs.insert(scc);
4006            if self.graph.is_dirty(vertex) {
4007                dirty_sccs.insert(scc);
4008                self.retained_scc_dirty_at_begin.push((vertex, scc));
4009            }
4010        }
4011        let reused_members = self
4012            .retained_scc_members
4013            .values()
4014            .filter(|scc| !dirty_sccs.contains(scc))
4015            .count();
4016        let t = &mut self.last_cycle_telemetry;
4017        t.reused_sccs = all_sccs.len() - dirty_sccs.len();
4018        t.reused_scc_members = reused_members;
4019    }
4020
4021    pub fn virtual_dep_fallback_activations(&self) -> u64 {
4022        self.virtual_dep_fallback_activations
4023    }
4024
4025    pub(crate) fn last_lookup_index_cache_report(&self) -> LookupIndexCacheReport {
4026        self.lookup_index_cache.report()
4027    }
4028
4029    fn lookup_view_contains_volatile(&self, view: &RangeView<'_>, sheet_id: SheetId) -> bool {
4030        let start_row = view.start_row();
4031        let end_row = view.end_row();
4032        let start_col = view.start_col();
4033        let end_col = view.end_col();
4034        for row in start_row..=end_row {
4035            let Ok(row_u32) = u32::try_from(row) else {
4036                return true;
4037            };
4038            for col in start_col..=end_col {
4039                let Ok(col_u32) = u32::try_from(col) else {
4040                    return true;
4041                };
4042                let cell_ref = self
4043                    .graph
4044                    .make_cell_ref_internal(sheet_id, row_u32, col_u32);
4045                if let Some(vertex_id) = self.graph.get_vertex_id_for_address(&cell_ref)
4046                    && self.graph.is_volatile(*vertex_id)
4047                {
4048                    return true;
4049                }
4050            }
4051        }
4052        false
4053    }
4054
4055    fn build_lookup_index_impl(
4056        &self,
4057        view: &RangeView<'_>,
4058        axis: LookupAxis,
4059    ) -> Option<Arc<LookupIndex>> {
4060        let (rows, cols) = view.dims();
4061        if rows == 0 || cols == 0 {
4062            self.lookup_index_cache.note_skipped_tiny();
4063            return None;
4064        }
4065        let len = match axis {
4066            LookupAxis::ColumnInView(col) => {
4067                if col >= cols {
4068                    self.lookup_index_cache.note_skipped_tiny();
4069                    return None;
4070                }
4071                rows
4072            }
4073            LookupAxis::RowInView(row) => {
4074                if row >= rows {
4075                    self.lookup_index_cache.note_skipped_tiny();
4076                    return None;
4077                }
4078                cols
4079            }
4080        };
4081        if len < 64 {
4082            self.lookup_index_cache.note_skipped_tiny();
4083            return None;
4084        }
4085
4086        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
4087        let key = LookupIndexKey {
4088            sheet_id,
4089            start_row: u32::try_from(view.start_row()).ok()?,
4090            start_col: u32::try_from(view.start_col()).ok()?,
4091            end_row: u32::try_from(view.end_row()).ok()?,
4092            end_col: u32::try_from(view.end_col()).ok()?,
4093            axis,
4094            snapshot_id: self.data_snapshot_id(),
4095        };
4096        if let Some(index) = self.lookup_index_cache.get(&key) {
4097            return Some(index);
4098        }
4099        if self
4100            .lookup_index_cache
4101            .would_exceed_cap(estimate_bytes(len, 0))
4102        {
4103            self.lookup_index_cache.note_skipped_cap();
4104            return None;
4105        }
4106        if !self.lookup_index_cache.should_build(key) {
4107            return None;
4108        }
4109        if self.lookup_index_cache.is_known_volatile(&key) {
4110            self.lookup_index_cache.note_skipped_volatile();
4111            return None;
4112        }
4113        if self.lookup_view_contains_volatile(view, sheet_id) {
4114            self.lookup_index_cache.note_volatile_key(key);
4115            self.lookup_index_cache.note_skipped_volatile();
4116            return None;
4117        }
4118        match LookupIndex::build(view, axis, self.config.date_system).ok()? {
4119            BuildOutcome::Built(index) => self.lookup_index_cache.insert_if_room(key, index),
4120            BuildOutcome::ErrorInLookupAxis => {
4121                self.lookup_index_cache.note_skipped_error();
4122                None
4123            }
4124            BuildOutcome::Degenerate => {
4125                self.lookup_index_cache.note_skipped_tiny();
4126                None
4127            }
4128        }
4129    }
4130
4131    fn reset_virtual_dep_telemetry_if_disabled(&mut self) {
4132        if !self.config.enable_virtual_dep_telemetry {
4133            self.last_virtual_dep_telemetry = VirtualDepTelemetry {
4134                fallback_mode_activations: self.virtual_dep_fallback_activations,
4135                ..VirtualDepTelemetry::default()
4136            };
4137        }
4138    }
4139
4140    fn source_cache_session(&self) -> SourceCacheSession {
4141        self.clear_source_cache();
4142        SourceCacheSession {
4143            cache: self.source_cache.clone(),
4144        }
4145    }
4146
4147    fn resolve_source_scalar_cached(
4148        &self,
4149        name: &str,
4150        version: Option<u64>,
4151    ) -> Result<LiteralValue, ExcelError> {
4152        let key = (name.to_string(), version);
4153        if let Ok(mut g) = self.source_cache.write() {
4154            if let Some(v) = g.scalars.get(&key) {
4155                return Ok(v.clone());
4156            }
4157
4158            let v = self.resolver.resolve_source_scalar(name).map_err(|err| {
4159                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4160                    ExcelError::new(ExcelErrorKind::Ref)
4161                        .with_message(format!("Unresolved source scalar: {name}"))
4162                } else {
4163                    err
4164                }
4165            })?;
4166            g.scalars.insert(key, v.clone());
4167            Ok(v)
4168        } else {
4169            self.resolver.resolve_source_scalar(name).map_err(|err| {
4170                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4171                    ExcelError::new(ExcelErrorKind::Ref)
4172                        .with_message(format!("Unresolved source scalar: {name}"))
4173                } else {
4174                    err
4175                }
4176            })
4177        }
4178    }
4179
4180    fn resolve_source_table_cached(
4181        &self,
4182        name: &str,
4183        version: Option<u64>,
4184    ) -> Result<Arc<dyn crate::traits::Table>, ExcelError> {
4185        let key = (name.to_string(), version);
4186        if let Ok(mut g) = self.source_cache.write() {
4187            if let Some(t) = g.tables.get(&key) {
4188                return Ok(t.clone());
4189            }
4190
4191            let t = self.resolver.resolve_source_table(name).map_err(|err| {
4192                if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4193                    ExcelError::new(ExcelErrorKind::Ref)
4194                        .with_message(format!("Unresolved source table: {name}"))
4195                } else {
4196                    err
4197                }
4198            })?;
4199            let t: Arc<dyn crate::traits::Table> = Arc::from(t);
4200            g.tables.insert(key, t.clone());
4201            Ok(t)
4202        } else {
4203            self.resolver
4204                .resolve_source_table(name)
4205                .map_err(|err| {
4206                    if matches!(err.kind, ExcelErrorKind::Name | ExcelErrorKind::NImpl) {
4207                        ExcelError::new(ExcelErrorKind::Ref)
4208                            .with_message(format!("Unresolved source table: {name}"))
4209                    } else {
4210                        err
4211                    }
4212                })
4213                .map(Arc::from)
4214        }
4215    }
4216
4217    fn source_table_to_range_view(
4218        &self,
4219        table: &dyn crate::traits::Table,
4220        spec: &Option<formualizer_parse::parser::TableSpecifier>,
4221    ) -> Result<RangeView<'static>, ExcelError> {
4222        use formualizer_parse::parser::{SpecialItem, TableSpecifier};
4223
4224        let owned = match spec {
4225            Some(TableSpecifier::Column(c)) => {
4226                let c = c.trim();
4227                if c == "@" || c.contains('[') || c.contains(']') || c.contains(',') {
4228                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4229                        "Complex structured references not yet supported".to_string(),
4230                    ));
4231                }
4232                table.get_column(c)?.materialise().into_owned()
4233            }
4234            Some(TableSpecifier::ColumnRange(start, end)) => {
4235                let cols = table.columns();
4236                let start = start.trim();
4237                let end = end.trim();
4238                let start_key = start.to_lowercase();
4239                let end_key = end.to_lowercase();
4240                let start_idx = cols.iter().position(|n| n.to_lowercase() == start_key);
4241                let end_idx = cols.iter().position(|n| n.to_lowercase() == end_key);
4242                if let (Some(mut si), Some(mut ei)) = (start_idx, end_idx) {
4243                    if si > ei {
4244                        std::mem::swap(&mut si, &mut ei);
4245                    }
4246                    let h = table.data_height();
4247                    let w = ei - si + 1;
4248                    let mut rows = vec![vec![LiteralValue::Empty; w]; h];
4249                    for (offset, ci) in (si..=ei).enumerate() {
4250                        let cname = &cols[ci];
4251                        let col_range = table.get_column(cname)?;
4252                        let (rh, _) = col_range.dimensions();
4253                        for (r, row) in rows.iter_mut().enumerate().take(h.min(rh)) {
4254                            row[offset] = col_range.get(r, 0)?;
4255                        }
4256                    }
4257                    rows
4258                } else {
4259                    return Err(ExcelError::new(ExcelErrorKind::Ref)
4260                        .with_message("Column range refers to unknown column(s)".to_string()));
4261                }
4262            }
4263            Some(TableSpecifier::SpecialItem(SpecialItem::Headers))
4264            | Some(TableSpecifier::Headers) => table
4265                .headers_row()
4266                .map(|r| r.materialise().into_owned())
4267                .unwrap_or_default(),
4268            Some(TableSpecifier::SpecialItem(SpecialItem::Totals))
4269            | Some(TableSpecifier::Totals) => table
4270                .totals_row()
4271                .map(|r| r.materialise().into_owned())
4272                .unwrap_or_default(),
4273            Some(TableSpecifier::SpecialItem(SpecialItem::Data)) | Some(TableSpecifier::Data) => {
4274                table
4275                    .data_body()
4276                    .map(|r| r.materialise().into_owned())
4277                    .unwrap_or_default()
4278            }
4279            Some(TableSpecifier::SpecialItem(SpecialItem::All)) | Some(TableSpecifier::All) => {
4280                let mut out: Vec<Vec<LiteralValue>> = Vec::new();
4281                if let Some(h) = table.headers_row() {
4282                    out.extend(h.iter_rows());
4283                }
4284                if let Some(body) = table.data_body() {
4285                    out.extend(body.iter_rows());
4286                }
4287                if let Some(tr) = table.totals_row() {
4288                    out.extend(tr.iter_rows());
4289                }
4290                out
4291            }
4292            Some(TableSpecifier::SpecialItem(SpecialItem::ThisRow)) => {
4293                return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
4294                    "@ (This Row) requires table-aware context; not yet supported".to_string(),
4295                ));
4296            }
4297            Some(TableSpecifier::Row(_)) | Some(TableSpecifier::Combination(_)) => {
4298                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4299                    .with_message("Complex structured references not yet supported".to_string()));
4300            }
4301            None => {
4302                return Err(ExcelError::new(ExcelErrorKind::NImpl)
4303                    .with_message("Table reference without specifier is unsupported".to_string()));
4304            }
4305        };
4306
4307        Ok(RangeView::from_owned_rows(owned, self.config.date_system))
4308    }
4309
4310    pub fn default_sheet_id(&self) -> SheetId {
4311        self.graph.default_sheet_id()
4312    }
4313
4314    pub fn default_sheet_name(&self) -> &str {
4315        self.graph.default_sheet_name()
4316    }
4317
4318    /// Update the workbook seed for deterministic RNGs in functions.
4319    pub fn set_workbook_seed(&mut self, seed: u64) {
4320        self.config.workbook_seed = seed;
4321    }
4322
4323    /// Set the volatile level policy (Always/OnRecalc/OnOpen)
4324    pub fn set_volatile_level(&mut self, level: crate::traits::VolatileLevel) {
4325        self.config.volatile_level = level;
4326    }
4327
4328    /// Set public temporal materialisation to native values or raw serials.
4329    pub fn set_temporal_egress(&mut self, policy: crate::engine::TemporalEgress) {
4330        self.config.temporal_egress = policy;
4331    }
4332
4333    pub fn temporal_egress(&self) -> crate::engine::TemporalEgress {
4334        self.config.temporal_egress
4335    }
4336
4337    /// Enable/disable deterministic evaluation mode (fixed clock + timezone).
4338    pub fn set_deterministic_mode(
4339        &mut self,
4340        mode: crate::engine::DeterministicMode,
4341    ) -> Result<(), ExcelError> {
4342        let clock = mode.build_clock()?;
4343        self.config.deterministic_mode = mode;
4344        self.clock = crate::timezone::SnapshotClock::new(clock);
4345        Ok(())
4346    }
4347
4348    /// Inject a custom [`ClockProvider`](crate::timezone::ClockProvider) for
4349    /// volatile date/time builtins (`NOW()`, `TODAY()`).
4350    ///
4351    /// The provider is the clock *source*; per spec §7.11 the engine samples
4352    /// it once at the start of every evaluation request and all reads within
4353    /// that recalc (including SCC iteration passes) observe the frozen
4354    /// sample.
4355    pub fn set_clock(&mut self, clock: Arc<dyn crate::timezone::ClockProvider>) {
4356        self.clock = crate::timezone::SnapshotClock::new(clock);
4357    }
4358
4359    fn validate_deterministic_mode(&self) -> Result<(), ExcelError> {
4360        self.config.deterministic_mode.validate()
4361    }
4362
4363    pub fn sheet_id(&self, name: &str) -> Option<SheetId> {
4364        self.graph.sheet_id(name)
4365    }
4366
4367    pub fn sheet_id_mut(&mut self, name: &str) -> SheetId {
4368        self.add_sheet(name)
4369            .unwrap_or_else(|_| self.graph.sheet_id_mut(name))
4370    }
4371
4372    pub fn sheet_name(&self, id: SheetId) -> &str {
4373        self.graph.sheet_name(id)
4374    }
4375
4376    pub fn add_sheet(&mut self, name: &str) -> Result<SheetId, ExcelError> {
4377        let id = self.graph.add_sheet(name)?;
4378        self.ensure_arrow_sheet(name);
4379        // Adding a sheet does not change any existing span result or dependency.
4380        self.mark_topology_edited();
4381        Ok(id)
4382    }
4383
4384    pub fn duplicate_sheet(&mut self, source: &str, new_name: &str) -> Result<SheetId, ExcelError> {
4385        let source_id = self.graph.sheet_id(source).ok_or_else(|| {
4386            ExcelError::new(ExcelErrorKind::Value).with_message("Source sheet does not exist")
4387        })?;
4388        if new_name.is_empty() || new_name.len() > 255 {
4389            return Err(ExcelError::new(ExcelErrorKind::Value).with_message("Invalid sheet name"));
4390        }
4391        if self.graph.sheet_id(new_name).is_some() {
4392            return Err(ExcelError::new(ExcelErrorKind::Value)
4393                .with_message(format!("Sheet '{new_name}' already exists")));
4394        }
4395        // Materialize only spans on the source sheet so graph duplication sees
4396        // the formulas being copied. Spans on unrelated sheets remain active.
4397        self.demote_spans_preserving_computed_overlays(source_id, Region::whole_sheet(source_id))
4398            .map_err(Self::editor_error_to_excel)?;
4399        let new_id = self.graph.duplicate_sheet(source_id, new_name)?;
4400
4401        if let Some(source_sheet) = self.arrow_sheets.sheet(source).cloned() {
4402            let mut copied_sheet = source_sheet;
4403            copied_sheet.name = Arc::<str>::from(new_name);
4404            self.arrow_sheets.sheets.push(copied_sheet);
4405        } else {
4406            self.ensure_arrow_sheet(new_name);
4407        }
4408
4409        let duplicated_formulas = self
4410            .graph
4411            .formula_vertices()
4412            .into_iter()
4413            .filter(|vertex| {
4414                self.graph
4415                    .get_cell_ref(*vertex)
4416                    .is_some_and(|cell| cell.sheet_id == new_id)
4417            })
4418            .collect::<Vec<_>>();
4419        self.graph.mark_vertices_dirty_batch(&duplicated_formulas);
4420        self.mark_topology_edited();
4421        Ok(new_id)
4422    }
4423
4424    fn ensure_arrow_sheet(&mut self, name: &str) {
4425        if self.arrow_sheets.sheet(name).is_some() {
4426            return;
4427        }
4428        self.arrow_sheets
4429            .sheets
4430            .push(crate::arrow_store::ArrowSheet {
4431                name: std::sync::Arc::<str>::from(name),
4432                date_system: self.config.date_system,
4433                columns: Vec::new(),
4434                nrows: 0,
4435                chunk_starts: Vec::new(),
4436                chunk_rows: 32 * 1024,
4437            });
4438    }
4439
4440    pub fn remove_sheet(&mut self, sheet_id: SheetId) -> Result<(), ExcelError> {
4441        let name = self.graph.sheet_name(sheet_id).to_string();
4442        // Removing a sheet only affects spans on that sheet and spans reading
4443        // from that sheet. Preserve spans on unrelated sheets so sheet
4444        // lifecycle operations do not collapse the whole FormulaPlane.
4445        self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
4446            .map_err(Self::editor_error_to_excel)?;
4447        self.purge_derived_formats_for_sheet(sheet_id);
4448        self.graph.remove_sheet(sheet_id)?;
4449        self.arrow_sheets.sheets.retain(|s| s.name.as_ref() != name);
4450        // Sheet removal can change cross-sheet refs, names, and default-sheet
4451        // resolution. Until those domains have a complete exact dependency
4452        // proof, retain the documented graph-owned global invalidation.
4453        self.clear_all_computed_overlays();
4454        self.mark_all_formula_vertices_dirty();
4455        self.clear_staged_formulas_for_sheet(&name);
4456        if self.row_visibility.remove(&sheet_id).is_some() {
4457            self.invalidate_row_visibility_mask_cache();
4458        }
4459        self.record_formula_plane_structural_change(StructuralScope::RemovedSheet(sheet_id));
4460        self.mark_topology_edited();
4461        Ok(())
4462    }
4463
4464    /// Helper to synchronize the Arrow-backed storage layer.
4465    fn rename_sheet_in_arrow_store(&mut self, target_name: &str, new_name: &str) -> bool {
4466        if let Some(asheet) = self
4467            .arrow_sheets
4468            .sheets
4469            .iter_mut()
4470            .find(|s| s.name.as_ref() == target_name)
4471        {
4472            asheet.name = std::sync::Arc::<str>::from(new_name);
4473            return true;
4474        }
4475        false
4476    }
4477
4478    pub fn rename_sheet(&mut self, sheet_id: SheetId, new_name: &str) -> Result<(), ExcelError> {
4479        let old_name = self.graph.sheet_name(sheet_id).to_string();
4480
4481        // Speculative Storage Update
4482        // Update name in storage FIRST so the Evaluator can find it during Graph rescue.
4483        self.rename_sheet_in_arrow_store(&old_name, new_name);
4484
4485        // Graph Update (Metadata + Rescue Logic)
4486        match self.graph.rename_sheet(sheet_id, new_name) {
4487            Ok(_) => {
4488                self.rename_staged_formula_sheet(&old_name, new_name);
4489                // Success! Invalidate cache for the moved sheet
4490                let sheet_vertices: Vec<VertexId> = self
4491                    .graph
4492                    .grid_vertices_in_sheet(sheet_id)
4493                    .map(|(id, _)| id)
4494                    .collect();
4495                for v_id in sheet_vertices {
4496                    self.graph.mark_vertex_dirty(v_id);
4497                }
4498                // Sheet rename preserves SheetId and therefore formula dependencies.
4499                self.mark_topology_edited();
4500                Ok(())
4501            }
4502            Err(e) => {
4503                // ROLLBACK: Revert storage if graph rejected the name
4504                self.rename_sheet_in_arrow_store(new_name, &old_name);
4505                Err(e)
4506            }
4507        }
4508    }
4509
4510    pub fn named_ranges_iter(
4511        &self,
4512    ) -> impl Iterator<Item = (&String, &crate::engine::named_range::NamedRange)> {
4513        self.graph.named_ranges_iter()
4514    }
4515
4516    pub fn sheet_named_ranges_iter(
4517        &self,
4518    ) -> impl Iterator<Item = (&(SheetId, String), &crate::engine::named_range::NamedRange)> {
4519        self.graph.sheet_named_ranges_iter()
4520    }
4521
4522    pub fn resolve_name_entry(
4523        &self,
4524        name: &str,
4525        current_sheet: SheetId,
4526    ) -> Option<&crate::engine::named_range::NamedRange> {
4527        self.graph.resolve_name_entry(name, current_sheet)
4528    }
4529
4530    /// The [`NameScope`] an optional scope-sheet argument denotes.
4531    ///
4532    /// `None` means **workbook scope**, not "the default sheet": a caller that
4533    /// supplies no sheet context is asking about workbook-scoped names only.
4534    /// An unknown sheet name is a malformed query and errors rather than
4535    /// silently degrading to another sheet's scope (issue #110).
4536    ///
4537    /// This is the one owned derivation from `Option<&str>` to a name scope;
4538    /// every scope-taking entry point routes through it.
4539    pub(crate) fn name_query_scope(
4540        &self,
4541        scope_sheet: Option<&str>,
4542    ) -> Result<NameScope, ExcelError> {
4543        match scope_sheet {
4544            None => Ok(NameScope::Workbook),
4545            Some(sheet) => self
4546                .graph
4547                .sheet_id(sheet)
4548                .map(NameScope::Sheet)
4549                .ok_or_else(|| {
4550                    ExcelError::new(ExcelErrorKind::Ref)
4551                        .with_message(format!("name scope sheet not found: {sheet}"))
4552                }),
4553        }
4554    }
4555
4556    /// Whether `name` resolves in the scope denoted by `scope_sheet`.
4557    ///
4558    /// `scope_sheet == None` asks about workbook scope only; a name scoped to a
4559    /// single sheet (including the default sheet) does not answer it. An unknown
4560    /// sheet name resolves nothing.
4561    /// Resolve a [`SharedSheetLocator`](crate::reference::SharedSheetLocator)
4562    /// against an explicit context sheet.
4563    ///
4564    /// Thin forwarder to
4565    /// [`SheetRegistry::resolve_locator`](crate::engine::sheet_registry::SheetRegistry::resolve_locator),
4566    /// the single owned derivation. `Current` resolves to `context_sheet`, never
4567    /// to the workbook's default sheet.
4568    fn resolve_sheet_locator(
4569        &self,
4570        locator: &crate::reference::SharedSheetLocator<'_>,
4571        context_sheet: SheetId,
4572    ) -> Result<SheetId, ExcelError> {
4573        self.graph
4574            .sheet_reg()
4575            .resolve_locator(locator, context_sheet)
4576    }
4577
4578    pub fn has_name(&self, name: &str, scope_sheet: Option<&str>) -> bool {
4579        let Ok(scope) = self.name_query_scope(scope_sheet) else {
4580            return false;
4581        };
4582        self.graph
4583            .resolve_name_entry_in_scope(name, scope)
4584            .is_some()
4585    }
4586
4587    /// The current value of `name` in the scope denoted by `scope_sheet`.
4588    ///
4589    /// Scoping follows [`Self::has_name`]: `None` is workbook scope only.
4590    pub fn resolved_name_value(
4591        &self,
4592        name: &str,
4593        scope_sheet: Option<&str>,
4594    ) -> Option<LiteralValue> {
4595        let scope = self.name_query_scope(scope_sheet).ok()?;
4596        let entry = self.graph.resolve_name_entry_in_scope(name, scope)?;
4597        self.graph.get_value(entry.vertex)
4598    }
4599
4600    pub fn table_metadata(&self, name: &str) -> Option<TableMetadata> {
4601        let entry = self.graph.resolve_table_entry(name)?;
4602        Some(TableMetadata {
4603            name: entry.name.clone(),
4604            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
4605            start_row: entry.range.start.coord.row() + 1,
4606            start_col: entry.range.start.coord.col() + 1,
4607            end_row: entry.range.end.coord.row() + 1,
4608            end_col: entry.range.end.coord.col() + 1,
4609            header_row: entry.header_row,
4610            headers: entry.headers.clone(),
4611            totals_row: entry.totals_row,
4612        })
4613    }
4614
4615    /// Metadata for every defined table, ordered by name.
4616    pub fn tables(&self) -> Vec<TableMetadata> {
4617        self.graph
4618            .table_names()
4619            .into_iter()
4620            .filter_map(|name| self.table_metadata(&name))
4621            .collect()
4622    }
4623
4624    pub fn named_ranges_snapshot(&self) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4625        let mut out: Vec<crate::engine::named_range::NamedRangeSnapshot> = Vec::new();
4626
4627        for (name, named) in self.graph.named_ranges_iter() {
4628            out.push(crate::engine::named_range::NamedRangeSnapshot {
4629                name: name.clone(),
4630                scope: NameScope::Workbook,
4631                definition: named.definition.clone(),
4632            });
4633        }
4634
4635        for ((sheet_id, name), named) in self.graph.sheet_named_ranges_iter() {
4636            out.push(crate::engine::named_range::NamedRangeSnapshot {
4637                name: name.clone(),
4638                scope: NameScope::Sheet(*sheet_id),
4639                definition: named.definition.clone(),
4640            });
4641        }
4642
4643        out.sort_by(|a, b| {
4644            let a_scope = match a.scope {
4645                NameScope::Workbook => (0u8, 0u32),
4646                NameScope::Sheet(id) => (1u8, u32::from(id)),
4647            };
4648            let b_scope = match b.scope {
4649                NameScope::Workbook => (0u8, 0u32),
4650                NameScope::Sheet(id) => (1u8, u32::from(id)),
4651            };
4652            a_scope.cmp(&b_scope).then_with(|| a.name.cmp(&b.name))
4653        });
4654
4655        out
4656    }
4657
4658    pub fn named_ranges_snapshot_for_sheet(
4659        &self,
4660        sheet_id: SheetId,
4661    ) -> Vec<crate::engine::named_range::NamedRangeSnapshot> {
4662        self.named_ranges_snapshot()
4663            .into_iter()
4664            .filter(|entry| match entry.scope {
4665                NameScope::Workbook => true,
4666                NameScope::Sheet(id) => id == sheet_id,
4667            })
4668            .collect()
4669    }
4670
4671    pub fn define_name(
4672        &mut self,
4673        name: &str,
4674        definition: NamedDefinition,
4675        scope: NameScope,
4676    ) -> Result<(), ExcelError> {
4677        // A new define can flip resolution for spans that previously resolved
4678        // the same name through another scope (e.g. a sheet-scoped name
4679        // shadowing a workbook-scoped one). Demote those spans BEFORE the
4680        // registry changes so their cells re-ingest and re-resolve through
4681        // the normal legacy path.
4682        self.graph.validate_define_name(name, scope)?;
4683        let prepared = self
4684            .prepare_name_dependent_span_demotion([name])
4685            .map_err(Self::editor_error_to_excel)?;
4686        let demoted = self
4687            .commit_name_dependent_span_demotion(prepared)
4688            .map_err(Self::editor_error_to_excel)?;
4689        self.graph.define_name(name, definition, scope)?;
4690        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4691        if !demoted {
4692            self.mark_topology_edited();
4693        }
4694        Ok(())
4695    }
4696
4697    pub fn update_name(
4698        &mut self,
4699        name: &str,
4700        definition: NamedDefinition,
4701        scope: NameScope,
4702    ) -> Result<(), ExcelError> {
4703        // Demote name-dependent spans BEFORE the registry update: the demoted
4704        // cells re-materialize as legacy vertices attached to the name vertex
4705        // (via their resolved-name dep plans), so the registry update's
4706        // dependent dirtying reaches them exactly like long-lived legacy
4707        // formulas.
4708        self.graph.validate_existing_name(name, scope)?;
4709        let prepared = self
4710            .prepare_name_dependent_span_demotion([name])
4711            .map_err(Self::editor_error_to_excel)?;
4712        let demoted = self
4713            .commit_name_dependent_span_demotion(prepared)
4714            .map_err(Self::editor_error_to_excel)?;
4715        self.graph.update_name(name, definition, scope)?;
4716        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4717        if !demoted {
4718            self.mark_topology_edited();
4719        }
4720        Ok(())
4721    }
4722
4723    pub fn delete_name(&mut self, name: &str, scope: NameScope) -> Result<(), ExcelError> {
4724        // Demote first (see update_name): the demoted legacy vertices become
4725        // dependents of the name vertex, so delete_name dirties them and they
4726        // re-evaluate to #NAME? exactly as legacy formulas do.
4727        self.graph.validate_existing_name(name, scope)?;
4728        let prepared = self
4729            .prepare_name_dependent_span_demotion([name])
4730            .map_err(Self::editor_error_to_excel)?;
4731        let demoted = self
4732            .commit_name_dependent_span_demotion(prepared)
4733            .map_err(Self::editor_error_to_excel)?;
4734        self.graph.delete_name(name, scope)?;
4735        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4736        if !demoted {
4737            self.mark_topology_edited();
4738        }
4739        Ok(())
4740    }
4741
4742    fn exact_name_dependent_span_refs<'a>(
4743        &self,
4744        names: impl IntoIterator<Item = &'a str>,
4745    ) -> Vec<FormulaSpanRef> {
4746        let authority = self.graph.formula_authority();
4747        let mut refs = names
4748            .into_iter()
4749            .flat_map(|name| authority.plane.name_dependent_span_refs(name))
4750            .collect::<Vec<_>>();
4751        refs.sort_unstable_by_key(|span_ref| {
4752            (span_ref.id.0, span_ref.generation, span_ref.version)
4753        });
4754        refs.dedup();
4755        refs
4756    }
4757
4758    fn name_event_names(events: &[ChangeEvent]) -> Vec<String> {
4759        events
4760            .iter()
4761            .filter_map(|event| match event {
4762                ChangeEvent::DefineName { name, .. }
4763                | ChangeEvent::UpdateName { name, .. }
4764                | ChangeEvent::DeleteName { name, .. }
4765                | ChangeEvent::NamedRangeAdjusted { name, .. } => Some(name.clone()),
4766                _ => None,
4767            })
4768            .collect()
4769    }
4770
4771    fn prepare_name_dependent_span_demotion<'a>(
4772        &mut self,
4773        names: impl IntoIterator<Item = &'a str>,
4774    ) -> Result<Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>, crate::engine::EditorError>
4775    {
4776        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
4777            return Ok(None);
4778        }
4779        let refs = self.exact_name_dependent_span_refs(names);
4780        if refs.is_empty() {
4781            return Ok(None);
4782        }
4783
4784        // Exact demotion creates dirty legacy vertices. Preserve the old
4785        // span's clean/dirty state so the following name mutation remains the
4786        // sole authority that dirties formulas whose resolution changed.
4787        let dirty_coords = self.compute_current_formula_plane_dirty_result_coords()?;
4788        let clean_cells = {
4789            let authority = self.graph.formula_authority();
4790            refs.iter()
4791                .filter_map(|span_ref| authority.plane.spans.get(*span_ref))
4792                .flat_map(|span| span.domain.iter())
4793                .filter(|coord| !dirty_coords.contains(&(coord.sheet_id, coord.row, coord.col)))
4794                .map(|coord| {
4795                    CellRef::new(coord.sheet_id, Coord::new(coord.row, coord.col, true, true))
4796                })
4797                .collect::<Vec<_>>()
4798        };
4799        self.prepare_formula_span_demotion(&refs)
4800            .map(|prepared| Some((prepared, clean_cells)))
4801            .map_err(|error| match error {
4802                FormulaSpanDemotionError::Resource(error) => error.into(),
4803                error => crate::engine::EditorError::TransactionFailed {
4804                    reason: format!(
4805                        "FormulaPlane name-dependent demotion preparation failed: {error}"
4806                    ),
4807                },
4808            })
4809    }
4810
4811    fn commit_name_dependent_span_demotion(
4812        &mut self,
4813        prepared: Option<(PreparedFormulaSpanDemotion, Vec<CellRef>)>,
4814    ) -> Result<bool, crate::engine::EditorError> {
4815        let Some((prepared, clean_cells)) = prepared else {
4816            return Ok(false);
4817        };
4818        self.commit_prepared_formula_span_demotion(prepared)
4819            .map_err(|error| crate::engine::EditorError::TransactionFailed {
4820                reason: format!("FormulaPlane name-dependent demotion commit failed: {error}"),
4821            })?;
4822        for cell in clean_cells {
4823            if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
4824                self.graph.set_dirty(vertex_id, false);
4825            }
4826        }
4827        Ok(true)
4828    }
4829
4830    pub fn define_table(
4831        &mut self,
4832        name: &str,
4833        range: crate::reference::RangeRef,
4834        header_row: bool,
4835        headers: Vec<String>,
4836        totals_row: bool,
4837    ) -> Result<(), ExcelError> {
4838        self.graph
4839            .define_table(name, range, header_row, headers, totals_row)?;
4840        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
4841        self.mark_topology_edited();
4842        Ok(())
4843    }
4844
4845    pub fn define_source_scalar(
4846        &mut self,
4847        name: &str,
4848        version: Option<u64>,
4849    ) -> Result<(), ExcelError> {
4850        self.graph.define_source_scalar(name, version)?;
4851        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4852        self.mark_topology_edited();
4853        Ok(())
4854    }
4855
4856    pub fn define_source_table(
4857        &mut self,
4858        name: &str,
4859        version: Option<u64>,
4860    ) -> Result<(), ExcelError> {
4861        self.graph.define_source_table(name, version)?;
4862        self.record_formula_plane_structural_change(StructuralScope::OpaqueGlobal);
4863        self.mark_topology_edited();
4864        Ok(())
4865    }
4866
4867    pub fn set_source_scalar_version(
4868        &mut self,
4869        name: &str,
4870        version: Option<u64>,
4871    ) -> Result<(), ExcelError> {
4872        self.graph.set_source_scalar_version(name, version)?;
4873        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4874            self.graph
4875                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4876        }
4877        Ok(())
4878    }
4879
4880    pub fn set_source_table_version(
4881        &mut self,
4882        name: &str,
4883        version: Option<u64>,
4884    ) -> Result<(), ExcelError> {
4885        self.graph.set_source_table_version(name, version)?;
4886        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4887            self.graph
4888                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4889        }
4890        Ok(())
4891    }
4892
4893    pub fn invalidate_source(&mut self, name: &str) -> Result<(), ExcelError> {
4894        self.graph.invalidate_source(name)?;
4895        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
4896            self.graph
4897                .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
4898        }
4899        Ok(())
4900    }
4901
4902    pub fn vertex_value(&self, vertex: VertexId) -> Option<LiteralValue> {
4903        self.graph.get_value(vertex)
4904    }
4905
4906    pub fn graph_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
4907        self.graph.get_cell_value(sheet, row, col)
4908    }
4909
4910    pub fn vertex_for_cell(&self, cell: &CellRef) -> Option<VertexId> {
4911        self.graph.get_vertex_for_cell(cell)
4912    }
4913
4914    pub fn evaluation_vertices(&self) -> Vec<VertexId> {
4915        self.graph.get_evaluation_vertices()
4916    }
4917
4918    /// Return read-only baseline counters for FormulaPlane/dispatch benchmarking.
4919    pub fn baseline_stats(&self) -> EngineBaselineStats {
4920        let graph = self.graph.baseline_stats();
4921        let formula_authority = self.graph.formula_authority();
4922        let formula_dirty = self.graph.formula_dirty_stats();
4923        EngineBaselineStats {
4924            graph_vertex_count: graph.graph_vertex_count,
4925            graph_formula_vertex_count: graph.graph_formula_vertex_count,
4926            graph_edge_count: graph.graph_edge_count,
4927            dirty_vertex_count: graph.dirty_vertex_count,
4928            evaluation_vertex_count: graph.evaluation_vertex_count,
4929            formula_ast_root_count: graph.formula_ast_root_count,
4930            formula_ast_node_count: graph.formula_ast_node_count,
4931            staged_formula_count: self.staged_formula_count(),
4932            formula_plane_active_span_count: formula_authority.active_span_count(),
4933            formula_plane_producer_result_entries: formula_authority.producer_results.len(),
4934            formula_plane_consumer_read_entries: formula_authority.consumer_reads.len(),
4935            formula_plane_mixed_topology_cache_builds: self.mixed_topology_cache_builds,
4936            formula_plane_mixed_topology_cache_hits: self.mixed_topology_cache_hits,
4937            formula_plane_mixed_topology_cache_overflows: self.mixed_topology_cache_overflows,
4938            formula_plane_dirty_pending_events: formula_dirty.pending_events,
4939            formula_plane_dirty_region_events_recorded: formula_dirty.region_events_recorded,
4940            formula_plane_dirty_span_region_events_recorded: formula_dirty
4941                .span_region_events_recorded,
4942            formula_plane_dirty_whole_span_seeds_recorded: formula_dirty.whole_span_seeds_recorded,
4943            formula_plane_dirty_global_invalidations: formula_dirty.global_whole_span_invalidations,
4944            formula_plane_structural_span_candidates: self.formula_plane_structural_span_candidates,
4945            formula_plane_cycle_member_span_demotions: self
4946                .formula_plane_cycle_member_span_demotions,
4947            formula_plane_array_result_span_demotions: self
4948                .formula_plane_array_result_span_demotions,
4949            retained_scc_members: self.retained_scc_members.len(),
4950        }
4951    }
4952
4953    /// Mutation revision captured by read-only engine reports.
4954    pub(crate) fn inspection_mutation_revision(&self) -> u64 {
4955        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
4956    }
4957
4958    #[cfg(test)]
4959    pub(crate) fn used_axis_bounds_cache_stats(&self) -> (usize, usize, usize, usize) {
4960        self.used_axis_bounds_cache
4961            .read()
4962            .ok()
4963            .and_then(|guard| {
4964                guard.as_ref().map(|cache| {
4965                    (
4966                        cache.row_hits.load(Ordering::Relaxed),
4967                        cache.row_misses.load(Ordering::Relaxed),
4968                        cache.col_hits.load(Ordering::Relaxed),
4969                        cache.col_misses.load(Ordering::Relaxed),
4970                    )
4971                })
4972            })
4973            .unwrap_or((0, 0, 0, 0))
4974    }
4975
4976    pub fn set_first_load_assume_new(&mut self, enabled: bool) {
4977        self.graph.set_first_load_assume_new(enabled);
4978    }
4979
4980    pub fn first_load_assume_new(&self) -> bool {
4981        self.graph.first_load_assume_new()
4982    }
4983
4984    pub fn reset_ensure_touched(&mut self) {
4985        self.graph.reset_ensure_touched();
4986    }
4987
4988    pub fn finalize_sheet_index(&mut self, sheet: &str) {
4989        self.graph.finalize_sheet_index(sheet);
4990    }
4991
4992    /// Execute a named Engine action.
4993    ///
4994    /// Ticket 614 introduces this as the stable Engine-level transaction surface.
4995    /// For now actions are commit-only: they do not create changelog boundaries and they do not
4996    /// provide rollback/atomicity.
4997    ///
4998    /// Nested actions are deterministically handled by *disallowing* nesting: calling
4999    /// `Engine::action` while another action is active returns `EditorError::TransactionFailed`.
5000    pub fn action<T>(
5001        &mut self,
5002        name: impl AsRef<str>,
5003        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5004    ) -> Result<T, crate::engine::EditorError> {
5005        if self.action_depth != 0 {
5006            return Err(crate::engine::EditorError::TransactionFailed {
5007                reason: "Nested Engine::action calls are not supported (ticket 614: commit-only surface)"
5008                    .to_string(),
5009            });
5010        }
5011
5012        self.action_depth = 1;
5013        let engine_ptr: *mut Engine<R> = self;
5014        let _guard = ActionDepthGuard {
5015            engine: engine_ptr,
5016            _marker: std::marker::PhantomData,
5017        };
5018
5019        let mut tx = EngineAction {
5020            engine: self,
5021            name: name.as_ref().to_string(),
5022            capture: None,
5023            arrow_undo: None,
5024            atomic_policy: false,
5025        };
5026        f(&mut tx)
5027    }
5028
5029    /// Execute a named Engine action with atomic commit/rollback semantics.
5030    ///
5031    /// This variant does not require a `ChangeLog` and uses an internal journal for rollback.
5032    pub fn action_atomic<T>(
5033        &mut self,
5034        name: impl Into<String>,
5035        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5036    ) -> Result<T, crate::engine::EditorError> {
5037        let (v, _j) = self.action_atomic_journal(name, f)?;
5038        Ok(v)
5039    }
5040
5041    /// Like `action_atomic`, but returns the committed journal entry for undo/redo storage.
5042    pub fn action_atomic_journal<T>(
5043        &mut self,
5044        name: impl Into<String>,
5045        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5046    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
5047        if self.action_depth != 0 {
5048            return Err(crate::engine::EditorError::TransactionFailed {
5049                reason: "Nested Engine::action calls are not supported (deterministic rule)"
5050                    .to_string(),
5051            });
5052        }
5053
5054        self.action_depth = 1;
5055        let engine_ptr: *mut Engine<R> = self;
5056        let _guard = ActionDepthGuard {
5057            engine: engine_ptr,
5058            _marker: std::marker::PhantomData,
5059        };
5060
5061        let name_str = name.into();
5062        let mut capture = MutationCapture::new(Default::default());
5063        let start_len = capture.len();
5064        self.action_atomic_impl(&mut capture, start_len, name_str, f)
5065    }
5066
5067    fn action_atomic_impl<T>(
5068        &mut self,
5069        capture: &mut MutationCapture,
5070        start_len: usize,
5071        name: String,
5072        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5073    ) -> Result<(T, crate::engine::ActionJournal), crate::engine::EditorError> {
5074        let invalidation_baseline = self.invalidation_baseline();
5075        let mut arrow_undo = crate::engine::ArrowUndoBatch::default();
5076        let arrow_ptr: *mut crate::engine::ArrowUndoBatch = &mut arrow_undo;
5077
5078        let capture_ptr: *mut MutationCapture = capture;
5079        let mut tx = EngineAction {
5080            engine: self,
5081            name: name.clone(),
5082            capture: Some(capture_ptr),
5083            arrow_undo: Some(arrow_ptr),
5084            atomic_policy: true,
5085        };
5086
5087        let res = f(&mut tx);
5088
5089        // Capture graph structural delta for this action.
5090        let graph_events: Vec<crate::engine::ChangeEvent> =
5091            unsafe { (&*capture_ptr).events() }[start_len..].to_vec();
5092        let graph_batch = crate::engine::GraphUndoBatch {
5093            events: graph_events,
5094        };
5095        let affected_cells = arrow_undo.ops.len();
5096        let journal = crate::engine::ActionJournal {
5097            name,
5098            graph: graph_batch,
5099            arrow: arrow_undo,
5100            affected_cells,
5101        };
5102
5103        match res {
5104            Ok(v) => {
5105                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5106                    for event in &journal.graph.events {
5107                        self.record_formula_plane_change_for_event(event);
5108                    }
5109                    self.invalidate_for_action_journal(
5110                        &journal,
5111                        LoggedEditDirection::Original,
5112                        invalidation_baseline,
5113                    );
5114                }
5115                Ok((v, journal))
5116            }
5117            Err(e) => {
5118                if let Err(rb) = self.rollback_from_action_journal(&journal, invalidation_baseline)
5119                {
5120                    return Err(crate::engine::EditorError::TransactionFailed {
5121                        reason: format!(
5122                            "Engine::action_atomic rollback failed after error '{e}': {rb}"
5123                        ),
5124                    });
5125                }
5126                if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5127                    for event in &journal.graph.events {
5128                        self.record_formula_plane_change_for_event(event);
5129                    }
5130                }
5131                Err(e)
5132            }
5133        }
5134    }
5135
5136    /// Execute a named Engine action, logging graph changes into the provided ChangeLog.
5137    ///
5138    /// Ticket 615: this variant provides atomicity. If the action returns an error, it rolls back:
5139    /// - Dependency graph structural edits (via inverse ChangeEvents)
5140    /// - Arrow-truth overlay writes mirrored from ChangeEvents
5141    /// - ChangeLog entries (published only after a successful commit)
5142    pub fn action_with_logger<T>(
5143        &mut self,
5144        log: &mut crate::engine::ChangeLog,
5145        name: impl AsRef<str>,
5146        f: impl FnOnce(&mut EngineAction<'_, R>) -> Result<T, crate::engine::EditorError>,
5147    ) -> Result<T, crate::engine::EditorError> {
5148        if self.action_depth != 0 {
5149            return Err(crate::engine::EditorError::TransactionFailed {
5150                reason: "Nested Engine::action calls are not supported (deterministic rule)"
5151                    .to_string(),
5152            });
5153        }
5154
5155        self.action_depth = 1;
5156        let engine_ptr: *mut Engine<R> = self;
5157        let _guard = ActionDepthGuard {
5158            engine: engine_ptr,
5159            _marker: std::marker::PhantomData,
5160        };
5161
5162        let name_str = name.as_ref().to_string();
5163        let mut capture = MutationCapture::new(log.current_meta());
5164        let start_len = capture.len();
5165        capture.begin_compound(name_str.clone());
5166
5167        // Mutation correctness uses the complete private capture. The provided ChangeLog remains
5168        // an observability sink and is not touched until the action outcome is known.
5169        let res = self.action_atomic_impl(&mut capture, start_len, name_str, f);
5170        capture.close_compounds();
5171
5172        match res {
5173            Ok((v, _journal)) => {
5174                log.publish_capture(capture);
5175                Ok(v)
5176            }
5177            Err(e) => {
5178                // Preserve sequence/group gaps without retaining failed events or evicting history.
5179                log.discard_capture(capture);
5180                Err(e)
5181            }
5182        }
5183    }
5184
5185    fn rollback_from_action_journal(
5186        &mut self,
5187        journal: &crate::engine::ActionJournal,
5188        invalidation_baseline: InvalidationBaseline,
5189    ) -> Result<(), crate::engine::EditorError> {
5190        // Invalidate first so a partial inverse failure cannot leave a changed
5191        // graph behind an apparently current schedule or lookup cache.
5192        self.invalidate_for_action_journal(
5193            journal,
5194            LoggedEditDirection::InverseReplay,
5195            invalidation_baseline,
5196        );
5197        // 1) Roll back the dependency graph structure.
5198        journal.graph.undo(&mut self.graph)?;
5199        // 2) Roll back engine row-visibility sidecar events.
5200        self.apply_inverse_row_visibility_events(&journal.graph.events);
5201        // 3) Roll back Arrow-truth overlays.
5202        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5203        Ok(())
5204    }
5205
5206    fn rollback_from_change_events(
5207        &mut self,
5208        events: &[crate::engine::ChangeEvent],
5209        invalidation_baseline: InvalidationBaseline,
5210    ) -> Result<(), crate::engine::EditorError> {
5211        use crate::engine::ChangeEvent;
5212
5213        // Fail closed before applying inverses because replay can return after
5214        // only part of the batch has been restored.
5215        self.invalidate_for_change_events(
5216            events,
5217            LoggedEditDirection::InverseReplay,
5218            invalidation_baseline,
5219        );
5220
5221        // 1) Roll back the dependency graph.
5222        {
5223            let mut editor = crate::engine::VertexEditor::new(&mut self.graph);
5224            let mut compound_stack: Vec<usize> = Vec::new();
5225            for ev in events.iter().rev() {
5226                match ev {
5227                    ChangeEvent::CompoundEnd { depth } => compound_stack.push(*depth),
5228                    ChangeEvent::CompoundStart { depth, .. } => {
5229                        if compound_stack.last() == Some(depth) {
5230                            compound_stack.pop();
5231                        }
5232                    }
5233                    ChangeEvent::SetRowVisibility { .. } => {
5234                        // Engine-side metadata handled after dropping graph editor borrow.
5235                    }
5236                    _ => {
5237                        editor.apply_inverse(ev.clone())?;
5238                    }
5239                }
5240            }
5241        }
5242
5243        // 2) Roll back engine row-visibility metadata.
5244        for ev in events.iter().rev() {
5245            self.apply_inverse_row_visibility_event(ev);
5246        }
5247
5248        // 3) Roll back Arrow-truth overlays mirrored from those ChangeEvents.
5249        for ev in events.iter().rev() {
5250            self.mirror_inverse_change_to_arrow(ev);
5251        }
5252
5253        Ok(())
5254    }
5255
5256    fn read_cell_formula_ast(&self, sheet: &str, row: u32, col: u32) -> Option<ASTNode> {
5257        let sheet_id = self.graph.sheet_id(sheet)?;
5258        let coord = Coord::from_excel(row, col, true, true);
5259        let cell = CellRef::new(sheet_id, coord);
5260        let vid = self.graph.get_vertex_for_cell(&cell)?;
5261        let ast_id = self.graph.get_formula_id(vid)?;
5262        self.graph
5263            .data_store()
5264            .retrieve_ast(ast_id, self.graph.sheet_reg())
5265    }
5266
5267    pub fn define_name_with_logger(
5268        &mut self,
5269        log: &mut crate::engine::ChangeLog,
5270        name: &str,
5271        definition: NamedDefinition,
5272        scope: NameScope,
5273    ) -> Result<(), crate::engine::EditorError> {
5274        self.graph
5275            .validate_define_name(name, scope)
5276            .map_err(crate::engine::EditorError::Excel)?;
5277        let has_dependent_spans = !self.exact_name_dependent_span_refs([name]).is_empty();
5278        if has_dependent_spans && matches!(definition, NamedDefinition::Formula { .. }) {
5279            return Err(crate::engine::EditorError::TransactionUnsupported {
5280                reason: "logged formula-name shadowing with active FormulaPlane dependents is not supported"
5281                    .to_string(),
5282            });
5283        }
5284        let prepared = self.prepare_name_dependent_span_demotion([name])?;
5285        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5286        {
5287            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5288            editor.define_name(name, definition, scope)?;
5289        }
5290        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5291        if !demoted {
5292            self.mark_topology_edited();
5293        }
5294        Ok(())
5295    }
5296
5297    pub fn update_name_with_logger(
5298        &mut self,
5299        log: &mut crate::engine::ChangeLog,
5300        name: &str,
5301        definition: NamedDefinition,
5302        scope: NameScope,
5303    ) -> Result<(), crate::engine::EditorError> {
5304        self.graph
5305            .validate_existing_name(name, scope)
5306            .map_err(crate::engine::EditorError::Excel)?;
5307        let prepared = self.prepare_name_dependent_span_demotion([name])?;
5308        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5309        {
5310            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5311            editor.update_name(name, definition, scope)?;
5312        }
5313        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5314        if !demoted {
5315            self.mark_topology_edited();
5316        }
5317        Ok(())
5318    }
5319
5320    pub fn delete_name_with_logger(
5321        &mut self,
5322        log: &mut crate::engine::ChangeLog,
5323        name: &str,
5324        scope: NameScope,
5325    ) -> Result<(), crate::engine::EditorError> {
5326        self.graph
5327            .validate_existing_name(name, scope)
5328            .map_err(crate::engine::EditorError::Excel)?;
5329        let prepared = self.prepare_name_dependent_span_demotion([name])?;
5330        let demoted = self.commit_name_dependent_span_demotion(prepared)?;
5331        {
5332            let mut editor = crate::engine::VertexEditor::with_logger(&mut self.graph, log);
5333            editor.delete_name(name, scope)?;
5334        }
5335        self.record_formula_plane_structural_change(StructuralScope::AllSheets);
5336        if !demoted {
5337            self.mark_topology_edited();
5338        }
5339        Ok(())
5340    }
5341
5342    pub fn edit_with_logger<T>(
5343        &mut self,
5344        log: &mut crate::engine::ChangeLog,
5345        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5346    ) -> Result<T, crate::engine::EditorError> {
5347        let mut capture = MutationCapture::new(log.current_meta());
5348        let result = self.edit_with_capture(&mut capture, f);
5349        capture.close_compounds();
5350        match result {
5351            Ok(value) => {
5352                log.publish_capture(capture);
5353                Ok(value)
5354            }
5355            Err(error) => {
5356                log.discard_capture(capture);
5357                Err(error)
5358            }
5359        }
5360    }
5361
5362    fn edit_with_capture<T>(
5363        &mut self,
5364        capture: &mut MutationCapture,
5365        f: impl FnOnce(&mut crate::engine::VertexEditor) -> T,
5366    ) -> Result<T, crate::engine::EditorError> {
5367        let invalidation_baseline = self.invalidation_baseline();
5368        let start_len = capture.len();
5369
5370        // Provide a spill snapshot reader so VertexEditor can snapshot Arrow-truth spill values
5371        // (graph value cache is intentionally empty in canonical mode).
5372        struct ArrowSpillReader<'a> {
5373            sheets: &'a crate::arrow_store::SheetStore,
5374        }
5375        impl crate::engine::graph::editor::vertex_editor::SpillValueReader for ArrowSpillReader<'_> {
5376            fn read_cell_value(
5377                &self,
5378                sheet: &str,
5379                row: u32,
5380                col: u32,
5381            ) -> Option<formualizer_common::LiteralValue> {
5382                use formualizer_common::LiteralValue;
5383                let asheet = self.sheets.sheet(sheet)?;
5384                let r0 = row.saturating_sub(1) as usize;
5385                let c0 = col.saturating_sub(1) as usize;
5386                let v = asheet.get_cell_value(r0, c0);
5387                if matches!(v, LiteralValue::Empty) {
5388                    None
5389                } else {
5390                    Some(v)
5391                }
5392            }
5393        }
5394
5395        let ret = {
5396            let spill_reader = ArrowSpillReader {
5397                sheets: &self.arrow_sheets,
5398            };
5399            let mut editor = crate::engine::VertexEditor::with_logger_and_spill_reader(
5400                &mut self.graph,
5401                capture,
5402                &spill_reader,
5403            );
5404            f(&mut editor)
5405        };
5406
5407        let new_events = capture.events()[start_len..].to_vec();
5408        if new_events.iter().any(|event| {
5409            matches!(
5410                event,
5411                ChangeEvent::DefineName { .. }
5412                    | ChangeEvent::UpdateName { .. }
5413                    | ChangeEvent::DeleteName { .. }
5414            )
5415        }) {
5416            self.rollback_from_change_events(&new_events, invalidation_baseline)?;
5417            return Err(crate::engine::EditorError::TransactionUnsupported {
5418                reason: "name mutations must use Engine's prepared logged-name APIs".to_string(),
5419            });
5420        }
5421
5422        // Mirror value-impacting graph events to Arrow for forward edits.
5423        // This keeps Arrow overlays (delta + computed) consistent when edits clear/commit spills.
5424        self.clear_logged_cell_format_states(&new_events);
5425        for ev in &new_events {
5426            self.mirror_forward_change_to_arrow(ev);
5427        }
5428        for ev in &new_events {
5429            self.record_formula_plane_change_for_event(ev);
5430        }
5431
5432        // Atomic EngineAction calls publish one invalidation for their complete
5433        // journal at commit/rollback. Direct logged edits publish here.
5434        if self.action_depth == 0 {
5435            self.invalidate_for_change_events(
5436                &new_events,
5437                LoggedEditDirection::Original,
5438                invalidation_baseline,
5439            );
5440        }
5441
5442        Ok(ret)
5443    }
5444
5445    pub(crate) fn preflight_replay_admission(
5446        &mut self,
5447        events: &[ChangeEvent],
5448        forward: bool,
5449    ) -> Result<(), crate::engine::EditorError> {
5450        if !self.graph_admission_enabled() {
5451            return Ok(());
5452        }
5453        let mut vertex_delta = 0i128;
5454        let mut edge_delta = 0i128;
5455        let mut added_vertices = 0usize;
5456        let mut added_edges = 0usize;
5457        let mut formula_cells = BTreeSet::new();
5458        for event in events {
5459            match event {
5460                ChangeEvent::AddVertex {
5461                    formula,
5462                    coord,
5463                    sheet_id,
5464                    ..
5465                } => {
5466                    let delta = if forward { 1 } else { -1 };
5467                    vertex_delta += delta;
5468                    if forward {
5469                        added_vertices = added_vertices.saturating_add(1);
5470                        if formula.is_some() {
5471                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5472                        }
5473                    }
5474                }
5475                ChangeEvent::RemoveVertex {
5476                    old_formula,
5477                    coord,
5478                    sheet_id,
5479                    ..
5480                } => {
5481                    let delta = if forward { -1 } else { 1 };
5482                    vertex_delta += delta;
5483                    if !forward {
5484                        added_vertices = added_vertices.saturating_add(1);
5485                        if old_formula.is_some()
5486                            && let (Some(sheet_id), Some(coord)) = (sheet_id, coord)
5487                        {
5488                            formula_cells.insert((*sheet_id, coord.row(), coord.col()));
5489                        }
5490                    }
5491                }
5492                ChangeEvent::EdgeAdded { .. } => {
5493                    let delta = if forward { 1 } else { -1 };
5494                    edge_delta += delta;
5495                    if forward {
5496                        added_edges = added_edges.saturating_add(1);
5497                    }
5498                }
5499                ChangeEvent::EdgeRemoved { .. } => {
5500                    let delta = if forward { -1 } else { 1 };
5501                    edge_delta += delta;
5502                    if !forward {
5503                        added_edges = added_edges.saturating_add(1);
5504                    }
5505                }
5506                ChangeEvent::SetFormula {
5507                    addr, old_formula, ..
5508                } => {
5509                    if forward || old_formula.is_some() {
5510                        formula_cells.insert((addr.sheet_id, addr.coord.row(), addr.coord.col()));
5511                    }
5512                }
5513                _ => {}
5514            }
5515        }
5516        let stats = self.graph.baseline_stats();
5517        let final_vertices = i128::try_from(stats.graph_vertex_count)
5518            .ok()
5519            .and_then(|count| count.checked_add(vertex_delta))
5520            .and_then(|count| usize::try_from(count).ok())
5521            .ok_or_else(|| {
5522                crate::engine::EditorError::Excel(
5523                    ExcelError::new(ExcelErrorKind::NImpl)
5524                        .with_message("replay vertex count overflow"),
5525                )
5526            })?;
5527        let final_edges = i128::try_from(stats.graph_edge_count)
5528            .ok()
5529            .and_then(|count| count.checked_add(edge_delta))
5530            .and_then(|count| usize::try_from(count).ok())
5531            .ok_or_else(|| {
5532                crate::engine::EditorError::Excel(
5533                    ExcelError::new(ExcelErrorKind::NImpl)
5534                        .with_message("replay edge count overflow"),
5535                )
5536            })?;
5537        self.preflight_graph_admission(crate::engine::resource_ledger::GraphAdmission {
5538            final_vertices,
5539            final_edges,
5540            materialization_cells: formula_cells.len() as u64,
5541            added_vertices,
5542            added_edges,
5543        })
5544        .map_err(crate::engine::EditorError::Excel)
5545    }
5546
5547    /// Undo the last group still retained by the provided audit log.
5548    ///
5549    /// Disabled, zero-cap, and evicted history is unavailable on this index-based path. Use an
5550    /// explicit `ActionJournal` with `undo_action` when undo must be independent of audit retention.
5551    pub fn undo_logged(
5552        &mut self,
5553        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5554        log: &mut crate::engine::ChangeLog,
5555    ) -> Result<(), crate::engine::EditorError> {
5556        let pending_events = log
5557            .last_group_indices()
5558            .into_iter()
5559            .map(|index| log.events()[index].clone())
5560            .collect::<Vec<_>>();
5561        self.preflight_replay_admission(&pending_events, false)?;
5562        let invalidation_baseline = self.invalidation_baseline();
5563        let names = Self::name_event_names(&pending_events);
5564        let prepared =
5565            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5566        self.commit_name_dependent_span_demotion(prepared)?;
5567        // UndoEngine can fail after partially applying the batch, so publish
5568        // invalidation before replay rather than only on the success path.
5569        self.invalidate_for_change_events(
5570            &pending_events,
5571            LoggedEditDirection::InverseReplay,
5572            invalidation_baseline,
5573        );
5574        let batch = undo.undo(&mut self.graph, log)?;
5575        for item in batch.iter().rev() {
5576            self.apply_inverse_row_visibility_event(&item.event);
5577            self.apply_inverse_staged_formula_event(&item.event);
5578        }
5579        if !batch.is_empty() {
5580            let events = batch
5581                .iter()
5582                .map(|item| item.event.clone())
5583                .collect::<Vec<_>>();
5584            self.clear_logged_cell_format_states(&events);
5585        }
5586        self.mirror_undo_batch_to_arrow(&batch);
5587        if !batch.is_empty() {
5588            for item in &batch {
5589                self.record_formula_plane_change_for_event(&item.event);
5590            }
5591        }
5592        Ok(())
5593    }
5594
5595    pub fn redo_logged(
5596        &mut self,
5597        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5598        log: &mut crate::engine::ChangeLog,
5599    ) -> Result<(), crate::engine::EditorError> {
5600        let pending_events = undo.pending_redo_events();
5601        self.preflight_replay_admission(&pending_events, true)?;
5602        let invalidation_baseline = self.invalidation_baseline();
5603        let names = Self::name_event_names(&pending_events);
5604        let prepared =
5605            self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str))?;
5606        self.commit_name_dependent_span_demotion(prepared)?;
5607        self.invalidate_for_change_events(
5608            &pending_events,
5609            LoggedEditDirection::ForwardReplay,
5610            invalidation_baseline,
5611        );
5612        let batch = undo.redo(&mut self.graph, log)?;
5613        for item in &batch {
5614            self.apply_forward_row_visibility_event(&item.event);
5615            self.apply_forward_staged_formula_event(&item.event);
5616        }
5617        if !batch.is_empty() {
5618            let events = batch
5619                .iter()
5620                .map(|item| item.event.clone())
5621                .collect::<Vec<_>>();
5622            self.clear_logged_cell_format_states(&events);
5623        }
5624        self.mirror_redo_batch_to_arrow(&batch);
5625        if !batch.is_empty() {
5626            for item in &batch {
5627                self.record_formula_plane_change_for_event(&item.event);
5628            }
5629        }
5630        Ok(())
5631    }
5632
5633    /// Undo the last committed atomic action using the journal stack.
5634    ///
5635    /// This path does not require a `ChangeLog`.
5636    pub fn undo_action(
5637        &mut self,
5638        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5639    ) -> Result<(), crate::engine::EditorError> {
5640        let Some(journal) = undo.pop_undo_action() else {
5641            return Ok(());
5642        };
5643        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, false) {
5644            undo.push_done_action(journal);
5645            return Err(error);
5646        }
5647        let invalidation_baseline = self.invalidation_baseline();
5648        let names = Self::name_event_names(&journal.graph.events);
5649        let prepared =
5650            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5651                Ok(prepared) => prepared,
5652                Err(error) => {
5653                    undo.push_done_action(journal);
5654                    return Err(error);
5655                }
5656            };
5657        if let Err(error) = self.commit_name_dependent_span_demotion(prepared) {
5658            undo.push_done_action(journal);
5659            return Err(error);
5660        }
5661
5662        self.invalidate_for_action_journal(
5663            &journal,
5664            LoggedEditDirection::InverseReplay,
5665            invalidation_baseline,
5666        );
5667        journal.graph.undo(&mut self.graph)?;
5668        self.apply_inverse_row_visibility_events(&journal.graph.events);
5669        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ true);
5670        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5671            for event in &journal.graph.events {
5672                self.record_formula_plane_change_for_event(event);
5673            }
5674        }
5675
5676        undo.push_redo_action(journal);
5677        Ok(())
5678    }
5679
5680    /// Redo the last undone atomic action using the journal stack.
5681    ///
5682    /// This path does not require a `ChangeLog`.
5683    pub fn redo_action(
5684        &mut self,
5685        undo: &mut crate::engine::graph::editor::undo_engine::UndoEngine,
5686    ) -> Result<(), crate::engine::EditorError> {
5687        let Some(journal) = undo.pop_redo_action() else {
5688            return Ok(());
5689        };
5690        if let Err(error) = self.preflight_replay_admission(&journal.graph.events, true) {
5691            undo.push_redo_action(journal);
5692            return Err(error);
5693        }
5694        let invalidation_baseline = self.invalidation_baseline();
5695        let names = Self::name_event_names(&journal.graph.events);
5696        let prepared =
5697            match self.prepare_name_dependent_span_demotion(names.iter().map(String::as_str)) {
5698                Ok(prepared) => prepared,
5699                Err(error) => {
5700                    undo.push_redo_action(journal);
5701                    return Err(error);
5702                }
5703            };
5704        if let Err(error) = self.commit_name_dependent_span_demotion(prepared) {
5705            undo.push_redo_action(journal);
5706            return Err(error);
5707        }
5708
5709        self.invalidate_for_action_journal(
5710            &journal,
5711            LoggedEditDirection::ForwardReplay,
5712            invalidation_baseline,
5713        );
5714        journal.graph.redo(&mut self.graph)?;
5715        self.apply_forward_row_visibility_events(&journal.graph.events);
5716        self.apply_arrow_undo_batch(&journal.arrow, /*undo=*/ false);
5717        if !journal.graph.is_empty() || !journal.arrow.is_empty() {
5718            for event in &journal.graph.events {
5719                self.record_formula_plane_change_for_event(event);
5720            }
5721        }
5722
5723        undo.push_done_action(journal);
5724        Ok(())
5725    }
5726
5727    fn cellref_to_sheet_row_col(&self, addr: &crate::reference::CellRef) -> (String, u32, u32) {
5728        let sheet = self.graph.sheet_name(addr.sheet_id).to_string();
5729        // Coord stores 0-based indices.
5730        let row = addr.coord.row() + 1;
5731        let col = addr.coord.col() + 1;
5732        (sheet, row, col)
5733    }
5734
5735    fn mirror_undo_batch_to_arrow(
5736        &mut self,
5737        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5738    ) {
5739        // Undo applies inverses in reverse order.
5740        for item in batch.iter().rev() {
5741            self.mirror_inverse_change_to_arrow(&item.event);
5742        }
5743    }
5744
5745    fn mirror_redo_batch_to_arrow(
5746        &mut self,
5747        batch: &[crate::engine::graph::editor::undo_engine::UndoBatchItem],
5748    ) {
5749        // Redo applies events in forward order.
5750        for item in batch.iter() {
5751            self.mirror_forward_change_to_arrow(&item.event);
5752        }
5753    }
5754
5755    fn mirror_inverse_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5756        use crate::engine::ChangeEvent;
5757        use formualizer_common::LiteralValue;
5758
5759        match ev {
5760            ChangeEvent::SetValue {
5761                addr,
5762                old_value,
5763                old_formula,
5764                ..
5765            } => {
5766                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5767                if old_formula.is_some() {
5768                    self.clear_delta_overlay_cell(&sheet, row, col);
5769                } else {
5770                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5771                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5772                }
5773            }
5774            ChangeEvent::SetFormula {
5775                addr,
5776                old_value,
5777                old_formula,
5778                ..
5779            } => {
5780                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5781                if old_formula.is_some() {
5782                    self.clear_delta_overlay_cell(&sheet, row, col);
5783                } else {
5784                    let v = old_value.clone().unwrap_or(LiteralValue::Empty);
5785                    self.mirror_value_to_overlay(&sheet, row, col, &v);
5786                }
5787            }
5788            ChangeEvent::SpillCommitted { old, new, .. } => {
5789                // Inverse: restore `old` (or clear if none).
5790                self.mirror_spill_snapshot(new, /*clear_only=*/ true);
5791                if let Some(snap) = old {
5792                    self.mirror_spill_snapshot(snap, /*clear_only=*/ false);
5793                }
5794            }
5795            ChangeEvent::SpillCleared { old, .. } => {
5796                // Inverse: restore prior spill.
5797                self.mirror_spill_snapshot(old, /*clear_only=*/ false);
5798            }
5799            ChangeEvent::SetRowVisibility { .. } => {
5800                // Engine-side metadata only; no Arrow overlay effect.
5801            }
5802            _ => {}
5803        }
5804    }
5805
5806    fn mirror_forward_change_to_arrow(&mut self, ev: &crate::engine::ChangeEvent) {
5807        use crate::engine::ChangeEvent;
5808
5809        match ev {
5810            ChangeEvent::SetValue { addr, new, .. } => {
5811                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5812                self.mirror_value_to_overlay(&sheet, row, col, new);
5813            }
5814            ChangeEvent::SetFormula { addr, .. } => {
5815                let (sheet, row, col) = self.cellref_to_sheet_row_col(addr);
5816                self.clear_delta_overlay_cell(&sheet, row, col);
5817                // Keep any computed overlay for this cell as-is; it will be recomputed on demand.
5818            }
5819            ChangeEvent::SpillCommitted { old, new, .. } => {
5820                if let Some(snap) = old {
5821                    self.mirror_spill_snapshot(snap, /*clear_only=*/ true);
5822                }
5823                self.mirror_spill_snapshot(new, /*clear_only=*/ false);
5824            }
5825            ChangeEvent::SpillCleared { old, .. } => {
5826                self.mirror_spill_snapshot(old, /*clear_only=*/ true);
5827            }
5828            ChangeEvent::SetRowVisibility { .. } => {
5829                // Engine-side metadata only; no Arrow overlay effect.
5830            }
5831            _ => {
5832                // Other graph structural operations do not have direct value effects in Arrow.
5833            }
5834        }
5835    }
5836
5837    fn mirror_spill_snapshot(
5838        &mut self,
5839        snap: &crate::engine::graph::editor::change_log::SpillSnapshot,
5840        clear_only: bool,
5841    ) {
5842        use formualizer_common::LiteralValue;
5843
5844        let mut i = 0usize;
5845        for row in &snap.values {
5846            for v in row {
5847                if let Some(cell) = snap.target_cells.get(i) {
5848                    let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5849                    let out = if clear_only {
5850                        LiteralValue::Empty
5851                    } else {
5852                        v.clone()
5853                    };
5854                    self.mirror_value_to_computed_overlay(&sheet, r, c, &out);
5855                }
5856                i += 1;
5857            }
5858        }
5859        // If target_cells is longer than values (should not happen), clear remaining cells.
5860        if clear_only {
5861            for cell in snap.target_cells.iter().skip(i) {
5862                let (sheet, r, c) = self.cellref_to_sheet_row_col(cell);
5863                self.mirror_value_to_computed_overlay(&sheet, r, c, &LiteralValue::Empty);
5864            }
5865        }
5866    }
5867
5868    pub fn set_default_sheet_by_name(&mut self, name: &str) {
5869        self.graph.set_default_sheet_by_name(name);
5870    }
5871
5872    pub fn set_default_sheet_by_id(&mut self, id: SheetId) {
5873        self.graph.set_default_sheet_by_id(id);
5874    }
5875
5876    pub fn set_sheet_index_mode(&mut self, mode: crate::engine::SheetIndexMode) {
5877        self.graph.set_sheet_index_mode(mode);
5878    }
5879
5880    #[cfg(feature = "test-support")]
5881    #[doc(hidden)]
5882    pub fn mark_all_formulas_dirty_for_test(&mut self) {
5883        self.mark_all_formula_vertices_dirty();
5884        self.graph
5885            .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
5886    }
5887
5888    #[cfg(feature = "test-support")]
5889    #[doc(hidden)]
5890    pub fn take_criteria_mask_work_for_test() -> (usize, usize) {
5891        criteria_mask_test_hooks::take_mask_work()
5892    }
5893
5894    #[cfg(feature = "test-support")]
5895    #[doc(hidden)]
5896    pub fn lookup_index_cache_report_for_test(&self) -> LookupIndexCacheReport {
5897        self.lookup_index_cache.report()
5898    }
5899
5900    #[cfg(any(test, feature = "benchmark_internal"))]
5901    #[doc(hidden)]
5902    pub fn reset_recalc_reuse_probe(&mut self) {
5903        *self.recalc_reuse_probe.get_mut().unwrap() = RecalcReuseProbe::default();
5904    }
5905
5906    #[cfg(any(test, feature = "benchmark_internal"))]
5907    #[doc(hidden)]
5908    pub fn recalc_reuse_probe(&self) -> RecalcReuseProbe {
5909        let mut probe = self.recalc_reuse_probe.lock().unwrap().clone();
5910        if let Some(cached) = self.cached_static_schedule.as_ref() {
5911            probe.schedule_retained_bytes = std::mem::size_of::<CachedScheduleEntry>()
5912                + cached.candidate_vertices.capacity() * std::mem::size_of::<VertexId>()
5913                + std::mem::size_of::<crate::engine::Schedule>()
5914                + 2 * std::mem::size_of::<usize>()
5915                + schedule_probe_retained_bytes(&cached.schedule);
5916        }
5917        probe
5918    }
5919
5920    #[cfg(test)]
5921    pub(crate) fn cached_static_schedule_for_test(&self) -> Option<Arc<crate::engine::Schedule>> {
5922        self.cached_static_schedule
5923            .as_ref()
5924            .map(|cached| Arc::clone(&cached.schedule))
5925    }
5926
5927    fn clear_cached_static_schedule(&mut self) {
5928        self.cached_static_schedule = None;
5929    }
5930
5931    fn invalidation_baseline(&self) -> InvalidationBaseline {
5932        InvalidationBaseline {
5933            snapshot_id: self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed),
5934            topology_epoch: self.topology_epoch,
5935        }
5936    }
5937
5938    fn classify_change_events(
5939        events: &[crate::engine::ChangeEvent],
5940        direction: LoggedEditDirection,
5941    ) -> LoggedEditImpact {
5942        use crate::engine::ChangeEvent;
5943
5944        events
5945            .iter()
5946            .map(|event| match event {
5947                ChangeEvent::CompoundStart { .. } | ChangeEvent::CompoundEnd { .. } => {
5948                    LoggedEditImpact::NoOp
5949                }
5950                ChangeEvent::SetRowVisibility { .. }
5951                | ChangeEvent::SetValue {
5952                    old_formula: None,
5953                    old_value: Some(_),
5954                    ..
5955                } => LoggedEditImpact::DataOnly,
5956                // Original canonical writes can lack an Arrow old value while
5957                // updating an existing placeholder. Inverse replay actually
5958                // removes that vertex; a later redo recreates it.
5959                ChangeEvent::SetValue {
5960                    old_formula: None,
5961                    old_value: None,
5962                    ..
5963                } if direction == LoggedEditDirection::Original => LoggedEditImpact::DataOnly,
5964                ChangeEvent::SetValue { .. }
5965                | ChangeEvent::SetFormula { .. }
5966                | ChangeEvent::AddVertex { .. }
5967                | ChangeEvent::RemoveVertex { .. }
5968                | ChangeEvent::VertexMoved { .. }
5969                | ChangeEvent::FormulaAdjusted { .. }
5970                | ChangeEvent::NamedRangeAdjusted { .. }
5971                | ChangeEvent::EdgeAdded { .. }
5972                | ChangeEvent::EdgeRemoved { .. }
5973                | ChangeEvent::DefineName { .. }
5974                | ChangeEvent::UpdateName { .. }
5975                | ChangeEvent::DeleteName { .. }
5976                | ChangeEvent::SpillCommitted { .. }
5977                | ChangeEvent::SpillCleared { .. }
5978                | ChangeEvent::StagedFormulaCellChanged { .. } => LoggedEditImpact::Topology,
5979            })
5980            .max()
5981            .unwrap_or(LoggedEditImpact::NoOp)
5982    }
5983
5984    fn classify_arrow_undo(arrow: &crate::engine::ArrowUndoBatch) -> LoggedEditImpact {
5985        use crate::engine::ArrowOp;
5986
5987        arrow
5988            .ops
5989            .iter()
5990            .map(|op| match op {
5991                ArrowOp::SetDeltaCell { .. } | ArrowOp::SetComputedCell { .. } => {
5992                    LoggedEditImpact::DataOnly
5993                }
5994                ArrowOp::RestoreComputedRect { .. }
5995                | ArrowOp::InsertRows { .. }
5996                | ArrowOp::InsertCols { .. } => LoggedEditImpact::Topology,
5997            })
5998            .max()
5999            .unwrap_or(LoggedEditImpact::NoOp)
6000    }
6001
6002    fn apply_logged_edit_impact(
6003        &mut self,
6004        impact: LoggedEditImpact,
6005        baseline: InvalidationBaseline,
6006    ) {
6007        match impact {
6008            LoggedEditImpact::NoOp => {}
6009            LoggedEditImpact::DataOnly => {
6010                if self.topology_epoch == baseline.topology_epoch
6011                    && self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
6012                        == baseline.snapshot_id
6013                {
6014                    self.mark_data_edited();
6015                }
6016            }
6017            LoggedEditImpact::Topology => {
6018                // FormulaPlane demotion and some structural entry points already
6019                // publish topology invalidation. Do not bump the same batch twice.
6020                if self.topology_epoch == baseline.topology_epoch {
6021                    self.mark_topology_edited();
6022                }
6023            }
6024        }
6025    }
6026
6027    fn invalidate_for_change_events(
6028        &mut self,
6029        events: &[crate::engine::ChangeEvent],
6030        direction: LoggedEditDirection,
6031        baseline: InvalidationBaseline,
6032    ) {
6033        self.apply_logged_edit_impact(Self::classify_change_events(events, direction), baseline);
6034    }
6035
6036    fn invalidate_for_action_journal(
6037        &mut self,
6038        journal: &crate::engine::ActionJournal,
6039        direction: LoggedEditDirection,
6040        baseline: InvalidationBaseline,
6041    ) {
6042        let impact = Self::classify_change_events(&journal.graph.events, direction)
6043            .max(Self::classify_arrow_undo(&journal.arrow));
6044        self.apply_logged_edit_impact(impact, baseline);
6045    }
6046
6047    /// Mark data edited: bump snapshot and set edited flag.
6048    /// Value-only edits keep the stable-topology schedule cache alive.
6049    pub fn mark_data_edited(&mut self) {
6050        self.lookup_index_cache.clear();
6051        self.snapshot_id
6052            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6053        self.has_edited = true;
6054    }
6055
6056    /// Mark a topology-changing edit: bump snapshot + topology epoch and invalidate cached schedules.
6057    pub fn mark_topology_edited(&mut self) {
6058        self.lookup_index_cache.clear();
6059        self.snapshot_id
6060            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
6061        self.topology_epoch = self.topology_epoch.wrapping_add(1);
6062        self.graph.bump_topology_revision();
6063        self.clear_cached_static_schedule();
6064        self.cached_mixed_topology = None;
6065        if let Some(ledger) = self.active_resource_ledger.as_mut() {
6066            let released = ledger.account_mixed_cache(0);
6067            debug_assert!(released.is_ok());
6068        }
6069        self.has_edited = true;
6070    }
6071
6072    fn mark_all_formula_vertices_dirty(&mut self) {
6073        let vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
6074        for vertex in vertices {
6075            self.graph.mark_vertex_dirty(vertex);
6076        }
6077    }
6078
6079    fn mark_moved_formula_vertices_dirty(
6080        &mut self,
6081        summary: &crate::engine::graph::editor::vertex_editor::ShiftSummary,
6082    ) {
6083        for vertex in &summary.vertices_moved {
6084            if self.graph.get_formula_id(*vertex).is_some() {
6085                self.graph.mark_vertex_dirty(*vertex);
6086            }
6087        }
6088    }
6089
6090    /// Access Arrow sheet store (read-only)
6091    pub fn sheet_store(&self) -> &SheetStore {
6092        &self.arrow_sheets
6093    }
6094
6095    /// True when any sheet carries manual/filter row-visibility state.
6096    /// Used by load-time freshness checks (see `Engine::adopt_file_sheets`).
6097    pub(crate) fn has_row_visibility_state(&self) -> bool {
6098        !self.row_visibility.is_empty()
6099    }
6100
6101    /// Access Arrow sheet store (mutable)
6102    pub fn sheet_store_mut(&mut self) -> &mut SheetStore {
6103        &mut self.arrow_sheets
6104    }
6105
6106    pub fn has_staged_formulas(&self) -> bool {
6107        !self.staged_formulas.is_empty()
6108    }
6109
6110    pub fn staged_formula_count(&self) -> usize {
6111        self.staged_formulas.values().map(StagedSheet::len).sum()
6112    }
6113
6114    /// Stage a formula text instead of inserting into the graph (used when deferring is enabled).
6115    pub fn stage_formula_text(&mut self, sheet: &str, row: u32, col: u32, text: String) {
6116        self.staged_formulas
6117            .entry(sheet.to_string())
6118            .or_default()
6119            .stage(row, col, text);
6120        self.staged_formula_index.stage(sheet, row, col);
6121    }
6122
6123    fn index_removed_staged_sheet(&mut self, sheet: &str, staged: &StagedSheet) {
6124        for (row, col, _) in &staged.entries {
6125            self.staged_formula_index.remove(sheet, *row, *col);
6126        }
6127        if staged.deferred_package.is_some() {
6128            self.staged_formula_index.set_package(sheet, None);
6129        }
6130    }
6131
6132    fn restore_staged_sheet(&mut self, sheet: String, staged: StagedSheet) {
6133        self.staged_formulas.insert(sheet, staged);
6134    }
6135
6136    #[doc(hidden)]
6137    pub fn source_formula_ingress(&mut self) -> SourceFormulaIngress<'_, R> {
6138        SourceFormulaIngress { engine: self }
6139    }
6140
6141    #[doc(hidden)]
6142    /// Test-only fault-injection seam. Not part of the supported API; it exists so
6143    /// integration tests in sibling crates can fail a commit at an exact point.
6144    #[doc(hidden)]
6145    #[cfg(any(test, feature = "test-support"))]
6146    pub fn set_before_prepared_span_commit_hook(
6147        &mut self,
6148        hook: impl FnOnce() + Send + Sync + 'static,
6149    ) {
6150        self.before_prepared_span_commit_hook = Some(Box::new(hook));
6151    }
6152
6153    #[doc(hidden)]
6154    /// Test-only fault-injection seam, matching `set_after_eager_proposal_commit_hook`.
6155    #[cfg(test)]
6156    pub(crate) fn set_before_target_preparation_commit_hook(
6157        &mut self,
6158        hook: impl FnOnce() + Send + Sync + 'static,
6159    ) {
6160        self.before_target_preparation_commit_hook = Some(Box::new(hook));
6161    }
6162
6163    fn stage_deferred_formula_package(&mut self, package: crate::engine::DeferredFormulaPackage) {
6164        let sheet = package.sheet_name.clone();
6165        let staged = self.staged_formulas.entry(sheet.clone()).or_default();
6166        debug_assert!(staged.deferred_package.is_none());
6167        staged.deferred_package = Some(package);
6168        staged.reconcile_attached_deferred_package();
6169        self.staged_formula_index
6170            .set_package(&sheet, staged.deferred_package.as_ref());
6171    }
6172
6173    pub fn clear_staged_formula_text(&mut self, sheet: &str, row: u32, col: u32) -> Option<String> {
6174        let mut removed = None;
6175        let mut remove_sheet = false;
6176        let mut had_package = false;
6177        if let Some(entries) = self.staged_formulas.get_mut(sheet) {
6178            had_package = entries.deferred_package.is_some();
6179            removed = entries.remove(row, col);
6180            remove_sheet = entries.is_empty();
6181        }
6182        let ordinary_removed = self.staged_formula_index.remove(sheet, row, col);
6183        if !ordinary_removed && had_package {
6184            self.staged_formula_index.touch_package(sheet);
6185        }
6186        if remove_sheet {
6187            self.staged_formulas.remove(sheet);
6188            self.staged_formula_index.set_package(sheet, None);
6189        }
6190        removed
6191    }
6192
6193    pub fn clear_staged_formulas_for_sheet(&mut self, sheet: &str) {
6194        if self.staged_formulas.remove(sheet).is_some() {
6195            self.staged_formula_index.clear_sheet(sheet);
6196        }
6197    }
6198
6199    pub fn rename_staged_formula_sheet(&mut self, old: &str, new: &str) {
6200        let Some(entries) = self.staged_formulas.remove(old) else {
6201            return;
6202        };
6203        self.staged_formula_index.clear_sheet(old);
6204        let (formulas, mut package) = entries.into_parts();
6205        for (row, col, text) in formulas {
6206            self.stage_formula_text(new, row, col, text);
6207        }
6208        if let Some(package) = package.as_mut() {
6209            package.sheet_name = new.to_string();
6210        }
6211        if let Some(package) = package {
6212            self.stage_deferred_formula_package(package);
6213        }
6214    }
6215
6216    /// Get a staged formula text for a given cell if present (cloned).
6217    pub fn get_staged_formula_text(&self, sheet: &str, row: u32, col: u32) -> Option<String> {
6218        self.staged_formulas
6219            .get(sheet)
6220            .and_then(|v| v.get(row, col))
6221    }
6222
6223    pub fn formula_parse_diagnostics(&self) -> &[FormulaParseDiagnostic] {
6224        &self.formula_parse_diagnostics
6225    }
6226
6227    pub fn take_formula_parse_diagnostics(&mut self) -> Vec<FormulaParseDiagnostic> {
6228        std::mem::take(&mut self.formula_parse_diagnostics)
6229    }
6230
6231    pub fn clear_formula_parse_diagnostics(&mut self) {
6232        self.formula_parse_diagnostics.clear();
6233    }
6234
6235    pub fn last_formula_ingest_report(&self) -> Option<&FormulaIngestReport> {
6236        self.last_formula_ingest_report.as_ref()
6237    }
6238
6239    pub fn formula_ingest_report_total(&self) -> &FormulaIngestReport {
6240        &self.formula_ingest_report_total
6241    }
6242
6243    #[cfg(test)]
6244    pub(crate) fn set_before_target_planning_snapshot_hook_for_test(
6245        &mut self,
6246        hook: impl FnOnce() + Send + Sync + 'static,
6247    ) {
6248        self.before_target_planning_snapshot_hook = Some(Box::new(hook));
6249    }
6250
6251    #[cfg(test)]
6252    pub(crate) fn inject_target_semantic_stale_once_for_test(&mut self) {
6253        self.inject_target_semantic_stale_once_for_test = true;
6254    }
6255
6256    #[cfg(test)]
6257    pub(crate) fn force_virtual_dep_changes_for_test(&mut self, rounds: usize) {
6258        self.force_virtual_dep_changes_remaining_for_test = rounds;
6259    }
6260
6261    #[cfg(test)]
6262    pub(crate) fn fail_evaluation_commit_preflight_once_for_test(&mut self) {
6263        self.fail_evaluation_commit_preflight_once_for_test = true;
6264    }
6265
6266    #[cfg(test)]
6267    pub(crate) fn cancel_before_formula_plane_layer_commit_once_for_test(&mut self) {
6268        self.cancel_before_formula_plane_layer_commit_once_for_test = true;
6269    }
6270
6271    #[cfg(test)]
6272    pub(crate) fn set_target_preparation_fault_for_test(
6273        &mut self,
6274        fault: crate::engine::target_preparation::TargetPreparationFault,
6275    ) {
6276        self.target_preparation_fault_for_test = Some(fault);
6277    }
6278
6279    #[cfg(test)]
6280    pub(crate) fn staged_formula_index_revision_for_test(&self) -> u64 {
6281        self.staged_formula_index.revision()
6282    }
6283
6284    #[cfg(test)]
6285    pub(crate) fn staged_formula_index_is_consistent_for_test(&self) -> bool {
6286        let ordinary_storage = self
6287            .staged_formulas
6288            .values()
6289            .map(|sheet| sheet.entries.len())
6290            .sum::<usize>();
6291        let package_storage = self
6292            .staged_formulas
6293            .values()
6294            .filter(|sheet| sheet.deferred_package.is_some())
6295            .count();
6296        ordinary_storage == self.staged_formula_index.ordinary_count()
6297            && package_storage == self.staged_formula_index.package_count()
6298            && self.staged_formulas.iter().all(|(name, sheet)| {
6299                sheet.entries.iter().all(|(row, col, _)| {
6300                    let leases = self
6301                        .staged_formula_index
6302                        .leases_in_region(name, *row, *col, *row, *col);
6303                    leases.len() == 1 && leases[0].row == *row && leases[0].col == *col
6304                })
6305            })
6306    }
6307
6308    #[cfg(test)]
6309    pub(crate) fn last_formula_plane_span_eval_report(&self) -> Option<&SpanEvalReport> {
6310        self.last_formula_plane_span_eval_report.as_ref()
6311    }
6312
6313    #[cfg(test)]
6314    pub(crate) fn evaluation_request_begin_count_for_test(&self) -> u64 {
6315        self.evaluation_request_begin_count_for_test
6316    }
6317
6318    #[cfg(test)]
6319    pub(crate) fn force_source_family_fallback_for_test(&mut self, force: bool) {
6320        self.force_source_family_fallback = force;
6321    }
6322
6323    #[cfg(test)]
6324    pub(crate) fn rerecord_cycle_retry_span_after_lease_extension_for_test(&mut self) {
6325        self.rerecord_cycle_retry_span_after_lease_extension_for_test = true;
6326    }
6327
6328    #[cfg(test)]
6329    pub(crate) fn set_fragmented_commit_fault_for_test(
6330        &mut self,
6331        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
6332    ) {
6333        self.fragmented_commit_fault_for_test = Some(fault);
6334    }
6335
6336    #[cfg(test)]
6337    pub(crate) fn set_before_legacy_fallback_final_provider_sample_hook(
6338        &mut self,
6339        hook: impl FnOnce() + Send + Sync + 'static,
6340    ) {
6341        self.before_legacy_fallback_final_provider_sample_hook = Some(Box::new(hook));
6342    }
6343
6344    #[cfg(test)]
6345    pub(crate) fn set_after_eager_proposal_commit_hook(
6346        &mut self,
6347        hook: impl FnOnce() + Send + Sync + 'static,
6348    ) {
6349        self.after_eager_proposal_commit_hook = Some(Box::new(hook));
6350    }
6351
6352    #[cfg(test)]
6353    pub(crate) fn formula_plane_indexes_epoch(&self) -> u64 {
6354        self.graph.formula_authority().indexes_epoch()
6355    }
6356
6357    #[cfg(test)]
6358    pub(crate) fn formula_plane_capacity_bailouts(&self) -> u64 {
6359        self.formula_plane_capacity_bailouts
6360    }
6361
6362    #[cfg(test)]
6363    pub(crate) fn topology_epoch_for_test(&self) -> u64 {
6364        self.topology_epoch
6365    }
6366
6367    #[cfg(test)]
6368    pub(crate) fn graph_topology_revision_for_test(&self) -> u64 {
6369        self.graph.topology_revision()
6370    }
6371
6372    #[cfg(test)]
6373    pub(crate) fn mixed_topology_cache_present_for_test(&self) -> bool {
6374        self.cached_mixed_topology.is_some()
6375    }
6376
6377    fn record_formula_ingest_report(&mut self, report: FormulaIngestReport) {
6378        self.formula_ingest_report_total.mode = report.mode;
6379        self.formula_ingest_report_total.accumulate(&report);
6380        self.last_formula_ingest_report = Some(report);
6381    }
6382
6383    fn prepare_source_formula_family(
6384        &mut self,
6385        sheet_id: SheetId,
6386        family: &crate::engine::SourceFormulaFamily,
6387        allow_function_closure: bool,
6388        planning_provider: Option<&dyn crate::traits::FunctionProvider>,
6389    ) -> Result<
6390        (
6391            crate::formula_plane::placement::PreparedAnchorOncePlacement,
6392            bool,
6393        ),
6394        SourceFamilyPreparationError,
6395    > {
6396        let transport = family
6397            .validated_complete_domain(&self.workbook_load_limits)
6398            .map_err(SourceFamilyPreparationError::contract)?;
6399        let domain = match transport {
6400            crate::engine::PlacementDomainTransport::RowRun {
6401                row_start,
6402                row_end,
6403                col,
6404            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
6405            crate::engine::PlacementDomainTransport::ColRun {
6406                row,
6407                col_start,
6408                col_end,
6409            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
6410            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
6411                sheet_id,
6412                rect.start.row,
6413                rect.end.row,
6414                rect.start.col,
6415                rect.end.col,
6416            ),
6417        };
6418        let formula = if family.anchor_text.starts_with('=') {
6419            family.anchor_text.to_string()
6420        } else {
6421            format!("={}", family.anchor_text)
6422        };
6423        let ast = formualizer_parse::parser::parse(&formula)
6424            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6425        let relocation = if allow_function_closure {
6426            validate_anchor_once_shadow_relocation(
6427                &ast,
6428                family.anchor_coord0.row,
6429                family.anchor_coord0.col,
6430                &domain,
6431                planning_provider.unwrap_or(&self.resolver),
6432            )
6433        } else {
6434            validate_anchor_once_syntax(
6435                &ast,
6436                family.anchor_coord0.row,
6437                family.anchor_coord0.col,
6438                &domain,
6439            )
6440        };
6441        relocation.map_err(SourceFamilyPreparationError::ast)?;
6442        let ast_id = self.intern_formula_ast(&ast);
6443        let placement = CellRef::new(
6444            sheet_id,
6445            Coord::from_excel(
6446                family.anchor_coord0.row + 1,
6447                family.anchor_coord0.col + 1,
6448                true,
6449                true,
6450            ),
6451        );
6452        let ingested = {
6453            let mut pipeline = match planning_provider {
6454                Some(provider) => self.graph.ingest_pipeline(provider),
6455                None => self.ingest_pipeline(),
6456            };
6457            if allow_function_closure {
6458                pipeline = pipeline.enable_function_semantics();
6459            }
6460            pipeline.ingest_formula(
6461                FormulaAstInput::RawArena(ast_id),
6462                placement,
6463                Some(family.anchor_text.clone()),
6464            )
6465        }
6466        .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6467        let function_semantics_used = ingested
6468            .labels
6469            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6470        let candidate = FormulaPlacementCandidate::new(
6471            sheet_id,
6472            family.anchor_coord0.row,
6473            family.anchor_coord0.col,
6474            ingested.ast_id,
6475            Some(family.anchor_text.clone()),
6476        );
6477        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6478            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6479        prepare_anchor_once_family(candidate, analysis, domain, family.member_count)
6480            .map(|prepared| (prepared, function_semantics_used))
6481            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))
6482    }
6483
6484    fn placement_domain_from_transport(
6485        sheet_id: SheetId,
6486        transport: crate::engine::PlacementDomainTransport,
6487    ) -> PlacementDomain {
6488        match transport {
6489            crate::engine::PlacementDomainTransport::RowRun {
6490                row_start,
6491                row_end,
6492                col,
6493            } => PlacementDomain::row_run(sheet_id, row_start, row_end, col),
6494            crate::engine::PlacementDomainTransport::ColRun {
6495                row,
6496                col_start,
6497                col_end,
6498            } => PlacementDomain::col_run(sheet_id, row, col_start, col_end),
6499            crate::engine::PlacementDomainTransport::Rect(rect) => PlacementDomain::rect(
6500                sheet_id,
6501                rect.start.row,
6502                rect.end.row,
6503                rect.start.col,
6504                rect.end.col,
6505            ),
6506        }
6507    }
6508
6509    #[cfg(test)]
6510    pub(crate) fn analyze_fragmented_exact_replay_record_for_test(
6511        &mut self,
6512        sheet_name: &str,
6513        source_id: crate::engine::SourceFamilyId,
6514        expected: crate::engine::PartitionLegacyMember,
6515        replay: crate::engine::DeferredReplayFormula,
6516    ) -> Result<crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula, String>
6517    {
6518        let sheet_id = self.graph.sheet_id_mut(sheet_name);
6519        self.graph
6520            .analyze_fragmented_exact_replay_record(
6521                &self.resolver,
6522                sheet_id,
6523                source_id,
6524                expected,
6525                replay,
6526            )
6527            .map_err(|error| format!("{error:?}"))
6528    }
6529
6530    pub(crate) fn prepare_fragmented_source_transaction(
6531        &mut self,
6532        source: &crate::engine::PartitionedSourceFormulaFamily,
6533        disposition: &crate::engine::FormulaReplayDisposition,
6534        prepared: crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
6535        legacy_formulas: Vec<
6536            crate::engine::fragmented_transaction::PreparedFragmentedLegacyFormula,
6537        >,
6538    ) -> Result<
6539        crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6540        crate::engine::fragmented_transaction::FragmentedTransactionPrepareError,
6541    > {
6542        let transaction = self.graph.prepare_fragmented_source_transaction(
6543            &self.source_formula_token,
6544            source,
6545            disposition,
6546            prepared,
6547            legacy_formulas,
6548        )?;
6549        self.prepared_legacy_admission(
6550            transaction.legacy_graph(),
6551            transaction.materialization_cells(),
6552        )
6553        .map_err(
6554            crate::engine::fragmented_transaction::FragmentedTransactionPrepareError::Admission,
6555        )?;
6556        Ok(transaction)
6557    }
6558
6559    pub(crate) fn commit_fragmented_source_transaction(
6560        &mut self,
6561        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6562        disposition: &crate::engine::FormulaReplayDisposition,
6563    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6564        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6565        let provider_revision = self.resolver.planning_semantic_revision();
6566        let epoch = registry_guard.epoch();
6567        let resolver = &self.resolver;
6568        let decision = self.graph.commit_fragmented_source_transaction(
6569            prepared,
6570            &self.source_formula_token,
6571            disposition,
6572            epoch,
6573            provider_revision,
6574            || resolver.planning_semantic_revision(),
6575            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6576        );
6577        drop(registry_guard);
6578        decision
6579    }
6580
6581    #[cfg(test)]
6582    pub(crate) fn commit_fragmented_source_transaction_with_fault_for_test(
6583        &mut self,
6584        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6585        disposition: &crate::engine::FormulaReplayDisposition,
6586        fault: crate::engine::fragmented_transaction::FragmentedCommitFault,
6587    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6588        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6589        let provider_revision = self.resolver.planning_semantic_revision();
6590        let epoch = registry_guard.epoch();
6591        let resolver = &self.resolver;
6592        let decision = self.graph.commit_fragmented_source_transaction(
6593            prepared,
6594            &self.source_formula_token,
6595            disposition,
6596            epoch,
6597            provider_revision,
6598            || resolver.planning_semantic_revision(),
6599            fault,
6600        );
6601        drop(registry_guard);
6602        decision
6603    }
6604
6605    #[cfg(test)]
6606    pub(crate) fn commit_fragmented_source_transaction_with_provider_flip_for_test(
6607        &mut self,
6608        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6609        disposition: &crate::engine::FormulaReplayDisposition,
6610        before_second_sample: impl FnOnce(),
6611    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6612        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
6613        let provider_revision = self.resolver.planning_semantic_revision();
6614        let epoch = registry_guard.epoch();
6615        let resolver = &self.resolver;
6616        let decision = self.graph.commit_fragmented_source_transaction(
6617            prepared,
6618            &self.source_formula_token,
6619            disposition,
6620            epoch,
6621            provider_revision,
6622            || {
6623                before_second_sample();
6624                resolver.planning_semantic_revision()
6625            },
6626            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6627        );
6628        drop(registry_guard);
6629        decision
6630    }
6631
6632    #[cfg(test)]
6633    pub(crate) fn commit_fragmented_source_transaction_with_revisions_for_test(
6634        &mut self,
6635        prepared: crate::engine::fragmented_transaction::PreparedFragmentedSourceTransaction,
6636        disposition: &crate::engine::FormulaReplayDisposition,
6637        function_semantic_epoch: u64,
6638        function_provider_revision: Option<u64>,
6639    ) -> crate::engine::fragmented_transaction::FragmentedCommitDecision {
6640        self.graph.commit_fragmented_source_transaction(
6641            prepared,
6642            &self.source_formula_token,
6643            disposition,
6644            function_semantic_epoch,
6645            function_provider_revision,
6646            || function_provider_revision,
6647            crate::engine::fragmented_transaction::FragmentedCommitFault::None,
6648        )
6649    }
6650
6651    pub(crate) fn analyze_partitioned_source_family_for_transaction(
6652        &mut self,
6653        sheet_name: &str,
6654        family: &crate::engine::PartitionedSourceFormulaFamily,
6655    ) -> Result<
6656        crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily,
6657        SourceFamilyPreparationError,
6658    > {
6659        family
6660            .validate(&self.workbook_load_limits)
6661            .map_err(SourceFamilyPreparationError::contract)?;
6662        let formula = if family.template_text.starts_with('=') {
6663            family.template_text.to_string()
6664        } else {
6665            format!("={}", family.template_text)
6666        };
6667        let ast = formualizer_parse::parser::parse(&formula)
6668            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6669        let mut requests = Vec::new();
6670        Self::collect_planning_function_requests(&ast, &mut requests);
6671        let snapshot = crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
6672            &self.resolver,
6673            requests,
6674        )
6675        .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
6676        let sheet_id = self.graph.sheet_id_mut(sheet_name);
6677        let domains: Vec<_> = family
6678            .fragments
6679            .iter()
6680            .copied()
6681            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6682            .collect();
6683        for domain in &domains {
6684            validate_anchor_once_shadow_relocation(
6685                &ast,
6686                family.template_origin0.row,
6687                family.template_origin0.col,
6688                domain,
6689                &snapshot,
6690            )
6691            .map_err(SourceFamilyPreparationError::ast)?;
6692        }
6693        let ast_id = self.intern_formula_ast(&ast);
6694        let placement = CellRef::new(
6695            sheet_id,
6696            Coord::from_excel(
6697                family.template_origin0.row + 1,
6698                family.template_origin0.col + 1,
6699                true,
6700                true,
6701            ),
6702        );
6703        let ingested = self
6704            .graph
6705            .ingest_pipeline(&snapshot)
6706            .enable_function_semantics()
6707            .ingest_formula(
6708                FormulaAstInput::RawArena(ast_id),
6709                placement,
6710                Some(family.template_text.clone()),
6711            )
6712            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6713        let function_semantics_used = ingested
6714            .labels
6715            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6716        let name_assumptions = self
6717            .graph
6718            .capture_fragmented_name_assumptions(sheet_id, &ingested.dep_plan.resolved_named_refs)
6719            .map_err(|error| SourceFamilyPreparationError::ast(format!("{error:?}")))?;
6720        let candidate = FormulaPlacementCandidate::new(
6721            sheet_id,
6722            family.template_origin0.row,
6723            family.template_origin0.col,
6724            ingested.ast_id,
6725            Some(family.template_text.clone()),
6726        );
6727        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6728            .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6729        let direct_cells = domains.iter().try_fold(0u64, |total, domain| {
6730            total
6731                .checked_add(domain.cell_count())
6732                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))
6733        })?;
6734        let mut placements = Vec::with_capacity(domains.len());
6735        for domain in domains {
6736            let member_count = domain.cell_count();
6737            placements.push(
6738                prepare_anchor_once_fragment(
6739                    candidate.clone(),
6740                    analysis.clone(),
6741                    domain,
6742                    member_count,
6743                    direct_cells,
6744                )
6745                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?,
6746            );
6747        }
6748        let fallback_cells = family.legacy_members.shared_member_count() as u64;
6749        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6750            return Err(SourceFamilyPreparationError::analysis(
6751                "PartitionMemberCountMismatch",
6752            ));
6753        }
6754        Ok(
6755            crate::engine::fragmented_transaction::PreparedPartitionedSourceFamily {
6756                engine_token: Arc::clone(&self.source_formula_token),
6757                sheet_id,
6758                sheet_name: Arc::from(sheet_name),
6759                source: family.clone(),
6760                placements,
6761                function_semantic_epoch: snapshot.epoch(),
6762                function_provider_revision: snapshot.provider_revision(),
6763                function_semantics_used,
6764                name_assumptions,
6765                direct_cells,
6766            },
6767        )
6768    }
6769
6770    fn prepare_partitioned_source_formula_family_shadow(
6771        &mut self,
6772        sheet_id: SheetId,
6773        family: &crate::engine::PartitionedSourceFormulaFamily,
6774    ) -> Result<PreparedPartitionShadow, SourceFamilyPreparationError> {
6775        family
6776            .validate(&self.workbook_load_limits)
6777            .map_err(SourceFamilyPreparationError::contract)?;
6778        let formula = if family.template_text.starts_with('=') {
6779            family.template_text.to_string()
6780        } else {
6781            format!("={}", family.template_text)
6782        };
6783        let ast = formualizer_parse::parser::parse(&formula)
6784            .map_err(|_| SourceFamilyPreparationError::parse("AnchorParseRejected"))?;
6785        let domains: Vec<_> = family
6786            .fragments
6787            .iter()
6788            .copied()
6789            .map(|transport| Self::placement_domain_from_transport(sheet_id, transport))
6790            .collect();
6791        for domain in &domains {
6792            validate_anchor_once_shadow_relocation(
6793                &ast,
6794                family.template_origin0.row,
6795                family.template_origin0.col,
6796                domain,
6797                &self.resolver,
6798            )
6799            .map_err(SourceFamilyPreparationError::ast)?;
6800        }
6801        let ast_id = self.intern_formula_ast(&ast);
6802        let placement = CellRef::new(
6803            sheet_id,
6804            Coord::from_excel(
6805                family.template_origin0.row + 1,
6806                family.template_origin0.col + 1,
6807                true,
6808                true,
6809            ),
6810        );
6811        let ingested = self
6812            .ingest_pipeline()
6813            .enable_function_semantics()
6814            .ingest_formula(
6815                FormulaAstInput::RawArena(ast_id),
6816                placement,
6817                Some(family.template_text.clone()),
6818            )
6819            .map_err(|_| SourceFamilyPreparationError::ast("UnsupportedCanonicalTemplate"))?;
6820        let function_semantics_used = ingested
6821            .labels
6822            .has_flag(crate::engine::arena::ast::CanonicalLabels::FLAG_CONTAINS_FUNCTION);
6823        let candidate = FormulaPlacementCandidate::new(
6824            sheet_id,
6825            family.template_origin0.row,
6826            family.template_origin0.col,
6827            ingested.ast_id,
6828            Some(family.template_text.clone()),
6829        );
6830        let analysis = CandidateAnalysis::from_ingested(&candidate, &ingested)
6831            .map_err(|reason| SourceFamilyPreparationError::ast(format!("{reason:?}")))?;
6832        let fragment_count = domains.len() as u64;
6833        let mut direct_cells = 0u64;
6834        for domain in domains {
6835            let member_count = domain.cell_count();
6836            validate_anchor_once_fragment_shadow(&analysis, &domain, member_count)
6837                .map_err(|reason| SourceFamilyPreparationError::analysis(format!("{reason:?}")))?;
6838            direct_cells = direct_cells
6839                .checked_add(member_count)
6840                .ok_or_else(|| SourceFamilyPreparationError::analysis("PartitionAreaOverflow"))?;
6841        }
6842        let fallback_cells = family.legacy_members.shared_member_count() as u64;
6843        if direct_cells.checked_add(fallback_cells) != Some(family.surviving_member_count) {
6844            return Err(SourceFamilyPreparationError::analysis(
6845                "PartitionMemberCountMismatch",
6846            ));
6847        }
6848        Ok(PreparedPartitionShadow {
6849            fragment_count,
6850            direct_cells,
6851            fallback_cells,
6852            function_semantics_used,
6853        })
6854    }
6855
6856    fn analyze_partitioned_formula_plane_shadow(
6857        &mut self,
6858        batches: &[(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)],
6859    ) -> FormulaIngestReport {
6860        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6861        for (sheet_name, families) in batches {
6862            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6863            for family in families {
6864                report.shadow_candidate_cells = report
6865                    .shadow_candidate_cells
6866                    .saturating_add(family.surviving_member_count);
6867                match self.prepare_partitioned_source_formula_family_shadow(sheet_id, family) {
6868                    Ok(prepared) => {
6869                        let fragment_count = prepared.fragment_count;
6870                        let descendants = family.surviving_member_count.saturating_sub(1);
6871                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6872                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6873                        report.source_anchor_analyses =
6874                            report.source_anchor_analyses.saturating_add(1);
6875                        report.source_partition_analyses_reused = report
6876                            .source_partition_analyses_reused
6877                            .saturating_add(fragment_count.saturating_sub(1));
6878                        report.source_partitioned_families_prepared = report
6879                            .source_partitioned_families_prepared
6880                            .saturating_add(1);
6881                        report.source_partition_fragments_prepared = report
6882                            .source_partition_fragments_prepared
6883                            .saturating_add(fragment_count);
6884                        report.source_partition_span_cells_prepared = report
6885                            .source_partition_span_cells_prepared
6886                            .saturating_add(prepared.direct_cells);
6887                        report.source_partition_fallback_cells = report
6888                            .source_partition_fallback_cells
6889                            .saturating_add(prepared.fallback_cells);
6890                        report.shadow_accepted_span_cells = report
6891                            .shadow_accepted_span_cells
6892                            .saturating_add(prepared.direct_cells);
6893                        report.shadow_fallback_cells = report
6894                            .shadow_fallback_cells
6895                            .saturating_add(prepared.fallback_cells);
6896                        report.source_descendant_strings_avoided = report
6897                            .source_descendant_strings_avoided
6898                            .saturating_add(descendants);
6899                        report.source_descendant_events_avoided = report
6900                            .source_descendant_events_avoided
6901                            .saturating_add(descendants);
6902                        report.source_descendant_analyses_avoided = report
6903                            .source_descendant_analyses_avoided
6904                            .saturating_add(descendants);
6905                        report.graph_formula_vertices_avoided_shadow = report
6906                            .graph_formula_vertices_avoided_shadow
6907                            .saturating_add(prepared.direct_cells);
6908                        report.ast_roots_avoided_shadow = report
6909                            .ast_roots_avoided_shadow
6910                            .saturating_add(prepared.direct_cells.saturating_sub(fragment_count));
6911                        report.edge_rows_avoided_shadow = report
6912                            .edge_rows_avoided_shadow
6913                            .saturating_add(prepared.direct_cells);
6914                        if prepared.function_semantics_used {
6915                            report.source_partition_function_semantics =
6916                                report.source_partition_function_semantics.saturating_add(1);
6917                        }
6918                    }
6919                    Err(error) => {
6920                        if error.parse_attempted {
6921                            report.source_anchor_parses =
6922                                report.source_anchor_parses.saturating_add(1);
6923                        }
6924                        if error.ast_created {
6925                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6926                        }
6927                        if error.analysis_created {
6928                            report.source_anchor_analyses =
6929                                report.source_anchor_analyses.saturating_add(1);
6930                        }
6931                        report.shadow_fallback_cells = report
6932                            .shadow_fallback_cells
6933                            .saturating_add(family.surviving_member_count);
6934                        report.source_partitioned_families_rejected = report
6935                            .source_partitioned_families_rejected
6936                            .saturating_add(1);
6937                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
6938                    }
6939                }
6940            }
6941        }
6942        report
6943    }
6944
6945    fn analyze_compressed_formula_plane_shadow(
6946        &mut self,
6947        batches: &[(String, Vec<crate::engine::SourceFormulaFamily>)],
6948    ) -> FormulaIngestReport {
6949        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
6950        let _active_epoch = self.graph.formula_authority().plane.epoch();
6951
6952        for (sheet_name, families) in batches {
6953            let sheet_id = self.graph.sheet_id_mut(sheet_name);
6954            for family in families {
6955                let descendants = family.member_count.saturating_sub(1);
6956                report.shadow_candidate_cells = report
6957                    .shadow_candidate_cells
6958                    .saturating_add(family.member_count);
6959
6960                match self.prepare_source_formula_family(sheet_id, family, true, None) {
6961                    Ok(_) => {
6962                        report.source_descendant_strings_avoided = report
6963                            .source_descendant_strings_avoided
6964                            .saturating_add(descendants);
6965                        report.source_descendant_events_avoided = report
6966                            .source_descendant_events_avoided
6967                            .saturating_add(descendants);
6968                        report.source_descendant_analyses_avoided = report
6969                            .source_descendant_analyses_avoided
6970                            .saturating_add(descendants);
6971                        report.source_anchor_parses = report.source_anchor_parses.saturating_add(1);
6972                        report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6973                        report.source_anchor_analyses =
6974                            report.source_anchor_analyses.saturating_add(1);
6975                        report.shadow_accepted_span_cells = report
6976                            .shadow_accepted_span_cells
6977                            .saturating_add(family.member_count);
6978                        report.source_compressed_families_prepared =
6979                            report.source_compressed_families_prepared.saturating_add(1);
6980                        report.source_compressed_cells_prepared = report
6981                            .source_compressed_cells_prepared
6982                            .saturating_add(family.member_count);
6983                        report.graph_formula_vertices_avoided_shadow = report
6984                            .graph_formula_vertices_avoided_shadow
6985                            .saturating_add(family.member_count);
6986                        report.ast_roots_avoided_shadow =
6987                            report.ast_roots_avoided_shadow.saturating_add(descendants);
6988                        report.edge_rows_avoided_shadow = report
6989                            .edge_rows_avoided_shadow
6990                            .saturating_add(family.member_count);
6991                    }
6992                    Err(error) => {
6993                        if error.parse_attempted {
6994                            report.source_anchor_parses =
6995                                report.source_anchor_parses.saturating_add(1);
6996                        }
6997                        if error.ast_created {
6998                            report.source_anchor_asts = report.source_anchor_asts.saturating_add(1);
6999                        }
7000                        if error.analysis_created {
7001                            report.source_anchor_analyses =
7002                                report.source_anchor_analyses.saturating_add(1);
7003                        }
7004                        report.shadow_fallback_cells = report
7005                            .shadow_fallback_cells
7006                            .saturating_add(family.member_count);
7007                        *report.fallback_reasons.entry(error.reason).or_default() += 1;
7008                    }
7009                }
7010            }
7011        }
7012        report
7013    }
7014
7015    fn analyze_formula_plane_shadow_candidates(
7016        &mut self,
7017        batches: &[FormulaIngestBatch],
7018    ) -> FormulaIngestReport {
7019        let mut report = FormulaIngestReport::with_mode(FormulaPlaneMode::Shadow);
7020        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7021
7022        // Touch graph-owned authority deliberately: Tranche 3 shadow analysis uses
7023        // scratch state, but FormulaPlane ownership now lives on DependencyGraph.
7024        let _active_epoch = self.graph.formula_authority().plane.epoch();
7025
7026        let batch_sheet_ids: Vec<SheetId> = batches
7027            .iter()
7028            .map(|batch| self.graph.sheet_id_mut(&batch.sheet_name))
7029            .collect();
7030        let mut groups: BTreeMap<
7031            (SheetId, u64, u32),
7032            Vec<(FormulaPlacementCandidate, CandidateAnalysis)>,
7033        > = BTreeMap::new();
7034        {
7035            let mut pipeline = self.ingest_pipeline();
7036            for (batch, sheet_id) in batches.iter().zip(batch_sheet_ids.iter().copied()) {
7037                for record in &batch.formulas {
7038                    if record.row == 0 || record.col == 0 {
7039                        report.shadow_candidate_cells =
7040                            report.shadow_candidate_cells.saturating_add(1);
7041                        report.shadow_fallback_cells =
7042                            report.shadow_fallback_cells.saturating_add(1);
7043                        Self::record_shadow_fallback_reason(
7044                            &mut report,
7045                            PlacementFallbackReason::UnsupportedShapeOrGaps,
7046                            1,
7047                        );
7048                        continue;
7049                    }
7050
7051                    let placement = CellRef::new(
7052                        sheet_id,
7053                        Coord::from_excel(record.row, record.col, true, true),
7054                    );
7055                    let ingested = match pipeline.ingest_formula(
7056                        FormulaAstInput::RawArena(record.ast_id),
7057                        placement,
7058                        record.formula_text.clone(),
7059                    ) {
7060                        Ok(ingested) => ingested,
7061                        Err(_) => {
7062                            report.shadow_candidate_cells =
7063                                report.shadow_candidate_cells.saturating_add(1);
7064                            report.shadow_fallback_cells =
7065                                report.shadow_fallback_cells.saturating_add(1);
7066                            Self::record_shadow_fallback_reason(
7067                                &mut report,
7068                                PlacementFallbackReason::UnsupportedCanonicalTemplate,
7069                                1,
7070                            );
7071                            continue;
7072                        }
7073                    };
7074                    let candidate = FormulaPlacementCandidate::new(
7075                        sheet_id,
7076                        record.row - 1,
7077                        record.col - 1,
7078                        ingested.ast_id,
7079                        record.formula_text.clone(),
7080                    );
7081                    let analysis = match CandidateAnalysis::from_ingested(&candidate, &ingested) {
7082                        Ok(analysis) => analysis,
7083                        Err(reason) => {
7084                            report.shadow_candidate_cells =
7085                                report.shadow_candidate_cells.saturating_add(1);
7086                            report.shadow_fallback_cells =
7087                                report.shadow_fallback_cells.saturating_add(1);
7088                            Self::record_shadow_fallback_reason(&mut report, reason, 1);
7089                            continue;
7090                        }
7091                    };
7092                    groups
7093                        .entry((
7094                            sheet_id,
7095                            ingested.parameterized_canonical_hash,
7096                            candidate.col,
7097                        ))
7098                        .or_default()
7099                        .push((candidate, analysis));
7100                }
7101            }
7102        }
7103
7104        let mut scratch_plane = FormulaPlane::default();
7105        for entries in groups.into_values() {
7106            let (candidates, analyses): (Vec<_>, Vec<_>) = entries.into_iter().unzip();
7107            for (component, component_analyses) in
7108                Self::split_candidate_components_with_analyses(candidates, analyses)
7109            {
7110                let placement_report = place_candidate_family_with_analyses(
7111                    &mut scratch_plane,
7112                    component,
7113                    component_analyses,
7114                );
7115                let counters = placement_report.counters;
7116                report.shadow_candidate_cells = report
7117                    .shadow_candidate_cells
7118                    .saturating_add(counters.formula_cells_seen);
7119                report.shadow_accepted_span_cells = report
7120                    .shadow_accepted_span_cells
7121                    .saturating_add(counters.accepted_span_cells);
7122                report.shadow_fallback_cells = report
7123                    .shadow_fallback_cells
7124                    .saturating_add(counters.legacy_cells);
7125                report.shadow_templates_interned = report
7126                    .shadow_templates_interned
7127                    .saturating_add(counters.templates_interned);
7128                report.shadow_spans_created = report
7129                    .shadow_spans_created
7130                    .saturating_add(counters.spans_created);
7131                report.graph_formula_vertices_avoided_shadow = report
7132                    .graph_formula_vertices_avoided_shadow
7133                    .saturating_add(counters.formula_vertices_avoided);
7134                report.ast_roots_avoided_shadow = report
7135                    .ast_roots_avoided_shadow
7136                    .saturating_add(counters.ast_roots_avoided);
7137                report.edge_rows_avoided_shadow = report
7138                    .edge_rows_avoided_shadow
7139                    .saturating_add(counters.edge_rows_avoided);
7140                for (reason, count) in counters.fallback_reasons {
7141                    Self::record_shadow_fallback_reason(&mut report, reason, count);
7142                }
7143            }
7144        }
7145        report
7146    }
7147
7148    fn record_shadow_fallback_reason(
7149        report: &mut FormulaIngestReport,
7150        reason: PlacementFallbackReason,
7151        count: u64,
7152    ) {
7153        *report
7154            .fallback_reasons
7155            .entry(format!("{reason:?}"))
7156            .or_default() += count;
7157    }
7158
7159    fn analyze_formula_plane_authoritative_ingest(
7160        &mut self,
7161        batches: &[FormulaIngestBatch],
7162    ) -> (
7163        FormulaIngestReport,
7164        Vec<FormulaIngestBatch>,
7165        PlannedFormulaMaterialize,
7166    ) {
7167        let existing_span_refs = self
7168            .graph
7169            .formula_authority()
7170            .active_span_refs()
7171            .into_iter()
7172            .collect::<rustc_hash::FxHashSet<_>>();
7173        let mut report =
7174            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
7175        report.formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
7176
7177        let mut pending_candidates: Vec<(String, FormulaPlacementCandidate)> = Vec::new();
7178        let mut fallback: BTreeMap<String, Vec<FormulaIngestRecord>> = BTreeMap::new();
7179        let mut planned_fallback: PlannedFormulaMaterialize = BTreeMap::new();
7180
7181        for batch in batches {
7182            let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
7183            for record in &batch.formulas {
7184                if record.row == 0 || record.col == 0 {
7185                    report.shadow_candidate_cells = report.shadow_candidate_cells.saturating_add(1);
7186                    report.shadow_fallback_cells = report.shadow_fallback_cells.saturating_add(1);
7187                    Self::record_shadow_fallback_reason(
7188                        &mut report,
7189                        PlacementFallbackReason::UnsupportedShapeOrGaps,
7190                        1,
7191                    );
7192                    fallback
7193                        .entry(batch.sheet_name.clone())
7194                        .or_default()
7195                        .push(record.clone());
7196                    continue;
7197                }
7198
7199                pending_candidates.push((
7200                    batch.sheet_name.clone(),
7201                    FormulaPlacementCandidate::new(
7202                        sheet_id,
7203                        record.row - 1,
7204                        record.col - 1,
7205                        record.ast_id,
7206                        record.formula_text.clone(),
7207                    ),
7208                ));
7209            }
7210        }
7211
7212        let mut groups: BTreeMap<(SheetId, u64, u32), Vec<usize>> = BTreeMap::new();
7213        let mut analyses_by_index: Vec<Option<CandidateAnalysis>> =
7214            (0..pending_candidates.len()).map(|_| None).collect();
7215        let mut plans_by_index: Vec<Option<DependencyPlanRow>> =
7216            (0..pending_candidates.len()).map(|_| None).collect();
7217        {
7218            let mut pipeline = self.ingest_pipeline();
7219            for (idx, (sheet_name, candidate)) in pending_candidates.iter_mut().enumerate() {
7220                let placement = CellRef::new(
7221                    candidate.sheet_id,
7222                    Coord::from_excel(
7223                        candidate.row.saturating_add(1),
7224                        candidate.col.saturating_add(1),
7225                        true,
7226                        true,
7227                    ),
7228                );
7229                let ingested = pipeline.ingest_formula(
7230                    FormulaAstInput::RawArena(candidate.ast_id),
7231                    placement,
7232                    candidate.formula_text.clone(),
7233                );
7234                match ingested {
7235                    Ok(ingested) => {
7236                        candidate.ast_id = ingested.ast_id;
7237                        let dep_plan = ingested.dep_plan.clone();
7238                        match CandidateAnalysis::from_ingested(candidate, &ingested) {
7239                            Ok(analysis) => {
7240                                analyses_by_index[idx] = Some(analysis);
7241                                plans_by_index[idx] = Some(dep_plan);
7242                            }
7243                            Err(reason) => {
7244                                report.shadow_candidate_cells =
7245                                    report.shadow_candidate_cells.saturating_add(1);
7246                                report.shadow_fallback_cells =
7247                                    report.shadow_fallback_cells.saturating_add(1);
7248                                Self::record_shadow_fallback_reason(&mut report, reason, 1);
7249                                planned_fallback
7250                                    .entry(sheet_name.clone())
7251                                    .or_default()
7252                                    .push((
7253                                        candidate.row.saturating_add(1),
7254                                        candidate.col.saturating_add(1),
7255                                        candidate.ast_id,
7256                                        dep_plan,
7257                                    ));
7258                            }
7259                        }
7260                    }
7261                    Err(_) => {
7262                        let reason = PlacementFallbackReason::UnsupportedCanonicalTemplate;
7263                        report.shadow_candidate_cells =
7264                            report.shadow_candidate_cells.saturating_add(1);
7265                        report.shadow_fallback_cells =
7266                            report.shadow_fallback_cells.saturating_add(1);
7267                        Self::record_shadow_fallback_reason(&mut report, reason, 1);
7268                        fallback.entry(sheet_name.clone()).or_default().push(
7269                            FormulaIngestRecord::new(
7270                                candidate.row.saturating_add(1),
7271                                candidate.col.saturating_add(1),
7272                                candidate.ast_id,
7273                                candidate.formula_text.clone(),
7274                            ),
7275                        );
7276                    }
7277                }
7278            }
7279        }
7280
7281        for (idx, (_, candidate)) in pending_candidates.iter().enumerate() {
7282            if analyses_by_index[idx].is_none() {
7283                continue;
7284            }
7285            let canonical_hash = analyses_by_index[idx]
7286                .as_ref()
7287                .expect("remaining candidate has an analysis")
7288                .parameterized_canonical_hash();
7289            groups
7290                .entry((candidate.sheet_id, canonical_hash, candidate.col))
7291                .or_default()
7292                .push(idx);
7293        }
7294
7295        for ((_sheet_id, _canonical_hash, _col), candidate_indices) in groups {
7296            let sheet_name = pending_candidates[candidate_indices[0]].0.clone();
7297            let mut plans_by_coord: BTreeMap<(u32, u32), Vec<DependencyPlanRow>> = BTreeMap::new();
7298            for idx in &candidate_indices {
7299                // Each candidate index belongs to exactly one group, so the
7300                // plan row can be moved out instead of deep-cloned.
7301                if let Some(plan) = plans_by_index[*idx].take() {
7302                    let candidate = &pending_candidates[*idx].1;
7303                    plans_by_coord
7304                        .entry((candidate.row, candidate.col))
7305                        .or_default()
7306                        .push(plan);
7307                }
7308            }
7309            let candidates: Vec<_> = candidate_indices
7310                .iter()
7311                .map(|idx| pending_candidates[*idx].1.clone())
7312                .collect();
7313            let components = Self::split_shadow_candidate_components(candidates);
7314            let analyzed_components =
7315                if components.len() == 1 && components[0].len() == candidate_indices.len() {
7316                    let component = components.into_iter().next().expect("one component");
7317                    let component_analyses = candidate_indices
7318                        .iter()
7319                        .map(|idx| {
7320                            analyses_by_index[*idx]
7321                                .take()
7322                                .expect("candidate analysis must be used once")
7323                        })
7324                        .collect();
7325                    vec![(component, component_analyses)]
7326                } else {
7327                    let mut indices_by_coord: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
7328                    for idx in candidate_indices.iter().rev() {
7329                        let candidate = &pending_candidates[*idx].1;
7330                        indices_by_coord
7331                            .entry((candidate.row, candidate.col))
7332                            .or_default()
7333                            .push(*idx);
7334                    }
7335
7336                    components
7337                        .into_iter()
7338                        .map(|component| {
7339                            let mut component_analyses = Vec::with_capacity(component.len());
7340                            for candidate in &component {
7341                                let idx = indices_by_coord
7342                                    .get_mut(&(candidate.row, candidate.col))
7343                                    .and_then(Vec::pop)
7344                                    .expect("component candidate must have a precomputed analysis");
7345                                component_analyses.push(
7346                                    analyses_by_index[idx]
7347                                        .take()
7348                                        .expect("candidate analysis must be used once"),
7349                                );
7350                            }
7351                            (component, component_analyses)
7352                        })
7353                        .collect()
7354                };
7355
7356            for (component, component_analyses) in analyzed_components {
7357                for (component, component_analyses) in
7358                    split_candidate_affine_literal_runs(component, component_analyses)
7359                {
7360                    let placement_report = {
7361                        let authority = self.graph.formula_authority_mut();
7362                        place_candidate_family_with_analyses(
7363                            &mut authority.plane,
7364                            component.clone(),
7365                            component_analyses,
7366                        )
7367                    };
7368                    Self::accumulate_formula_plane_placement_report(&mut report, &placement_report);
7369
7370                    // Index candidates by placement once per component. The
7371                    // previous per-result linear `find` made this fallback
7372                    // mapping O(N²) for rejected families (e.g. an N-cell
7373                    // chain rejected with `InternalDependency`), dominating
7374                    // first-eval ingest cost on large rejected families.
7375                    // First insert wins, matching the old `Iterator::find`
7376                    // semantics for duplicate placements.
7377                    let mut candidate_by_placement: FxHashMap<
7378                        crate::formula_plane::runtime::PlacementCoord,
7379                        &FormulaPlacementCandidate,
7380                    > = FxHashMap::with_capacity_and_hasher(component.len(), Default::default());
7381                    for candidate in &component {
7382                        candidate_by_placement
7383                            .entry(candidate.placement())
7384                            .or_insert(candidate);
7385                    }
7386                    for result in &placement_report.results {
7387                        let FormulaPlacementResult::Legacy { placement, .. } = result else {
7388                            continue;
7389                        };
7390                        if let Some(&candidate) = candidate_by_placement.get(placement) {
7391                            let plan = plans_by_coord
7392                                .get_mut(&(candidate.row, candidate.col))
7393                                .and_then(Vec::pop);
7394                            if let Some(plan) = plan {
7395                                planned_fallback
7396                                    .entry(sheet_name.clone())
7397                                    .or_default()
7398                                    .push((
7399                                        candidate.row.saturating_add(1),
7400                                        candidate.col.saturating_add(1),
7401                                        candidate.ast_id,
7402                                        plan,
7403                                    ));
7404                            } else {
7405                                fallback.entry(sheet_name.clone()).or_default().push(
7406                                    FormulaIngestRecord::new(
7407                                        candidate.row.saturating_add(1),
7408                                        candidate.col.saturating_add(1),
7409                                        candidate.ast_id,
7410                                        candidate.formula_text.clone(),
7411                                    ),
7412                                );
7413                            }
7414                        }
7415                    }
7416                }
7417            }
7418        }
7419
7420        let _index_report = self.graph.formula_authority_mut().rebuild_indexes();
7421        let new_span_refs = self
7422            .graph
7423            .formula_authority()
7424            .active_span_refs()
7425            .into_iter()
7426            .filter(|span_ref| !existing_span_refs.contains(span_ref))
7427            .collect::<Vec<_>>();
7428        self.graph
7429            .mark_formula_spans_dirty(new_span_refs, WholeSpanDirtyReason::NewSpan);
7430
7431        let fallback_batches = fallback
7432            .into_iter()
7433            .map(|(sheet_name, formulas)| FormulaIngestBatch::new(sheet_name, formulas))
7434            .collect();
7435        (report, fallback_batches, planned_fallback)
7436    }
7437
7438    fn accumulate_formula_plane_placement_report(
7439        report: &mut FormulaIngestReport,
7440        placement_report: &crate::formula_plane::placement::FormulaPlacementReport,
7441    ) {
7442        let counters = &placement_report.counters;
7443        report.shadow_candidate_cells = report
7444            .shadow_candidate_cells
7445            .saturating_add(counters.formula_cells_seen);
7446        report.shadow_accepted_span_cells = report
7447            .shadow_accepted_span_cells
7448            .saturating_add(counters.accepted_span_cells);
7449        report.shadow_fallback_cells = report
7450            .shadow_fallback_cells
7451            .saturating_add(counters.legacy_cells);
7452        report.shadow_templates_interned = report
7453            .shadow_templates_interned
7454            .saturating_add(counters.templates_interned);
7455        report.shadow_spans_created = report
7456            .shadow_spans_created
7457            .saturating_add(counters.spans_created);
7458        report.graph_formula_vertices_avoided_shadow = report
7459            .graph_formula_vertices_avoided_shadow
7460            .saturating_add(counters.formula_vertices_avoided);
7461        report.ast_roots_avoided_shadow = report
7462            .ast_roots_avoided_shadow
7463            .saturating_add(counters.ast_roots_avoided);
7464        report.edge_rows_avoided_shadow = report
7465            .edge_rows_avoided_shadow
7466            .saturating_add(counters.edge_rows_avoided);
7467        for (reason, count) in &counters.fallback_reasons {
7468            Self::record_shadow_fallback_reason(report, *reason, *count);
7469        }
7470    }
7471
7472    fn split_candidate_components_with_analyses(
7473        candidates: Vec<FormulaPlacementCandidate>,
7474        mut analyses: Vec<CandidateAnalysis>,
7475    ) -> Vec<(Vec<FormulaPlacementCandidate>, Vec<CandidateAnalysis>)> {
7476        let components = Self::split_shadow_candidate_components(candidates.clone());
7477        let mut analysis_by_coord: BTreeMap<(u32, u32), Vec<CandidateAnalysis>> = BTreeMap::new();
7478        for (candidate, analysis) in candidates.into_iter().zip(analyses.drain(..)) {
7479            analysis_by_coord
7480                .entry((candidate.row, candidate.col))
7481                .or_default()
7482                .push(analysis);
7483        }
7484        components
7485            .into_iter()
7486            .flat_map(|component| {
7487                let mut component_analyses = Vec::with_capacity(component.len());
7488                for candidate in &component {
7489                    let analysis = analysis_by_coord
7490                        .get_mut(&(candidate.row, candidate.col))
7491                        .and_then(Vec::pop)
7492                        .expect("component candidate must have a precomputed analysis");
7493                    component_analyses.push(analysis);
7494                }
7495                split_candidate_affine_literal_runs(component, component_analyses)
7496            })
7497            .collect()
7498    }
7499
7500    fn split_shadow_candidate_components(
7501        candidates: Vec<FormulaPlacementCandidate>,
7502    ) -> Vec<Vec<FormulaPlacementCandidate>> {
7503        if candidates.len() <= 1 {
7504            return vec![candidates];
7505        }
7506
7507        // Fast path: candidates already ordered as one contiguous
7508        // single-column (or single-row) run form exactly one 4-connected
7509        // component in their existing (row, col) order; skip the BFS.
7510        let is_row_run = candidates.windows(2).all(|w| {
7511            w[0].sheet_id == w[1].sheet_id && w[0].col == w[1].col && w[0].row + 1 == w[1].row
7512        });
7513        let is_col_run = candidates.windows(2).all(|w| {
7514            w[0].sheet_id == w[1].sheet_id && w[0].row == w[1].row && w[0].col + 1 == w[1].col
7515        });
7516        if is_row_run || is_col_run {
7517            return vec![candidates];
7518        }
7519
7520        let mut coord_to_indices: BTreeMap<(u32, u32), Vec<usize>> = BTreeMap::new();
7521        for (idx, candidate) in candidates.iter().enumerate() {
7522            coord_to_indices
7523                .entry((candidate.row, candidate.col))
7524                .or_default()
7525                .push(idx);
7526        }
7527
7528        let mut remaining: BTreeSet<usize> = (0..candidates.len()).collect();
7529        let mut components = Vec::new();
7530        while let Some(&start) = remaining.iter().next() {
7531            remaining.remove(&start);
7532            let mut queue = VecDeque::from([start]);
7533            let mut component_indices = Vec::new();
7534
7535            while let Some(idx) = queue.pop_front() {
7536                component_indices.push(idx);
7537                let candidate = &candidates[idx];
7538                let mut neighbor_coords = Vec::with_capacity(5);
7539                neighbor_coords.push((candidate.row, candidate.col));
7540                if let Some(row) = candidate.row.checked_sub(1) {
7541                    neighbor_coords.push((row, candidate.col));
7542                }
7543                neighbor_coords.push((candidate.row.saturating_add(1), candidate.col));
7544                if let Some(col) = candidate.col.checked_sub(1) {
7545                    neighbor_coords.push((candidate.row, col));
7546                }
7547                neighbor_coords.push((candidate.row, candidate.col.saturating_add(1)));
7548
7549                for coord in neighbor_coords {
7550                    if let Some(indices) = coord_to_indices.get(&coord) {
7551                        for &neighbor in indices {
7552                            if remaining.remove(&neighbor) {
7553                                queue.push_back(neighbor);
7554                            }
7555                        }
7556                    }
7557                }
7558            }
7559
7560            component_indices.sort_by_key(|idx| {
7561                let candidate = &candidates[*idx];
7562                (candidate.row, candidate.col, *idx)
7563            });
7564            components.push(
7565                component_indices
7566                    .into_iter()
7567                    .map(|idx| candidates[idx].clone())
7568                    .collect(),
7569            );
7570        }
7571
7572        components
7573    }
7574
7575    fn collect_planning_function_requests(
7576        ast: &ASTNode,
7577        requests: &mut Vec<(String, String, usize)>,
7578    ) {
7579        match &ast.node_type {
7580            ASTNodeType::Function { name, args } => {
7581                requests.push((String::new(), name.clone(), args.len()));
7582                for arg in args {
7583                    Self::collect_planning_function_requests(arg, requests);
7584                }
7585            }
7586            ASTNodeType::BinaryOp { left, right, .. } => {
7587                Self::collect_planning_function_requests(left, requests);
7588                Self::collect_planning_function_requests(right, requests);
7589            }
7590            ASTNodeType::UnaryOp { expr, .. } => {
7591                Self::collect_planning_function_requests(expr, requests);
7592            }
7593            ASTNodeType::Call { callee, args } => {
7594                Self::collect_planning_function_requests(callee, requests);
7595                for arg in args {
7596                    Self::collect_planning_function_requests(arg, requests);
7597                }
7598            }
7599            ASTNodeType::Array(rows) => {
7600                for cell in rows.iter().flatten() {
7601                    Self::collect_planning_function_requests(cell, requests);
7602                }
7603            }
7604            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => {}
7605        }
7606    }
7607
7608    fn prepared_placement_function_semantics_changed(
7609        &self,
7610        semantic_epoch: u64,
7611        prepared: &crate::formula_plane::placement::PreparedAnchorOncePlacement,
7612        guard: &crate::function_registry::SemanticEpochReadGuard,
7613    ) -> bool {
7614        if semantic_epoch == guard.epoch() {
7615            return false;
7616        }
7617        let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
7618        let Some(ast) = self
7619            .graph
7620            .data_store()
7621            .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
7622        else {
7623            // Missing planning evidence must never turn a semantic change into reuse.
7624            return true;
7625        };
7626        let mut requests = Vec::new();
7627        Self::collect_planning_function_requests(&ast, &mut requests);
7628        guard.semantic_changes_affect_requests_since(semantic_epoch, requests)
7629    }
7630
7631    fn prepared_function_semantics_changed(
7632        &self,
7633        preparation: &crate::engine::FormulaCompressedPreparation,
7634        guard: &crate::function_registry::SemanticEpochReadGuard,
7635    ) -> bool {
7636        if preparation.function_semantic_epoch == guard.epoch() {
7637            return false;
7638        }
7639
7640        let mut requests = Vec::new();
7641        for (_, _, prepared) in &preparation.prepared {
7642            let (_, _, _, ast_id, _, _) = prepared.ownership_proof();
7643            let Some(ast) = self
7644                .graph
7645                .data_store()
7646                .reconstruct_ast_node(ast_id, self.graph.sheet_reg())
7647            else {
7648                return true;
7649            };
7650            Self::collect_planning_function_requests(&ast, &mut requests);
7651        }
7652        guard.semantic_changes_affect_requests_since(preparation.function_semantic_epoch, requests)
7653    }
7654
7655    pub(crate) fn prepare_source_formula_families(
7656        &mut self,
7657        sheet_name: &str,
7658        families: &[crate::engine::SourceFormulaFamily],
7659    ) -> crate::engine::FormulaCompressedPreparation {
7660        let mut preparation = crate::engine::FormulaCompressedPreparation {
7661            engine_token: Arc::clone(&self.source_formula_token),
7662            function_semantic_epoch: crate::function_registry::semantic_epoch(),
7663            function_provider_revision: None,
7664            function_semantics_used: false,
7665            sheet_name: Arc::from(sheet_name),
7666            prepared: Vec::new(),
7667            rejected: BTreeMap::new(),
7668            fragmented: Vec::new(),
7669            fragmented_sources: BTreeMap::new(),
7670            eager_replay: Vec::new(),
7671            preparation_spool_replays: 0,
7672            clean_rejected_anchor_counts: [0; 3],
7673            fragmented_rejected_anchor_counts: [0; 3],
7674            exact_replay: None,
7675            replay_disposition: crate::engine::FormulaReplayDisposition::default(),
7676        };
7677        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7678            return preparation;
7679        }
7680
7681        let mut requests = Vec::new();
7682        for family in families {
7683            let formula = if family.anchor_text.starts_with('=') {
7684                family.anchor_text.to_string()
7685            } else {
7686                format!("={}", family.anchor_text)
7687            };
7688            if let Ok(ast) = formualizer_parse::parser::parse(&formula) {
7689                Self::collect_planning_function_requests(&ast, &mut requests);
7690            }
7691        }
7692        requests.sort();
7693        requests.dedup();
7694        let snapshot =
7695            match crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
7696                &self.resolver,
7697                requests,
7698            ) {
7699                Ok(snapshot) => snapshot,
7700                Err(error) => {
7701                    let reason = match error {
7702                        crate::function_registry::PlanningSnapshotError::RegistryChangedDuringCapture => {
7703                            "FunctionSemanticSnapshotUnavailable"
7704                        }
7705                        crate::function_registry::PlanningSnapshotError::ProviderRevisionUnavailable => {
7706                            "FunctionProviderRevisionUnavailable"
7707                        }
7708                    };
7709                    for family in families {
7710                        preparation
7711                            .rejected
7712                            .insert(family.source_id, reason.to_string());
7713                    }
7714                    return preparation;
7715                }
7716            };
7717        preparation.function_semantic_epoch = snapshot.epoch();
7718        preparation.function_provider_revision = snapshot.provider_revision();
7719
7720        let sheet_id = self.graph.sheet_id_mut(sheet_name);
7721        for family in families {
7722            #[cfg(test)]
7723            let force_replay = self.force_source_family_fallback
7724                || cfg!(feature = "benchmark_internal")
7725                    && std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7726            #[cfg(all(feature = "benchmark_internal", not(test)))]
7727            let force_replay =
7728                std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7729            #[cfg(any(test, feature = "benchmark_internal"))]
7730            if force_replay {
7731                preparation
7732                    .rejected
7733                    .insert(family.source_id, "ForcedReplay".to_string());
7734                continue;
7735            }
7736            match self.prepare_source_formula_family(sheet_id, family, true, Some(&snapshot)) {
7737                Ok((prepared, function_semantics_used)) => {
7738                    preparation.function_semantics_used |= function_semantics_used;
7739                    preparation
7740                        .replay_disposition
7741                        .set_family_direct(family.source_id);
7742                    preparation
7743                        .prepared
7744                        .push((family.source_id, family.source_order, prepared));
7745                }
7746                Err(error) => {
7747                    preparation.clean_rejected_anchor_counts[0] = preparation
7748                        .clean_rejected_anchor_counts[0]
7749                        .saturating_add(u64::from(error.parse_attempted));
7750                    preparation.clean_rejected_anchor_counts[1] = preparation
7751                        .clean_rejected_anchor_counts[1]
7752                        .saturating_add(u64::from(error.ast_created));
7753                    preparation.clean_rejected_anchor_counts[2] = preparation
7754                        .clean_rejected_anchor_counts[2]
7755                        .saturating_add(u64::from(error.analysis_created));
7756                    preparation.rejected.insert(family.source_id, error.reason);
7757                }
7758            }
7759        }
7760        preparation
7761    }
7762
7763    fn prepare_source_formula_proposals(
7764        &mut self,
7765        sheet_name: &str,
7766        families: &[crate::engine::SourceFormulaFamily],
7767        authority_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7768        replay_partitions: &[crate::engine::PartitionedSourceFormulaFamily],
7769        formula_record_count: u64,
7770        replay: Arc<std::sync::Mutex<Box<dyn crate::engine::DeferredFormulaReplay>>>,
7771        suppressed: &BTreeSet<(u32, u32)>,
7772    ) -> Result<crate::engine::FormulaCompressedPreparation, ExcelError> {
7773        let mut preparation = self.prepare_source_formula_families(sheet_name, families);
7774        preparation.exact_replay = Some(Arc::clone(&replay));
7775        preparation
7776            .replay_disposition
7777            .extend_suppressed_excel_coords(suppressed.iter().copied());
7778        if self.config.formula_plane_mode != FormulaPlaneMode::AuthoritativeExperimental {
7779            return Ok(preparation);
7780        }
7781        #[cfg(feature = "benchmark_internal")]
7782        let benchmark_forced_replay =
7783            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
7784        #[cfg(not(feature = "benchmark_internal"))]
7785        let benchmark_forced_replay = false;
7786        let authority_partitions = if benchmark_forced_replay {
7787            for partition in authority_partitions {
7788                preparation
7789                    .rejected
7790                    .insert(partition.source_id, "ForcedReplay".to_string());
7791            }
7792            &[][..]
7793        } else {
7794            authority_partitions
7795        };
7796        let authority_ids: BTreeSet<_> = authority_partitions
7797            .iter()
7798            .map(|partition| partition.source_id)
7799            .collect();
7800        for partition in replay_partitions {
7801            if !authority_ids.contains(&partition.source_id) {
7802                preparation
7803                    .replay_disposition
7804                    .register_partition(partition, false)
7805                    .map_err(|reason| {
7806                        ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7807                    })?;
7808            }
7809        }
7810        if replay_partitions.is_empty()
7811            && preparation.rejected.is_empty()
7812            && preparation.direct_cell_count() == formula_record_count
7813        {
7814            return Ok(preparation);
7815        }
7816
7817        let mut analyzed = Vec::new();
7818        for source in authority_partitions {
7819            preparation
7820                .fragmented_sources
7821                .insert(source.source_id, source.clone());
7822            match self.analyze_partitioned_source_family_for_transaction(sheet_name, source) {
7823                Ok(prepared) => {
7824                    let mut candidate = preparation.replay_disposition.clone();
7825                    match candidate.register_partition(source, true) {
7826                        Ok(()) => {
7827                            preparation.replay_disposition = candidate;
7828                            analyzed.push((source.clone(), prepared));
7829                        }
7830                        Err(reason) => {
7831                            preparation
7832                                .rejected
7833                                .insert(source.source_id, reason.to_string());
7834                            preparation
7835                                .replay_disposition
7836                                .register_partition(source, false)
7837                                .map_err(|reason| {
7838                                    ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7839                                })?;
7840                        }
7841                    }
7842                }
7843                Err(error) => {
7844                    preparation.fragmented_rejected_anchor_counts[0] = preparation
7845                        .fragmented_rejected_anchor_counts[0]
7846                        .saturating_add(u64::from(error.parse_attempted));
7847                    preparation.fragmented_rejected_anchor_counts[1] = preparation
7848                        .fragmented_rejected_anchor_counts[1]
7849                        .saturating_add(u64::from(error.ast_created));
7850                    preparation.fragmented_rejected_anchor_counts[2] = preparation
7851                        .fragmented_rejected_anchor_counts[2]
7852                        .saturating_add(u64::from(error.analysis_created));
7853                    preparation.rejected.insert(source.source_id, error.reason);
7854                    preparation
7855                        .replay_disposition
7856                        .register_partition(source, false)
7857                        .map_err(|reason| {
7858                            ExcelError::new(ExcelErrorKind::Value).with_message(reason)
7859                        })?;
7860                }
7861            }
7862        }
7863
7864        let initial_replay = replay
7865            .lock()
7866            .map_err(|_| {
7867                ExcelError::new(ExcelErrorKind::Value)
7868                    .with_message("compressed formula exact replay lock poisoned")
7869            })?
7870            .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7871            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
7872        preparation.preparation_spool_replays = 1;
7873        let sheet_id = self.graph.sheet_id_mut(sheet_name);
7874        let mut preparation_failed = false;
7875        for (source, prepared_family) in analyzed {
7876            let result = (|| {
7877                let expected: BTreeMap<_, _> = source
7878                    .legacy_members
7879                    .as_slice()
7880                    .iter()
7881                    .map(|member| (member.coord, member.kind))
7882                    .collect();
7883                let mut actual = BTreeMap::new();
7884                for record in initial_replay
7885                    .iter()
7886                    .filter(|record| record.partition_owner == Some(source.source_id))
7887                {
7888                    let coord = record
7889                        .row
7890                        .checked_sub(1)
7891                        .zip(record.col.checked_sub(1))
7892                        .map(|(row, col)| crate::engine::SourceCoord { row, col })
7893                        .ok_or_else(|| "ExactReplayRecordInvalidCoordinate".to_string())?;
7894                    let kind = match record.family {
7895                        Some(family) if family == source.source_id => {
7896                            crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
7897                        }
7898                        None => crate::engine::PartitionLegacyMemberKind::OrdinaryException,
7899                        Some(_) => return Err("ExactReplayRecordOwnerMismatch".to_string()),
7900                    };
7901                    if actual.insert(coord, (kind, record.clone())).is_some() {
7902                        return Err("ExactReplayRecordDuplicate".to_string());
7903                    }
7904                }
7905                let actual_kinds: BTreeMap<_, _> = actual
7906                    .iter()
7907                    .map(|(coord, (kind, _))| (*coord, *kind))
7908                    .collect();
7909                if actual_kinds != expected {
7910                    return Err("ExactReplayLegacySetMismatch".to_string());
7911                }
7912                let mut legacy = Vec::with_capacity(source.legacy_members.len());
7913                for expected in source.legacy_members.as_slice() {
7914                    let record = actual
7915                        .remove(&expected.coord)
7916                        .map(|(_, record)| record)
7917                        .ok_or_else(|| "ExactReplayRecordMissing".to_string())?;
7918                    legacy.push(
7919                        self.graph
7920                            .analyze_fragmented_exact_replay_record(
7921                                &self.resolver,
7922                                sheet_id,
7923                                source.source_id,
7924                                *expected,
7925                                record,
7926                            )
7927                            .map_err(|error| format!("{error:?}"))?,
7928                    );
7929                }
7930                Ok::<_, String>((prepared_family, legacy))
7931            })();
7932            match result {
7933                Ok((prepared_family, legacy)) => preparation.fragmented.push(
7934                    crate::engine::formula_source::PreparedFragmentedSourceProposal {
7935                        source: source.clone(),
7936                        prepared: prepared_family,
7937                        legacy,
7938                        replay: Arc::clone(&replay),
7939                    },
7940                ),
7941                Err(reason) => {
7942                    preparation_failed = true;
7943                    for count in &mut preparation.fragmented_rejected_anchor_counts {
7944                        *count = count.saturating_add(1);
7945                    }
7946                    preparation.rejected.insert(source.source_id, reason);
7947                    preparation
7948                        .replay_disposition
7949                        .force_family_legacy(source.source_id);
7950                }
7951            }
7952        }
7953
7954        preparation.eager_replay = if preparation_failed {
7955            preparation.preparation_spool_replays = 2;
7956            replay
7957                .lock()
7958                .map_err(|_| {
7959                    ExcelError::new(ExcelErrorKind::Value)
7960                        .with_message("compressed formula exact replay lock poisoned")
7961                })?
7962                .replay_partitioned(&preparation.replay_disposition, replay_partitions)
7963                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
7964        } else {
7965            initial_replay
7966        };
7967        Ok(preparation)
7968    }
7969
7970    fn formula_batch_from_exact_replay(
7971        &mut self,
7972        sheet_name: &str,
7973        replayed: impl IntoIterator<Item = crate::engine::DeferredReplayFormula>,
7974    ) -> Result<FormulaIngestBatch, ExcelError> {
7975        let mut cache = rustc_hash::FxHashMap::default();
7976        let mut formulas = Vec::new();
7977        for record in replayed {
7978            let key = if record.text.starts_with('=') {
7979                record.text
7980            } else {
7981                format!("={}", record.text)
7982            };
7983            let ast_id = if let Some(cached) = cache.get(&key) {
7984                *cached
7985            } else {
7986                let parsed = match formualizer_parse::parser::parse(&key) {
7987                    Ok(parsed) => parsed,
7988                    Err(error) => {
7989                        let Some(parsed) = self.handle_formula_parse_error(
7990                            sheet_name,
7991                            record.row,
7992                            record.col,
7993                            &key,
7994                            error.to_string(),
7995                        )?
7996                        else {
7997                            continue;
7998                        };
7999                        parsed
8000                    }
8001                };
8002                let ast_id = self.intern_formula_ast(&parsed);
8003                cache.insert(key.clone(), ast_id);
8004                ast_id
8005            };
8006            formulas.push(
8007                FormulaIngestRecord::new(record.row, record.col, ast_id, Some(Arc::from(key)))
8008                    .with_source_proof(record.source_order, record.family, record.partition_owner),
8009            );
8010        }
8011        Ok(FormulaIngestBatch::new(sheet_name.to_string(), formulas))
8012    }
8013
8014    fn replay_prepared_families_exact_records(
8015        &mut self,
8016        preparation: &crate::engine::FormulaCompressedPreparation,
8017    ) -> Result<Vec<crate::engine::DeferredReplayFormula>, ExcelError> {
8018        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
8019            ExcelError::new(ExcelErrorKind::Value)
8020                .with_message("stale compressed preparation has no exact replay owner")
8021        })?;
8022        let direct: BTreeSet<_> = preparation
8023            .prepared
8024            .iter()
8025            .map(|(family, _, _)| *family)
8026            .collect();
8027        let mut replay_disposition = preparation.replay_disposition.clone();
8028        for family in &direct {
8029            replay_disposition.force_family_legacy(*family);
8030        }
8031        let replayed = replay
8032            .lock()
8033            .map_err(|_| {
8034                ExcelError::new(ExcelErrorKind::Value)
8035                    .with_message("compressed formula exact replay lock poisoned")
8036            })?
8037            .replay(&replay_disposition)
8038            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?;
8039        let mut records: Vec<_> = replayed
8040            .into_iter()
8041            .filter(|record| record.family.is_some_and(|family| direct.contains(&family)))
8042            .collect();
8043        records.sort_by_key(|record| record.source_order);
8044        let expected: u64 = preparation
8045            .prepared
8046            .iter()
8047            .map(|(_, _, prepared)| prepared.member_count)
8048            .sum();
8049        if records.len() as u64 != expected {
8050            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
8051                "compressed formula exact replay incomplete: expected {expected}, got {}",
8052                records.len()
8053            )));
8054        }
8055        Ok(records)
8056    }
8057
8058    fn replay_one_prepared_family_exact(
8059        &mut self,
8060        preparation: &crate::engine::FormulaCompressedPreparation,
8061        family: crate::engine::SourceFamilyId,
8062        member_count: u64,
8063    ) -> Result<FormulaIngestBatch, ExcelError> {
8064        let replay = preparation.exact_replay.as_ref().ok_or_else(|| {
8065            ExcelError::new(ExcelErrorKind::Value)
8066                .with_message("stale compressed preparation has no exact replay owner")
8067        })?;
8068        let mut replay_disposition = preparation.replay_disposition.clone();
8069        replay_disposition.force_family_legacy(family);
8070        let replayed = replay
8071            .lock()
8072            .map_err(|_| {
8073                ExcelError::new(ExcelErrorKind::Value)
8074                    .with_message("compressed formula exact replay lock poisoned")
8075            })?
8076            .replay(&replay_disposition)
8077            .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
8078            .into_iter()
8079            .filter(|record| record.family == Some(family));
8080        let batch =
8081            self.formula_batch_from_exact_replay(preparation.sheet_name.as_ref(), replayed)?;
8082        if batch.formulas.len() as u64 != member_count {
8083            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
8084                "compressed formula exact replay incomplete: expected {member_count}, got {}",
8085                batch.formulas.len()
8086            )));
8087        }
8088        Ok(batch)
8089    }
8090
8091    fn exact_replay_record_is_prepared_partition_legacy(
8092        source: &crate::engine::PartitionedSourceFormulaFamily,
8093        record: &crate::engine::DeferredReplayFormula,
8094    ) -> bool {
8095        if record.partition_owner != Some(source.source_id) {
8096            return false;
8097        }
8098        let Some(coord) = record
8099            .row
8100            .checked_sub(1)
8101            .zip(record.col.checked_sub(1))
8102            .map(|(row, col)| crate::engine::SourceCoord { row, col })
8103        else {
8104            return false;
8105        };
8106        source.legacy_members.as_slice().iter().any(|member| {
8107            member.coord == coord
8108                && match member.kind {
8109                    crate::engine::PartitionLegacyMemberKind::SharedFamilyMember => {
8110                        record.family == Some(source.source_id)
8111                    }
8112                    crate::engine::PartitionLegacyMemberKind::OrdinaryException => {
8113                        record.family.is_none()
8114                    }
8115                }
8116        })
8117    }
8118
8119    fn validate_whole_partition_replay(
8120        source: &crate::engine::PartitionedSourceFormulaFamily,
8121        records: &[crate::engine::DeferredReplayFormula],
8122    ) -> Result<(), ExcelError> {
8123        let mut coordinates = BTreeSet::new();
8124        let mut shared = 0u64;
8125        let mut ordinary = 0u64;
8126        for record in records {
8127            let coord = record
8128                .row
8129                .checked_sub(1)
8130                .zip(record.col.checked_sub(1))
8131                .map(|(row, col)| crate::engine::SourceCoord { row, col })
8132                .ok_or_else(|| {
8133                    ExcelError::new(ExcelErrorKind::Value)
8134                        .with_message("whole-family replay contains an invalid coordinate")
8135                })?;
8136            if !coordinates.insert(coord) {
8137                return Err(ExcelError::new(ExcelErrorKind::Value)
8138                    .with_message("whole-family replay contains a duplicate coordinate"));
8139            }
8140            match (record.family, record.partition_owner) {
8141                (Some(family), Some(owner))
8142                    if family == source.source_id && owner == source.source_id =>
8143                {
8144                    let direct = source.fragments.iter().any(|fragment| {
8145                        let rect = fragment.rect();
8146                        coord.row >= rect.start.row
8147                            && coord.row <= rect.end.row
8148                            && coord.col >= rect.start.col
8149                            && coord.col <= rect.end.col
8150                    });
8151                    let legacy = source.legacy_members.as_slice().iter().any(|member| {
8152                        member.coord == coord
8153                            && member.kind
8154                                == crate::engine::PartitionLegacyMemberKind::SharedFamilyMember
8155                    });
8156                    if !direct && !legacy {
8157                        return Err(ExcelError::new(ExcelErrorKind::Value)
8158                            .with_message("whole-family replay contains an extra shared member"));
8159                    }
8160                    shared = shared.saturating_add(1);
8161                }
8162                (None, Some(owner)) if owner == source.source_id => {
8163                    if !source.legacy_members.as_slice().iter().any(|member| {
8164                        member.coord == coord
8165                            && member.kind
8166                                == crate::engine::PartitionLegacyMemberKind::OrdinaryException
8167                    }) {
8168                        return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
8169                            "whole-family replay contains an extra ordinary exception",
8170                        ));
8171                    }
8172                    ordinary = ordinary.saturating_add(1);
8173                }
8174                _ => {
8175                    return Err(ExcelError::new(ExcelErrorKind::Value)
8176                        .with_message("whole-family replay ownership mismatch"));
8177                }
8178            }
8179        }
8180        if shared != source.surviving_member_count
8181            || ordinary != source.reconciliation.ordinary_exceptions
8182        {
8183            return Err(ExcelError::new(ExcelErrorKind::Value)
8184                .with_message("whole-family replay is incomplete"));
8185        }
8186        Ok(())
8187    }
8188
8189    fn prepare_target_combined_legacy_graph(
8190        &self,
8191        packages: &[PreparedTargetSourcePackage],
8192        ordinary: &[PreparedOrdinaryStagedFormula],
8193    ) -> Result<(PreparedLegacyGraphPlan, usize), ExcelError> {
8194        let mut planned_by_coord = BTreeMap::new();
8195        for package in packages {
8196            for (row, col, ast_id, plan) in &package.legacy {
8197                planned_by_coord.insert((package.sheet_id, *row, *col), (*ast_id, plan.clone()));
8198            }
8199        }
8200        for formula in ordinary {
8201            if let Some((ast_id, plan)) = formula.ast_id.zip(formula.plan.clone()) {
8202                planned_by_coord.insert(
8203                    (formula.sheet_id, formula.lease.row, formula.lease.col),
8204                    (ast_id, plan),
8205                );
8206            }
8207        }
8208        let planned = planned_by_coord
8209            .into_iter()
8210            .map(|((sheet_id, row, col), (ast_id, plan))| (sheet_id, row, col, ast_id, plan))
8211            .collect::<Vec<_>>();
8212        let formula_count = planned.len();
8213        let graph = self
8214            .graph
8215            .prepare_legacy_graph_plan_multi_sheet(planned)
8216            .map_err(|error| {
8217                ExcelError::new(ExcelErrorKind::Value)
8218                    .with_message(format!("target graph preparation failed: {error}"))
8219            })?;
8220        Ok((graph, formula_count))
8221    }
8222
8223    fn materialize_target_package_direct_records(
8224        &mut self,
8225        package: &mut PreparedTargetSourcePackage,
8226        assumptions: &crate::engine::PreparationRevision,
8227        planning_requests: &mut BTreeSet<(String, String, usize)>,
8228        deadline: Option<std::time::Instant>,
8229    ) -> Result<(), ExcelError> {
8230        let existing = package
8231            .legacy
8232            .iter()
8233            .map(|(row, col, _, _)| (*row, *col))
8234            .collect::<BTreeSet<_>>();
8235        let mut missing = BTreeMap::new();
8236        for record in &package.replay_records {
8237            if !existing.contains(&(record.row, record.col)) {
8238                missing.insert((record.row, record.col), record.clone());
8239            }
8240        }
8241        let batch = self.formula_batch_from_exact_replay(&package.sheet, missing.into_values())?;
8242        for record in batch.formulas {
8243            self.target_preparation_checkpoint(deadline, 1)?;
8244            let ast = self
8245                .graph
8246                .data_store()
8247                .retrieve_ast(record.ast_id, self.graph.sheet_reg())
8248                .ok_or_else(|| {
8249                    ExcelError::new(ExcelErrorKind::Value)
8250                        .with_message("target fallback AST is unavailable")
8251                })?;
8252            let snapshot = self.target_planning_snapshot(&ast, planning_requests)?;
8253            if let Some(reason) =
8254                Self::target_planning_snapshot_stale_reason(&snapshot, assumptions)
8255            {
8256                return Err(Self::preparation_stale(
8257                    reason,
8258                    "target fallback planning snapshot became stale during materialization",
8259                ));
8260            }
8261            let placement = CellRef::new(
8262                package.sheet_id,
8263                Coord::from_excel(record.row, record.col, true, true),
8264            );
8265            let ingested = self
8266                .graph
8267                .ingest_pipeline(&snapshot)
8268                .enable_function_semantics()
8269                .ingest_formula(
8270                    FormulaAstInput::RawArena(record.ast_id),
8271                    placement,
8272                    record.formula_text,
8273                )?;
8274            package
8275                .legacy
8276                .push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
8277        }
8278        package.direct_families = 0;
8279        package.direct_cells = 0;
8280        package.direct_fragments = 0;
8281        package.direct_complete_families = 0;
8282        package.direct_complete_cells = 0;
8283        package.direct_partition_families = 0;
8284        package.direct_partition_cells = 0;
8285        Ok(())
8286    }
8287
8288    fn prepare_target_source_package(
8289        &mut self,
8290        sheet: &str,
8291        lease: StagedPackageLease,
8292        deadline: Option<std::time::Instant>,
8293    ) -> Result<PreparedTargetSourcePackage, ExcelError> {
8294        let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
8295            ExcelError::new(ExcelErrorKind::Ref)
8296                .with_message(format!("deferred source sheet not found: {sheet}"))
8297        })?;
8298        let (
8299            mut source_report,
8300            families,
8301            partitions,
8302            replay,
8303            invalidated,
8304            suppressed,
8305            reconciliation_replay,
8306        ) = {
8307            let package = self
8308                .staged_formulas
8309                .get(sheet)
8310                .and_then(|staged| staged.deferred_package.as_ref())
8311                .ok_or_else(|| {
8312                    ExcelError::new(ExcelErrorKind::Value)
8313                        .with_message("staged deferred source package is unavailable")
8314                })?;
8315            if package.sheet_name != sheet {
8316                return Err(ExcelError::new(ExcelErrorKind::Value)
8317                    .with_message("deferred formula package sheet mismatch"));
8318            }
8319            (
8320                package.report.clone(),
8321                package.families.clone(),
8322                package.partitioned_families.clone(),
8323                Arc::clone(&package.replay),
8324                package.invalidated.clone(),
8325                package.suppressed.clone(),
8326                package.reconciliation_replay.clone(),
8327            )
8328        };
8329
8330        let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
8331        for partition in &partitions {
8332            replay_disposition
8333                .register_partition(partition, false)
8334                .map_err(|reason| ExcelError::new(ExcelErrorKind::Value).with_message(reason))?;
8335        }
8336        replay_disposition.extend_suppressed_excel_coords(suppressed.iter().copied());
8337        self.target_preparation_checkpoint(deadline, 1)?;
8338        let mut replay_records = if let Some(mut records) = reconciliation_replay {
8339            records.retain(|record| {
8340                let Some((row, col)) = record.row.checked_sub(1).zip(record.col.checked_sub(1))
8341                else {
8342                    return true;
8343                };
8344                let coord = crate::engine::SourceCoord { row, col };
8345                let disposition = record.family.map_or_else(
8346                    || replay_disposition.ordinary_disposition(coord).0,
8347                    |family| replay_disposition.shared_disposition(family, coord),
8348                );
8349                !matches!(
8350                    disposition,
8351                    crate::engine::FormulaReplayCoordinateDisposition::Direct
8352                        | crate::engine::FormulaReplayCoordinateDisposition::Suppressed
8353                )
8354            });
8355            records
8356        } else {
8357            replay
8358                .lock()
8359                .map_err(|_| {
8360                    ExcelError::new(ExcelErrorKind::Value)
8361                        .with_message("deferred formula spool lock poisoned")
8362                })?
8363                .replay_partitioned(&replay_disposition, &partitions)
8364                .map_err(|message| ExcelError::new(ExcelErrorKind::Value).with_message(message))?
8365        };
8366        replay_records.sort_by_key(|record| record.source_order);
8367        for chunk in replay_records.chunks(256) {
8368            self.target_preparation_checkpoint(deadline, chunk.len() as u64)?;
8369        }
8370        if replay_records
8371            .windows(2)
8372            .any(|records| records[0].source_order == records[1].source_order)
8373        {
8374            return Err(ExcelError::new(ExcelErrorKind::Value)
8375                .with_message("duplicate deferred source-order proof"));
8376        }
8377
8378        let mut disposition = replay_disposition;
8379        let mut ordered_placements = Vec::new();
8380        let mut direct_families = 0usize;
8381        let mut direct_cells = 0u64;
8382        let mut direct_fragments = 0u64;
8383        let mut direct_complete_families = 0u64;
8384        let mut direct_complete_cells = 0u64;
8385        let mut direct_partition_families = 0u64;
8386        let mut direct_partition_cells = 0u64;
8387        let mut anchor_parses = 0u64;
8388        let mut anchor_asts = 0u64;
8389        let mut anchor_analyses = 0u64;
8390        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
8391            for family in &families {
8392                self.target_preparation_checkpoint(deadline, 1)?;
8393                if invalidated.contains(&family.source_id) {
8394                    continue;
8395                }
8396                match self.prepare_source_formula_family(sheet_id, family, true, None) {
8397                    Ok((prepared, _)) => {
8398                        anchor_parses = anchor_parses.saturating_add(1);
8399                        anchor_asts = anchor_asts.saturating_add(1);
8400                        anchor_analyses = anchor_analyses.saturating_add(1);
8401                        direct_families = direct_families.saturating_add(1);
8402                        direct_cells = direct_cells.saturating_add(prepared.member_count);
8403                        direct_complete_families = direct_complete_families.saturating_add(1);
8404                        direct_complete_cells =
8405                            direct_complete_cells.saturating_add(prepared.member_count);
8406                        if self.config.formula_plane_mode
8407                            == FormulaPlaneMode::AuthoritativeExperimental
8408                        {
8409                            disposition.set_family_direct(family.source_id);
8410                        }
8411                        ordered_placements.push((family.source_order, vec![prepared]));
8412                    }
8413                    Err(error) => {
8414                        anchor_parses =
8415                            anchor_parses.saturating_add(u64::from(error.parse_attempted));
8416                        anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
8417                        anchor_analyses =
8418                            anchor_analyses.saturating_add(u64::from(error.analysis_created));
8419                        *source_report
8420                            .fallback_reasons
8421                            .entry(error.reason)
8422                            .or_default() += 1;
8423                    }
8424                }
8425            }
8426
8427            for source in &partitions {
8428                self.target_preparation_checkpoint(deadline, 1)?;
8429                if invalidated.contains(&source.source_id) {
8430                    continue;
8431                }
8432                let prepared =
8433                    match self.analyze_partitioned_source_family_for_transaction(sheet, source) {
8434                        Ok(prepared) => {
8435                            anchor_parses = anchor_parses.saturating_add(1);
8436                            anchor_asts = anchor_asts.saturating_add(1);
8437                            anchor_analyses = anchor_analyses.saturating_add(1);
8438                            prepared
8439                        }
8440                        Err(error) => {
8441                            anchor_parses =
8442                                anchor_parses.saturating_add(u64::from(error.parse_attempted));
8443                            anchor_asts = anchor_asts.saturating_add(u64::from(error.ast_created));
8444                            anchor_analyses =
8445                                anchor_analyses.saturating_add(u64::from(error.analysis_created));
8446                            *source_report
8447                                .fallback_reasons
8448                                .entry(error.reason)
8449                                .or_default() += 1;
8450                            continue;
8451                        }
8452                    };
8453                let fragment_regions: Vec<_> = prepared
8454                    .placements
8455                    .iter()
8456                    .map(|placement| Region::from_domain(placement.fragment_dependency_proof().0))
8457                    .collect();
8458                let crosses_fragment =
8459                    prepared
8460                        .placements
8461                        .iter()
8462                        .enumerate()
8463                        .any(|(fragment_index, placement)| {
8464                            placement
8465                                .fragment_dependency_proof()
8466                                .1
8467                                .dependencies
8468                                .iter()
8469                                .any(|dependency| {
8470                                    fragment_regions.iter().enumerate().any(
8471                                        |(candidate_index, candidate)| {
8472                                            candidate_index != fragment_index
8473                                                && dependency.read_region.intersects(candidate)
8474                                        },
8475                                    )
8476                                })
8477                        });
8478                let family_records = replay_records
8479                    .iter()
8480                    .filter(|record| {
8481                        record.family == Some(source.source_id)
8482                            || record.partition_owner == Some(source.source_id)
8483                    })
8484                    .cloned()
8485                    .collect::<Vec<_>>();
8486                if crosses_fragment {
8487                    *source_report
8488                        .fallback_reasons
8489                        .entry("CrossFragmentDependency".to_string())
8490                        .or_default() += 1;
8491                    continue;
8492                }
8493                if let Err(error) = Self::validate_whole_partition_replay(source, &family_records) {
8494                    *source_report
8495                        .fallback_reasons
8496                        .entry(format!("TargetPartitionReplay:{error}"))
8497                        .or_default() += 1;
8498                    continue;
8499                }
8500                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
8501                    let mut candidate = disposition.clone();
8502                    if let Err(reason) = candidate.register_partition(source, true) {
8503                        *source_report
8504                            .fallback_reasons
8505                            .entry(reason.to_string())
8506                            .or_default() += 1;
8507                        continue;
8508                    }
8509                    disposition = candidate;
8510                }
8511                direct_families = direct_families.saturating_add(1);
8512                direct_cells = direct_cells.saturating_add(prepared.direct_cells);
8513                direct_partition_families = direct_partition_families.saturating_add(1);
8514                direct_partition_cells =
8515                    direct_partition_cells.saturating_add(prepared.direct_cells);
8516                direct_fragments =
8517                    direct_fragments.saturating_add(prepared.placements.len() as u64);
8518                ordered_placements.push((source.source_order, prepared.placements));
8519            }
8520        }
8521        ordered_placements.sort_by_key(|(source_order, _)| *source_order);
8522        let placements = ordered_placements
8523            .into_iter()
8524            .flat_map(|(_, placements)| placements)
8525            .collect();
8526
8527        Ok(PreparedTargetSourcePackage {
8528            sheet: sheet.to_string(),
8529            sheet_id,
8530            lease,
8531            source_report,
8532            replay_records,
8533            disposition,
8534            placements,
8535            legacy: Vec::new(),
8536            direct_families,
8537            direct_cells,
8538            direct_fragments,
8539            direct_complete_families,
8540            direct_complete_cells,
8541            direct_partition_families,
8542            direct_partition_cells,
8543            anchor_parses,
8544            anchor_asts,
8545            anchor_analyses,
8546        })
8547    }
8548
8549    fn fallback_planning_snapshot(
8550        &self,
8551        batch: &FormulaIngestBatch,
8552    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
8553        let mut requests = Vec::new();
8554        for formula in &batch.formulas {
8555            let ast = self
8556                .graph
8557                .data_store()
8558                .retrieve_ast(formula.ast_id, self.graph.sheet_reg())
8559                .ok_or_else(|| {
8560                    ExcelError::new(ExcelErrorKind::Value)
8561                        .with_message("ordered fallback AST is unavailable")
8562                })?;
8563            Self::collect_planning_function_requests(&ast, &mut requests);
8564        }
8565        requests.sort();
8566        requests.dedup();
8567        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
8568            &self.resolver,
8569            requests,
8570        )
8571        .map_err(|error| ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}")))
8572    }
8573
8574    fn prepare_legacy_batch_fallback(
8575        &mut self,
8576        batch: FormulaIngestBatch,
8577        function_provider: &dyn crate::traits::FunctionProvider,
8578    ) -> Result<
8579        (
8580            crate::engine::graph::prepared_legacy_graph::PreparedLegacyGraphPlan,
8581            u64,
8582        ),
8583        ExcelError,
8584    > {
8585        let formula_count = batch.formulas.len() as u64;
8586        let sheet_id = self.graph.sheet_id_mut(&batch.sheet_name);
8587        let mut planned = Vec::with_capacity(batch.formulas.len());
8588        for record in batch.formulas {
8589            let placement = CellRef::new(
8590                sheet_id,
8591                Coord::from_excel(record.row, record.col, true, true),
8592            );
8593            let ingested = self
8594                .graph
8595                .ingest_pipeline(function_provider)
8596                .enable_function_semantics()
8597                .ingest_formula(
8598                    FormulaAstInput::RawArena(record.ast_id),
8599                    placement,
8600                    record.formula_text,
8601                )
8602                .map_err(|error| {
8603                    ExcelError::new(ExcelErrorKind::Value).with_message(format!("{error:?}"))
8604                })?;
8605            planned.push((record.row, record.col, ingested.ast_id, ingested.dep_plan));
8606        }
8607        let plan = self
8608            .graph
8609            .prepare_legacy_graph_plan(sheet_id, planned)
8610            .map_err(|error| {
8611                ExcelError::new(ExcelErrorKind::Value).with_message(error.to_string())
8612            })?;
8613        Ok((plan, formula_count))
8614    }
8615
8616    fn publish_compressed_partial_report(
8617        &mut self,
8618        report: &FormulaIngestReport,
8619        direct_report: &FormulaIngestReport,
8620    ) {
8621        if direct_report.source_family_promoted == 0
8622            && direct_report.graph_formula_cells_materialized == 0
8623        {
8624            return;
8625        }
8626        let mut published = report.clone();
8627        published.accumulate(direct_report);
8628        self.record_formula_ingest_report(published);
8629    }
8630
8631    fn finish_compressed_formula_sources(
8632        &mut self,
8633        batches: Vec<(
8634            FormulaIngestBatch,
8635            crate::engine::FormulaCompressedSourceReport,
8636            crate::engine::FormulaCompressedPreparation,
8637        )>,
8638    ) -> Result<FormulaIngestReport, ExcelError> {
8639        self.observe_function_semantic_epoch()?;
8640        if batches.iter().any(|(_, _, preparation)| {
8641            !Arc::ptr_eq(&preparation.engine_token, &self.source_formula_token)
8642        }) {
8643            return Err(ExcelError::new(ExcelErrorKind::Value)
8644                .with_message("compressed source preparation belongs to another engine"));
8645        }
8646        if batches.iter().any(|(fallback, _, preparation)| {
8647            preparation.sheet_name.as_ref() != fallback.sheet_name
8648        }) {
8649            return Err(ExcelError::new(ExcelErrorKind::Value)
8650                .with_message("compressed source preparation sheet mismatch"));
8651        }
8652        let initial_guard = crate::function_registry::semantic_epoch_read_guard();
8653        let initial_provider_revision = self.resolver.planning_semantic_revision();
8654        let mut fallback_batches = Vec::with_capacity(batches.len());
8655        let mut stale_fallback_batches = Vec::new();
8656        let mut pending_preparations = Vec::new();
8657        for (mut fallback, mut source, mut preparation) in batches {
8658            for formula in fallback.formulas.drain(..) {
8659                let source_order = formula.source_order.ok_or_else(|| {
8660                    ExcelError::new(ExcelErrorKind::Value).with_message(
8661                        "compressed source supplied formulas without source-order proof",
8662                    )
8663                })?;
8664                let text = formula.formula_text.ok_or_else(|| {
8665                    ExcelError::new(ExcelErrorKind::Value).with_message(
8666                        "ordered compressed fallback formula has no exact source text",
8667                    )
8668                })?;
8669                preparation
8670                    .eager_replay
8671                    .push(crate::engine::DeferredReplayFormula {
8672                        source_order,
8673                        row: formula.row,
8674                        col: formula.col,
8675                        text: text.to_string(),
8676                        family: formula.source_family,
8677                        partition_owner: formula.partition_owner,
8678                    });
8679            }
8680            preparation
8681                .eager_replay
8682                .sort_by_key(|record| record.source_order);
8683            source.source_spool_replays = source
8684                .source_spool_replays
8685                .saturating_add(preparation.preparation_spool_replays);
8686            let stale_reason = preparation
8687                .function_semantics_used
8688                .then(|| {
8689                    if preparation.function_provider_revision != initial_provider_revision {
8690                        Some("FunctionProviderRevisionChanged")
8691                    } else if self.prepared_function_semantics_changed(&preparation, &initial_guard)
8692                    {
8693                        Some("FunctionSemanticEpochChanged")
8694                    } else {
8695                        None
8696                    }
8697                })
8698                .flatten();
8699            let stale_semantics = stale_reason.is_some();
8700            if let Some(reason) = stale_reason {
8701                for (family, _, _) in &preparation.prepared {
8702                    preparation.rejected.insert(*family, reason.to_string());
8703                }
8704            }
8705            for reason in preparation.rejected.values() {
8706                *source.fallback_reasons.entry(reason.clone()).or_default() += 1;
8707            }
8708            let clean_direct_cells: u64 = preparation
8709                .prepared
8710                .iter()
8711                .map(|(_, _, prepared)| prepared.member_count)
8712                .sum();
8713            let fragmented_direct_cells: u64 = preparation
8714                .fragmented
8715                .iter()
8716                .map(|fragment| fragment.source.surviving_member_count)
8717                .sum();
8718            let direct_cells = clean_direct_cells.saturating_add(fragmented_direct_cells);
8719            source.replay_families = source.families_seen.saturating_sub(
8720                preparation
8721                    .prepared
8722                    .len()
8723                    .saturating_add(preparation.fragmented.len()) as u64,
8724            );
8725            source.replay_cells = source.family_cells_seen.saturating_sub(direct_cells);
8726            let compressed = crate::engine::FormulaCompressedSourceBatch::new(
8727                fallback.sheet_name.clone(),
8728                source,
8729            );
8730            if stale_semantics {
8731                stale_fallback_batches.push((fallback, compressed));
8732            } else {
8733                fallback_batches.push((fallback, compressed));
8734            }
8735            pending_preparations.push((preparation, stale_semantics));
8736        }
8737        drop(initial_guard);
8738
8739        // Build known fallback graphs first. Stale batches are forced through legacy ingest.
8740        let configured_mode = self.config.formula_plane_mode;
8741        self.config.formula_plane_mode = FormulaPlaneMode::Off;
8742        let stale_result =
8743            self.ingest_compressed_formula_source_batches_inner(stale_fallback_batches, false);
8744        self.config.formula_plane_mode = configured_mode;
8745        let mut report = stale_result?;
8746        report.mode = configured_mode;
8747
8748        self.config.formula_plane_mode = FormulaPlaneMode::Off;
8749        let fallback_result =
8750            self.ingest_compressed_formula_source_batches_inner(fallback_batches, false);
8751        self.config.formula_plane_mode = configured_mode;
8752        match fallback_result {
8753            Ok(fallback_report) => report.accumulate(&fallback_report),
8754            Err(error) => {
8755                self.record_formula_ingest_report(report);
8756                return Err(error);
8757            }
8758        }
8759
8760        let mut direct_report =
8761            FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
8762        for (preparation, _) in &pending_preparations {
8763            direct_report.source_anchor_parses = direct_report.source_anchor_parses.saturating_add(
8764                preparation.clean_rejected_anchor_counts[0]
8765                    .saturating_add(preparation.fragmented_rejected_anchor_counts[0]),
8766            );
8767            direct_report.source_anchor_asts = direct_report.source_anchor_asts.saturating_add(
8768                preparation.clean_rejected_anchor_counts[1]
8769                    .saturating_add(preparation.fragmented_rejected_anchor_counts[1]),
8770            );
8771            direct_report.source_anchor_analyses =
8772                direct_report.source_anchor_analyses.saturating_add(
8773                    preparation.clean_rejected_anchor_counts[2]
8774                        .saturating_add(preparation.fragmented_rejected_anchor_counts[2]),
8775                );
8776            let rejected_partitions: Vec<_> = preparation
8777                .rejected
8778                .keys()
8779                .filter_map(|source_id| preparation.fragmented_sources.get(source_id))
8780                .collect();
8781            direct_report.source_partitioned_families_rejected = direct_report
8782                .source_partitioned_families_rejected
8783                .saturating_add(rejected_partitions.len() as u64);
8784            direct_report.source_partition_failures = direct_report
8785                .source_partition_failures
8786                .saturating_add(rejected_partitions.len() as u64);
8787            direct_report.source_partition_fallback_cells = direct_report
8788                .source_partition_fallback_cells
8789                .saturating_add(
8790                    rejected_partitions
8791                        .iter()
8792                        .map(|source| source.surviving_member_count)
8793                        .sum::<u64>(),
8794                );
8795        }
8796        loop {
8797            #[cfg(any(test, feature = "test-support"))]
8798            if let Some(hook) = self.before_prepared_span_commit_hook.take() {
8799                hook();
8800            }
8801            let commit_guard = crate::function_registry::semantic_epoch_read_guard();
8802            let commit_provider_revision = self.resolver.planning_semantic_revision();
8803            let mut newly_stale = Vec::new();
8804            let mut current = Vec::new();
8805            for pending in pending_preparations.drain(..) {
8806                let stale_reason = pending
8807                    .0
8808                    .function_semantics_used
8809                    .then(|| {
8810                        if pending.0.function_provider_revision != commit_provider_revision {
8811                            Some("FunctionProviderRevisionChanged")
8812                        } else if self
8813                            .prepared_function_semantics_changed(&pending.0, &commit_guard)
8814                        {
8815                            Some("FunctionSemanticEpochChanged")
8816                        } else {
8817                            None
8818                        }
8819                    })
8820                    .flatten();
8821                if let Some(reason) = stale_reason {
8822                    newly_stale.push((pending, reason));
8823                } else {
8824                    current.push(pending);
8825                }
8826            }
8827            if !newly_stale.is_empty() {
8828                drop(commit_guard);
8829                for ((mut preparation, was_initially_stale), reason) in newly_stale {
8830                    for (family, _, _) in &preparation.prepared {
8831                        preparation.rejected.insert(*family, reason.to_string());
8832                    }
8833                    let exact = match self.replay_prepared_families_exact_records(&preparation) {
8834                        Ok(exact) => exact,
8835                        Err(error) => {
8836                            self.publish_compressed_partial_report(&report, &direct_report);
8837                            return Err(error);
8838                        }
8839                    };
8840                    preparation.eager_replay.extend(exact);
8841                    preparation
8842                        .eager_replay
8843                        .sort_by_key(|record| record.source_order);
8844                    direct_report.source_spool_replays =
8845                        direct_report.source_spool_replays.saturating_add(1);
8846                    if !was_initially_stale {
8847                        *direct_report
8848                            .fallback_reasons
8849                            .entry(reason.to_string())
8850                            .or_default() += preparation.prepared.len() as u64;
8851                    }
8852                    direct_report.source_family_fallback = direct_report
8853                        .source_family_fallback
8854                        .saturating_add(preparation.prepared.len() as u64);
8855                    direct_report.source_family_fallback_cells =
8856                        direct_report.source_family_fallback_cells.saturating_add(
8857                            preparation
8858                                .prepared
8859                                .iter()
8860                                .map(|(_, _, prepared)| prepared.member_count)
8861                                .sum(),
8862                        );
8863                    preparation.prepared.clear();
8864                    preparation.function_semantics_used = false;
8865                    if !preparation.fragmented.is_empty() || !preparation.eager_replay.is_empty() {
8866                        current.push((preparation, false));
8867                    }
8868                }
8869                pending_preparations = current;
8870                continue;
8871            }
8872            drop(commit_guard);
8873
8874            // Preserve source order across clean and fragmented families. Every clean
8875            // placement uses E3's incremental append, so no global index rebuild is needed.
8876            enum SourceProposal {
8877                KnownFallback {
8878                    source_order: crate::engine::SourceFormulaOrder,
8879                    records: Vec<crate::engine::DeferredReplayFormula>,
8880                },
8881                Clean(
8882                    crate::engine::SourceFamilyId,
8883                    crate::engine::SourceFormulaOrder,
8884                    crate::formula_plane::placement::PreparedAnchorOncePlacement,
8885                ),
8886                Fragment(crate::engine::formula_source::PreparedFragmentedSourceProposal),
8887            }
8888
8889            impl SourceProposal {
8890                fn source_order(&self) -> crate::engine::SourceFormulaOrder {
8891                    match self {
8892                        Self::KnownFallback { source_order, .. } => *source_order,
8893                        Self::Clean(_, source_order, _) => *source_order,
8894                        Self::Fragment(fragment) => fragment.source.source_order,
8895                    }
8896                }
8897            }
8898
8899            for (mut preparation, _) in current {
8900                let mut proposals = Vec::with_capacity(
8901                    preparation
8902                        .prepared
8903                        .len()
8904                        .saturating_add(preparation.fragmented.len())
8905                        .saturating_add(preparation.eager_replay.len()),
8906                );
8907                proposals.extend(preparation.prepared.drain(..).map(
8908                    |(source_id, source_order, prepared)| {
8909                        SourceProposal::Clean(source_id, source_order, prepared)
8910                    },
8911                ));
8912
8913                let accepted_fragments: BTreeMap<_, _> = preparation
8914                    .fragmented
8915                    .iter()
8916                    .map(|fragment| (fragment.source.source_id, &fragment.source))
8917                    .collect();
8918                let mut family_fallbacks: BTreeMap<_, Vec<_>> = BTreeMap::new();
8919                for record in preparation.eager_replay.drain(..).filter(|record| {
8920                    !record.partition_owner.is_some_and(|owner| {
8921                        accepted_fragments.get(&owner).is_some_and(|source| {
8922                            Self::exact_replay_record_is_prepared_partition_legacy(source, record)
8923                        })
8924                    })
8925                }) {
8926                    if let Some(owner) = record.partition_owner.or(record.family) {
8927                        family_fallbacks.entry(owner).or_default().push(record);
8928                    } else {
8929                        proposals.push(SourceProposal::KnownFallback {
8930                            source_order: record.source_order,
8931                            records: vec![record],
8932                        });
8933                    }
8934                }
8935                for (_, mut records) in family_fallbacks {
8936                    records.sort_by_key(|record| record.source_order);
8937                    if records
8938                        .windows(2)
8939                        .any(|window| window[0].source_order == window[1].source_order)
8940                    {
8941                        self.publish_compressed_partial_report(&report, &direct_report);
8942                        return Err(ExcelError::new(ExcelErrorKind::Value)
8943                            .with_message("duplicate exact-replay source-order proof"));
8944                    }
8945                    let Some(source_order) = records.first().map(|record| record.source_order)
8946                    else {
8947                        self.publish_compressed_partial_report(&report, &direct_report);
8948                        return Err(ExcelError::new(ExcelErrorKind::Value)
8949                            .with_message("empty exact-replay fallback family"));
8950                    };
8951                    proposals.push(SourceProposal::KnownFallback {
8952                        source_order,
8953                        records,
8954                    });
8955                }
8956                proposals.extend(
8957                    preparation
8958                        .fragmented
8959                        .drain(..)
8960                        .map(SourceProposal::Fragment),
8961                );
8962                proposals.sort_by_key(SourceProposal::source_order);
8963                if proposals
8964                    .windows(2)
8965                    .any(|window| window[0].source_order() == window[1].source_order())
8966                {
8967                    self.publish_compressed_partial_report(&report, &direct_report);
8968                    return Err(ExcelError::new(ExcelErrorKind::Value)
8969                        .with_message("ambiguous compressed source-order proof"));
8970                }
8971
8972                for proposal in proposals {
8973                    match proposal {
8974                        SourceProposal::KnownFallback { records, .. } => {
8975                            let batch = match self.formula_batch_from_exact_replay(
8976                                preparation.sheet_name.as_ref(),
8977                                records,
8978                            ) {
8979                                Ok(batch) => batch,
8980                                Err(error) => {
8981                                    self.publish_compressed_partial_report(&report, &direct_report);
8982                                    return Err(error);
8983                                }
8984                            };
8985                            if batch.is_empty() {
8986                                continue;
8987                            }
8988                            let snapshot = match self.fallback_planning_snapshot(&batch) {
8989                                Ok(snapshot) => snapshot,
8990                                Err(error) => {
8991                                    self.publish_compressed_partial_report(&report, &direct_report);
8992                                    return Err(error);
8993                                }
8994                            };
8995                            let commit_guard =
8996                                crate::function_registry::semantic_epoch_read_guard();
8997                            let provider_revision_initial =
8998                                self.resolver.planning_semantic_revision();
8999                            if (commit_guard.epoch() != snapshot.epoch()
9000                                && snapshot.semantic_changes_affect_requests_since_guarded(
9001                                    &commit_guard,
9002                                    snapshot.epoch(),
9003                                ))
9004                                || snapshot.provider_revision().is_some_and(|revision| {
9005                                    Some(revision) != provider_revision_initial
9006                                })
9007                            {
9008                                drop(commit_guard);
9009                                self.publish_compressed_partial_report(&report, &direct_report);
9010                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9011                                    "ordered fallback planning snapshot became stale",
9012                                ));
9013                            }
9014                            let (plan, formula_count) = match self
9015                                .prepare_legacy_batch_fallback(batch, &snapshot)
9016                            {
9017                                Ok(plan) => plan,
9018                                Err(error) => {
9019                                    drop(commit_guard);
9020                                    self.publish_compressed_partial_report(&report, &direct_report);
9021                                    return Err(error);
9022                                }
9023                            };
9024                            let provider_revision_after =
9025                                self.resolver.planning_semantic_revision();
9026                            if provider_revision_after != provider_revision_initial {
9027                                drop(commit_guard);
9028                                self.publish_compressed_partial_report(&report, &direct_report);
9029                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9030                                    "function provider changed while preparing ordered fallback",
9031                                ));
9032                            }
9033                            let graph_vertices = plan.new_vertex_count();
9034                            let Some(graph_edges) = plan.planned_edge_count() else {
9035                                drop(commit_guard);
9036                                self.publish_compressed_partial_report(&report, &direct_report);
9037                                return Err(ExcelError::new(ExcelErrorKind::Value)
9038                                    .with_message("prepared ordered fallback size overflow"));
9039                            };
9040                            if let Err(error) = self.prepared_legacy_admission(&plan, formula_count)
9041                            {
9042                                drop(commit_guard);
9043                                self.publish_compressed_partial_report(&report, &direct_report);
9044                                return Err(error);
9045                            }
9046                            if let Err(error) =
9047                                self.graph.validate_prepared_legacy_graph_plan(&plan)
9048                            {
9049                                drop(commit_guard);
9050                                self.publish_compressed_partial_report(&report, &direct_report);
9051                                return Err(ExcelError::new(ExcelErrorKind::Value)
9052                                    .with_message(error.to_string()));
9053                            }
9054                            #[cfg(test)]
9055                            if let Some(hook) = self
9056                                .before_legacy_fallback_final_provider_sample_hook
9057                                .take()
9058                            {
9059                                hook();
9060                            }
9061                            let provider_revision_final =
9062                                self.resolver.planning_semantic_revision();
9063                            if provider_revision_final != provider_revision_initial {
9064                                drop(commit_guard);
9065                                self.publish_compressed_partial_report(&report, &direct_report);
9066                                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9067                                    "function provider changed after ordered fallback validation",
9068                                ));
9069                            }
9070                            let graph_formulas =
9071                                self.graph.apply_prevalidated_legacy_graph_plan(plan);
9072                            direct_report.formula_cells_seen = direct_report
9073                                .formula_cells_seen
9074                                .saturating_add(formula_count);
9075                            direct_report.graph_formula_cells_materialized = direct_report
9076                                .graph_formula_cells_materialized
9077                                .saturating_add(graph_formulas as u64);
9078                            direct_report.graph_vertices_created = direct_report
9079                                .graph_vertices_created
9080                                .saturating_add(graph_vertices as u64);
9081                            direct_report.graph_edges_created = direct_report
9082                                .graph_edges_created
9083                                .saturating_add(graph_edges as u64);
9084                        }
9085                        SourceProposal::Clean(source_id, _, prepared) => {
9086                            let commit_guard =
9087                                crate::function_registry::semantic_epoch_read_guard();
9088                            let commit_provider_revision =
9089                                self.resolver.planning_semantic_revision();
9090                            let semantic_changed = preparation.function_semantics_used
9091                                && self.prepared_placement_function_semantics_changed(
9092                                    preparation.function_semantic_epoch,
9093                                    &prepared,
9094                                    &commit_guard,
9095                                );
9096                            let cells = prepared.member_count;
9097                            let stats = self.graph.baseline_stats();
9098                            self.preflight_graph_admission(
9099                                crate::engine::resource_ledger::GraphAdmission {
9100                                    final_vertices: stats.graph_vertex_count,
9101                                    final_edges: stats.graph_edge_count,
9102                                    materialization_cells: 0,
9103                                    added_vertices: 0,
9104                                    added_edges: 0,
9105                                },
9106                            )?;
9107                            let append = self
9108                                .graph
9109                                .formula_authority()
9110                                .prepare_formula_plane_append(
9111                                    vec![prepared],
9112                                    self.graph.data_store(),
9113                                    self.graph.sheet_reg(),
9114                                )
9115                                .map_err(|error| error.to_string())
9116                                .and_then(|append| {
9117                                    if preparation.function_semantics_used
9118                                        && preparation.function_provider_revision
9119                                            != commit_provider_revision
9120                                    {
9121                                        return Err("FunctionProviderRevisionChanged".to_string());
9122                                    }
9123                                    if preparation.function_semantics_used
9124                                        && self.resolver.planning_semantic_revision()
9125                                            != commit_provider_revision
9126                                    {
9127                                        return Err("FunctionProviderRevisionChanged".to_string());
9128                                    }
9129                                    if semantic_changed {
9130                                        return Err("FunctionSemanticEpochChanged".to_string());
9131                                    }
9132                                    self.graph
9133                                        .formula_authority()
9134                                        .validate_prepared_formula_plane_append(
9135                                            &append,
9136                                            self.graph.data_store(),
9137                                            self.graph.sheet_reg(),
9138                                        )
9139                                        .map_err(|error| error.to_string())?;
9140                                    Ok(append)
9141                                });
9142                            match append {
9143                                Ok(append) => {
9144                                    let placement_report = self
9145                                        .graph
9146                                        .formula_authority_mut()
9147                                        .apply_prevalidated_formula_plane_append(append);
9148                                    self.graph.mark_formula_spans_dirty(
9149                                        placement_report.spans.iter().copied(),
9150                                        WholeSpanDirtyReason::NewSpan,
9151                                    );
9152                                    direct_report.formula_cells_seen =
9153                                        direct_report.formula_cells_seen.saturating_add(cells);
9154                                    direct_report.shadow_candidate_cells =
9155                                        direct_report.shadow_candidate_cells.saturating_add(cells);
9156                                    direct_report.shadow_accepted_span_cells = direct_report
9157                                        .shadow_accepted_span_cells
9158                                        .saturating_add(cells);
9159                                    direct_report.shadow_templates_interned =
9160                                        direct_report.shadow_templates_interned.saturating_add(
9161                                            placement_report.work.templates_to_append as u64,
9162                                        );
9163                                    direct_report.shadow_spans_created = direct_report
9164                                        .shadow_spans_created
9165                                        .saturating_add(placement_report.spans.len() as u64);
9166                                    direct_report.graph_formula_vertices_avoided_shadow =
9167                                        direct_report
9168                                            .graph_formula_vertices_avoided_shadow
9169                                            .saturating_add(cells);
9170                                    direct_report.ast_roots_avoided_shadow = direct_report
9171                                        .ast_roots_avoided_shadow
9172                                        .saturating_add(cells.saturating_sub(1));
9173                                    direct_report.edge_rows_avoided_shadow = direct_report
9174                                        .edge_rows_avoided_shadow
9175                                        .saturating_add(cells);
9176                                    direct_report.source_family_promoted =
9177                                        direct_report.source_family_promoted.saturating_add(1);
9178                                    direct_report.source_family_promoted_cells = direct_report
9179                                        .source_family_promoted_cells
9180                                        .saturating_add(cells);
9181                                    direct_report.source_anchor_parses =
9182                                        direct_report.source_anchor_parses.saturating_add(1);
9183                                    direct_report.source_anchor_asts =
9184                                        direct_report.source_anchor_asts.saturating_add(1);
9185                                    direct_report.source_anchor_analyses =
9186                                        direct_report.source_anchor_analyses.saturating_add(1);
9187                                    let descendants = cells.saturating_sub(1);
9188                                    direct_report.source_descendant_strings_avoided = direct_report
9189                                        .source_descendant_strings_avoided
9190                                        .saturating_add(descendants);
9191                                    direct_report.source_descendant_events_avoided = direct_report
9192                                        .source_descendant_events_avoided
9193                                        .saturating_add(descendants);
9194                                    direct_report.source_descendant_analyses_avoided =
9195                                        direct_report
9196                                            .source_descendant_analyses_avoided
9197                                            .saturating_add(descendants);
9198                                    direct_report.source_compressed_families_prepared =
9199                                        direct_report
9200                                            .source_compressed_families_prepared
9201                                            .saturating_add(1);
9202                                    direct_report.source_compressed_cells_prepared = direct_report
9203                                        .source_compressed_cells_prepared
9204                                        .saturating_add(cells);
9205                                }
9206                                Err(error) => {
9207                                    drop(commit_guard);
9208                                    let reason = format!("CleanFormulaPlaneAppend:{error}");
9209                                    let batch = match self.replay_one_prepared_family_exact(
9210                                        &preparation,
9211                                        source_id,
9212                                        cells,
9213                                    ) {
9214                                        Ok(batch) => batch,
9215                                        Err(error) => {
9216                                            self.publish_compressed_partial_report(
9217                                                &report,
9218                                                &direct_report,
9219                                            );
9220                                            return Err(error);
9221                                        }
9222                                    };
9223                                    let snapshot = match self.fallback_planning_snapshot(&batch) {
9224                                        Ok(snapshot) => snapshot,
9225                                        Err(error) => {
9226                                            self.publish_compressed_partial_report(
9227                                                &report,
9228                                                &direct_report,
9229                                            );
9230                                            return Err(error);
9231                                        }
9232                                    };
9233                                    let commit_guard =
9234                                        crate::function_registry::semantic_epoch_read_guard();
9235                                    let provider_revision =
9236                                        self.resolver.planning_semantic_revision();
9237                                    if (commit_guard.epoch() != snapshot.epoch()
9238                                        && snapshot.semantic_changes_affect_requests_since_guarded(
9239                                            &commit_guard,
9240                                            snapshot.epoch(),
9241                                        ))
9242                                        || snapshot.provider_revision().is_some_and(|revision| {
9243                                            Some(revision) != provider_revision
9244                                        })
9245                                    {
9246                                        drop(commit_guard);
9247                                        self.publish_compressed_partial_report(
9248                                            &report,
9249                                            &direct_report,
9250                                        );
9251                                        return Err(ExcelError::new(ExcelErrorKind::Value)
9252                                            .with_message(
9253                                                "clean-family fallback planning snapshot became stale",
9254                                            ));
9255                                    }
9256                                    let (plan, formula_count) = match self
9257                                        .prepare_legacy_batch_fallback(batch, &snapshot)
9258                                    {
9259                                        Ok(plan) => plan,
9260                                        Err(error) => {
9261                                            drop(commit_guard);
9262                                            self.publish_compressed_partial_report(
9263                                                &report,
9264                                                &direct_report,
9265                                            );
9266                                            return Err(error);
9267                                        }
9268                                    };
9269                                    let provider_revision_after =
9270                                        self.resolver.planning_semantic_revision();
9271                                    if provider_revision_after != provider_revision {
9272                                        drop(commit_guard);
9273                                        self.publish_compressed_partial_report(
9274                                            &report,
9275                                            &direct_report,
9276                                        );
9277                                        return Err(ExcelError::new(ExcelErrorKind::Value)
9278                                            .with_message(
9279                                                "function provider changed while preparing clean-family fallback",
9280                                            ));
9281                                    }
9282                                    let graph_vertices = plan.new_vertex_count();
9283                                    let Some(graph_edges) = plan.planned_edge_count() else {
9284                                        drop(commit_guard);
9285                                        self.publish_compressed_partial_report(
9286                                            &report,
9287                                            &direct_report,
9288                                        );
9289                                        return Err(ExcelError::new(ExcelErrorKind::Value)
9290                                            .with_message(
9291                                                "prepared clean-family fallback size overflow",
9292                                            ));
9293                                    };
9294                                    if let Err(error) =
9295                                        self.prepared_legacy_admission(&plan, formula_count)
9296                                    {
9297                                        drop(commit_guard);
9298                                        self.publish_compressed_partial_report(
9299                                            &report,
9300                                            &direct_report,
9301                                        );
9302                                        return Err(error);
9303                                    }
9304                                    if let Err(error) =
9305                                        self.graph.validate_prepared_legacy_graph_plan(&plan)
9306                                    {
9307                                        drop(commit_guard);
9308                                        self.publish_compressed_partial_report(
9309                                            &report,
9310                                            &direct_report,
9311                                        );
9312                                        return Err(ExcelError::new(ExcelErrorKind::Value)
9313                                            .with_message(error.to_string()));
9314                                    }
9315                                    #[cfg(test)]
9316                                    if let Some(hook) = self
9317                                        .before_legacy_fallback_final_provider_sample_hook
9318                                        .take()
9319                                    {
9320                                        hook();
9321                                    }
9322                                    let provider_revision_final =
9323                                        self.resolver.planning_semantic_revision();
9324                                    if provider_revision_final != provider_revision {
9325                                        drop(commit_guard);
9326                                        self.publish_compressed_partial_report(
9327                                            &report,
9328                                            &direct_report,
9329                                        );
9330                                        return Err(ExcelError::new(ExcelErrorKind::Value)
9331                                            .with_message(
9332                                                "function provider changed after clean-family fallback validation",
9333                                            ));
9334                                    }
9335                                    let graph_formulas =
9336                                        self.graph.apply_prevalidated_legacy_graph_plan(plan);
9337                                    direct_report.formula_cells_seen = direct_report
9338                                        .formula_cells_seen
9339                                        .saturating_add(formula_count);
9340                                    direct_report.graph_formula_cells_materialized = direct_report
9341                                        .graph_formula_cells_materialized
9342                                        .saturating_add(graph_formulas as u64);
9343                                    direct_report.graph_vertices_created = direct_report
9344                                        .graph_vertices_created
9345                                        .saturating_add(graph_vertices as u64);
9346                                    direct_report.graph_edges_created = direct_report
9347                                        .graph_edges_created
9348                                        .saturating_add(graph_edges as u64);
9349                                    direct_report.source_spool_replays =
9350                                        direct_report.source_spool_replays.saturating_add(1);
9351                                    direct_report.source_family_fallback =
9352                                        direct_report.source_family_fallback.saturating_add(1);
9353                                    direct_report.source_family_fallback_cells = direct_report
9354                                        .source_family_fallback_cells
9355                                        .saturating_add(cells);
9356                                    direct_report.source_anchor_parses =
9357                                        direct_report.source_anchor_parses.saturating_add(1);
9358                                    direct_report.source_anchor_asts =
9359                                        direct_report.source_anchor_asts.saturating_add(1);
9360                                    direct_report.source_anchor_analyses =
9361                                        direct_report.source_anchor_analyses.saturating_add(1);
9362                                    *direct_report.fallback_reasons.entry(reason).or_default() += 1;
9363                                }
9364                            }
9365                        }
9366                        SourceProposal::Fragment(fragment) => {
9367                            let commit_guard =
9368                                crate::function_registry::semantic_epoch_read_guard();
9369                            let commit_epoch = commit_guard.epoch();
9370                            let commit_provider_revision =
9371                                self.resolver.planning_semantic_revision();
9372                            let source = fragment.source;
9373                            let source_id = source.source_id;
9374                            #[cfg(test)]
9375                            let commit_fault = self
9376                                .fragmented_commit_fault_for_test
9377                                .take()
9378                                .unwrap_or(
9379                                crate::engine::fragmented_transaction::FragmentedCommitFault::None,
9380                            );
9381                            #[cfg(not(test))]
9382                            let commit_fault =
9383                                crate::engine::fragmented_transaction::FragmentedCommitFault::None;
9384                            let outcome = self
9385                                .prepare_fragmented_source_transaction(
9386                                    &source,
9387                                    &preparation.replay_disposition,
9388                                    fragment.prepared,
9389                                    fragment.legacy,
9390                                )
9391                                .map(|transaction| {
9392                                    let resolver = &self.resolver;
9393                                    self.graph.commit_fragmented_source_transaction(
9394                                        transaction,
9395                                        &self.source_formula_token,
9396                                        &preparation.replay_disposition,
9397                                        commit_epoch,
9398                                        commit_provider_revision,
9399                                        || resolver.planning_semantic_revision(),
9400                                        commit_fault,
9401                                    )
9402                                });
9403                            let fallback_reason = match outcome {
9404                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::Committed(success)) => {
9405                                    let mut delta = success.report_delta;
9406                                    delta.mode = FormulaPlaneMode::AuthoritativeExperimental;
9407                                    delta.source_family_promoted = 1;
9408                                    delta.source_family_promoted_cells =
9409                                        source.surviving_member_count;
9410                                    delta.source_partitioned_families_seen = 0;
9411                                    delta.source_partition_holes = 0;
9412                                    delta.source_partition_ordinary_exceptions = 0;
9413                                    delta.source_partition_surviving_cells = 0;
9414                                    delta.shadow_templates_interned =
9415                                        success.plane.work.templates_to_append as u64;
9416                                    delta.shadow_spans_created = success.plane.spans.len() as u64;
9417                                    let descendants =
9418                                        source.surviving_member_count.saturating_sub(1);
9419                                    delta.source_descendant_strings_avoided = descendants;
9420                                    delta.source_descendant_events_avoided = descendants;
9421                                    delta.source_descendant_analyses_avoided = descendants;
9422                                    direct_report.accumulate(&delta);
9423                                    None
9424                                }
9425                                Ok(crate::engine::fragmented_transaction::FragmentedCommitDecision::ReplayWholeFamily { reason, .. }) => {
9426                                    Some(format!("{reason:?}"))
9427                                }
9428                                Err(error) => Some(format!("{error:?}")),
9429                            };
9430                            if let Some(reason) = fallback_reason {
9431                                drop(commit_guard);
9432                                let mut replay_disposition = preparation.replay_disposition.clone();
9433                                replay_disposition.force_family_legacy(source_id);
9434                                let replayed = (|| {
9435                                    let records = fragment
9436                                        .replay
9437                                        .lock()
9438                                        .map_err(|_| {
9439                                            ExcelError::new(ExcelErrorKind::Value).with_message(
9440                                                "compressed formula exact replay lock poisoned",
9441                                            )
9442                                        })?
9443                                        .replay_partitioned(
9444                                            &replay_disposition,
9445                                            std::slice::from_ref(&source),
9446                                        )
9447                                        .map_err(|message| {
9448                                            ExcelError::new(ExcelErrorKind::Value)
9449                                                .with_message(message)
9450                                        })?
9451                                        .into_iter()
9452                                        .filter(|record| {
9453                                            record.family == Some(source_id)
9454                                                || (record.family.is_none()
9455                                                    && record.partition_owner == Some(source_id))
9456                                        })
9457                                        .collect::<Vec<_>>();
9458                                    Self::validate_whole_partition_replay(&source, &records)?;
9459                                    Ok::<_, ExcelError>(records)
9460                                })();
9461                                let replayed = match replayed {
9462                                    Ok(replayed) => replayed,
9463                                    Err(error) => {
9464                                        self.publish_compressed_partial_report(
9465                                            &report,
9466                                            &direct_report,
9467                                        );
9468                                        return Err(error);
9469                                    }
9470                                };
9471                                let batch = match self.formula_batch_from_exact_replay(
9472                                    preparation.sheet_name.as_ref(),
9473                                    replayed,
9474                                ) {
9475                                    Ok(batch) => batch,
9476                                    Err(error) => {
9477                                        self.publish_compressed_partial_report(
9478                                            &report,
9479                                            &direct_report,
9480                                        );
9481                                        return Err(error);
9482                                    }
9483                                };
9484                                let snapshot = match self.fallback_planning_snapshot(&batch) {
9485                                    Ok(snapshot) => snapshot,
9486                                    Err(error) => {
9487                                        self.publish_compressed_partial_report(
9488                                            &report,
9489                                            &direct_report,
9490                                        );
9491                                        return Err(error);
9492                                    }
9493                                };
9494                                let commit_guard =
9495                                    crate::function_registry::semantic_epoch_read_guard();
9496                                let provider_revision = self.resolver.planning_semantic_revision();
9497                                if (commit_guard.epoch() != snapshot.epoch()
9498                                    && snapshot.semantic_changes_affect_requests_since_guarded(
9499                                        &commit_guard,
9500                                        snapshot.epoch(),
9501                                    ))
9502                                    || snapshot
9503                                        .provider_revision()
9504                                        .is_some_and(|revision| Some(revision) != provider_revision)
9505                                {
9506                                    self.publish_compressed_partial_report(&report, &direct_report);
9507                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9508                                        format!(
9509                                            "whole-family fallback planning snapshot became stale: epoch {} != {}, provider {:?} != {:?}",
9510                                            snapshot.epoch(),
9511                                            commit_guard.epoch(),
9512                                            snapshot.provider_revision(),
9513                                            provider_revision,
9514                                        ),
9515                                    ));
9516                                }
9517                                let (plan, formula_count) =
9518                                    match self.prepare_legacy_batch_fallback(batch, &snapshot) {
9519                                        Ok(plan) => plan,
9520                                        Err(error) => {
9521                                            drop(commit_guard);
9522                                            self.publish_compressed_partial_report(
9523                                                &report,
9524                                                &direct_report,
9525                                            );
9526                                            return Err(error);
9527                                        }
9528                                    };
9529                                let provider_revision_after =
9530                                    self.resolver.planning_semantic_revision();
9531                                if provider_revision_after != provider_revision {
9532                                    drop(commit_guard);
9533                                    self.publish_compressed_partial_report(&report, &direct_report);
9534                                    return Err(ExcelError::new(ExcelErrorKind::Value)
9535                                        .with_message(
9536                                            "function provider changed while preparing whole-family fallback",
9537                                        ));
9538                                }
9539                                let graph_vertices = plan.new_vertex_count();
9540                                let Some(graph_edges) = plan.planned_edge_count() else {
9541                                    drop(commit_guard);
9542                                    self.publish_compressed_partial_report(&report, &direct_report);
9543                                    return Err(ExcelError::new(ExcelErrorKind::Value)
9544                                        .with_message(
9545                                            "prepared whole-family fallback size overflow",
9546                                        ));
9547                                };
9548                                if let Err(error) =
9549                                    self.prepared_legacy_admission(&plan, formula_count)
9550                                {
9551                                    drop(commit_guard);
9552                                    self.publish_compressed_partial_report(&report, &direct_report);
9553                                    return Err(error);
9554                                }
9555                                if let Err(error) =
9556                                    self.graph.validate_prepared_legacy_graph_plan(&plan)
9557                                {
9558                                    drop(commit_guard);
9559                                    self.publish_compressed_partial_report(&report, &direct_report);
9560                                    return Err(ExcelError::new(ExcelErrorKind::Value)
9561                                        .with_message(error.to_string()));
9562                                }
9563                                #[cfg(test)]
9564                                if let Some(hook) = self
9565                                    .before_legacy_fallback_final_provider_sample_hook
9566                                    .take()
9567                                {
9568                                    hook();
9569                                }
9570                                let provider_revision_final =
9571                                    self.resolver.planning_semantic_revision();
9572                                if provider_revision_final != provider_revision {
9573                                    drop(commit_guard);
9574                                    self.publish_compressed_partial_report(&report, &direct_report);
9575                                    return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9576                                        "function provider changed after whole-family fallback validation",
9577                                    ));
9578                                }
9579                                let graph_formulas =
9580                                    self.graph.apply_prevalidated_legacy_graph_plan(plan);
9581                                direct_report.formula_cells_seen = direct_report
9582                                    .formula_cells_seen
9583                                    .saturating_add(formula_count);
9584                                direct_report.graph_formula_cells_materialized = direct_report
9585                                    .graph_formula_cells_materialized
9586                                    .saturating_add(graph_formulas as u64);
9587                                direct_report.graph_vertices_created = direct_report
9588                                    .graph_vertices_created
9589                                    .saturating_add(graph_vertices as u64);
9590                                direct_report.graph_edges_created = direct_report
9591                                    .graph_edges_created
9592                                    .saturating_add(graph_edges as u64);
9593                                direct_report.source_spool_replays =
9594                                    direct_report.source_spool_replays.saturating_add(1);
9595                                direct_report.source_family_fallback =
9596                                    direct_report.source_family_fallback.saturating_add(1);
9597                                direct_report.source_family_fallback_cells = direct_report
9598                                    .source_family_fallback_cells
9599                                    .saturating_add(source.surviving_member_count);
9600                                direct_report.source_partitioned_families_rejected = direct_report
9601                                    .source_partitioned_families_rejected
9602                                    .saturating_add(1);
9603                                direct_report.source_partition_failures =
9604                                    direct_report.source_partition_failures.saturating_add(1);
9605                                direct_report.source_partition_fallback_cells = direct_report
9606                                    .source_partition_fallback_cells
9607                                    .saturating_add(source.surviving_member_count);
9608                                direct_report.source_anchor_parses =
9609                                    direct_report.source_anchor_parses.saturating_add(1);
9610                                direct_report.source_anchor_asts =
9611                                    direct_report.source_anchor_asts.saturating_add(1);
9612                                direct_report.source_anchor_analyses =
9613                                    direct_report.source_anchor_analyses.saturating_add(1);
9614                                *direct_report.fallback_reasons.entry(reason).or_default() += 1;
9615                            }
9616                        }
9617                    }
9618                    #[cfg(test)]
9619                    if let Some(hook) = self.after_eager_proposal_commit_hook.take() {
9620                        hook();
9621                    }
9622                }
9623            }
9624            break;
9625        }
9626        report.accumulate(&direct_report);
9627        self.record_formula_ingest_report(report.clone());
9628        Ok(report)
9629    }
9630    /// Ingest replayed per-cell formulas while preserving compressed source counters.
9631    pub(crate) fn ingest_compressed_formula_source_batches(
9632        &mut self,
9633        batches: Vec<(
9634            FormulaIngestBatch,
9635            crate::engine::FormulaCompressedSourceBatch,
9636        )>,
9637    ) -> Result<FormulaIngestReport, ExcelError> {
9638        self.ingest_compressed_formula_source_batches_inner(batches, true)
9639    }
9640
9641    fn ingest_compressed_formula_source_batches_inner(
9642        &mut self,
9643        batches: Vec<(
9644            FormulaIngestBatch,
9645            crate::engine::FormulaCompressedSourceBatch,
9646        )>,
9647        publish_report: bool,
9648    ) -> Result<FormulaIngestReport, ExcelError> {
9649        let mut source_counts = [0_u64; 11];
9650        let mut source_report = crate::engine::FormulaCompressedSourceReport::default();
9651        let mut formula_batches = Vec::with_capacity(batches.len());
9652        let mut compressed_families = Vec::new();
9653        let mut partitioned_families = Vec::new();
9654        for (batch, compressed_batch) in batches {
9655            let (sheet_name, compressed, families, partitions) = compressed_batch.into_parts();
9656            if sheet_name.as_ref() != batch.sheet_name {
9657                return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
9658                    "compressed formula source sheet does not match its replay batch",
9659                ));
9660            }
9661            compressed_families.push((batch.sheet_name.clone(), families));
9662            partitioned_families.push((batch.sheet_name.clone(), partitions));
9663            source_counts[0] = source_counts[0].saturating_add(compressed.source_formula_events);
9664            source_counts[1] = source_counts[1].saturating_add(compressed.source_ordinary_events);
9665            source_counts[2] =
9666                source_counts[2].saturating_add(compressed.source_shared_anchor_events);
9667            source_counts[3] =
9668                source_counts[3].saturating_add(compressed.source_shared_descendant_events);
9669            source_counts[4] = source_counts[4].saturating_add(compressed.source_unknown_events);
9670            source_counts[5] =
9671                source_counts[5].saturating_add(compressed.source_formula_records_spooled);
9672            source_counts[6] =
9673                source_counts[6].saturating_add(compressed.source_spool_encoded_bytes);
9674            source_counts[7] = source_counts[7].max(compressed.source_spool_peak_memory_bytes);
9675            source_counts[8] =
9676                source_counts[8].saturating_add(compressed.source_spool_spilled_bytes);
9677            source_counts[9] = source_counts[9].saturating_add(compressed.source_spool_spill_files);
9678            source_counts[10] = source_counts[10].saturating_add(compressed.source_spool_replays);
9679            source_report.families_seen = source_report
9680                .families_seen
9681                .saturating_add(compressed.families_seen);
9682            source_report.family_cells_seen = source_report
9683                .family_cells_seen
9684                .saturating_add(compressed.family_cells_seen);
9685            source_report.source_clean_families = source_report
9686                .source_clean_families
9687                .saturating_add(compressed.source_clean_families);
9688            source_report.source_clean_cells = source_report
9689                .source_clean_cells
9690                .saturating_add(compressed.source_clean_cells);
9691            source_report.source_fragmentable_families = source_report
9692                .source_fragmentable_families
9693                .saturating_add(compressed.source_fragmentable_families);
9694            source_report.source_fragmentable_cells = source_report
9695                .source_fragmentable_cells
9696                .saturating_add(compressed.source_fragmentable_cells);
9697            source_report.source_fragment_count = source_report
9698                .source_fragment_count
9699                .saturating_add(compressed.source_fragment_count);
9700            source_report.source_isolated_fallback_cells = source_report
9701                .source_isolated_fallback_cells
9702                .saturating_add(compressed.source_isolated_fallback_cells);
9703            source_report.source_hole_exclusions = source_report
9704                .source_hole_exclusions
9705                .saturating_add(compressed.source_hole_exclusions);
9706            source_report.source_ordinary_exclusions = source_report
9707                .source_ordinary_exclusions
9708                .saturating_add(compressed.source_ordinary_exclusions);
9709            source_report.source_partition_failures = source_report
9710                .source_partition_failures
9711                .saturating_add(compressed.source_partition_failures);
9712            source_report.replay_families = source_report
9713                .replay_families
9714                .saturating_add(compressed.replay_families);
9715            source_report.replay_cells = source_report
9716                .replay_cells
9717                .saturating_add(compressed.replay_cells);
9718            source_report.forward_descendants = source_report
9719                .forward_descendants
9720                .saturating_add(compressed.forward_descendants);
9721            source_report.evidence_limit_fallbacks = source_report
9722                .evidence_limit_fallbacks
9723                .saturating_add(compressed.evidence_limit_fallbacks);
9724            source_report.evidence_peak_bytes = source_report
9725                .evidence_peak_bytes
9726                .max(compressed.evidence_peak_bytes);
9727            for (reason, count) in compressed.fallback_reasons {
9728                *source_report.fallback_reasons.entry(reason).or_default() += count;
9729            }
9730            formula_batches.push(batch);
9731        }
9732        self.ingest_formula_batches_inner(
9733            formula_batches,
9734            source_counts,
9735            Some(source_report),
9736            compressed_families,
9737            partitioned_families,
9738            publish_report,
9739        )
9740    }
9741
9742    pub fn ingest_formula_batches(
9743        &mut self,
9744        batches: Vec<FormulaIngestBatch>,
9745    ) -> Result<FormulaIngestReport, ExcelError> {
9746        let has_formulas = batches.iter().any(|batch| !batch.formulas.is_empty());
9747        let report = self.ingest_formula_batches_inner(
9748            batches,
9749            [0; 11],
9750            None,
9751            Vec::new(),
9752            Vec::new(),
9753            true,
9754        )?;
9755        if has_formulas {
9756            self.mark_topology_edited();
9757        }
9758        Ok(report)
9759    }
9760
9761    fn ingest_formula_batches_unpublished(
9762        &mut self,
9763        batches: Vec<FormulaIngestBatch>,
9764    ) -> Result<FormulaIngestReport, ExcelError> {
9765        self.ingest_formula_batches_inner(batches, [0; 11], None, Vec::new(), Vec::new(), false)
9766    }
9767
9768    fn ingest_formula_batches_inner(
9769        &mut self,
9770        batches: Vec<FormulaIngestBatch>,
9771        source_counts: [u64; 11],
9772        source_report: Option<crate::engine::FormulaCompressedSourceReport>,
9773        compressed_families: Vec<(String, Vec<crate::engine::SourceFormulaFamily>)>,
9774        partitioned_families: Vec<(String, Vec<crate::engine::PartitionedSourceFormulaFamily>)>,
9775        publish_report: bool,
9776    ) -> Result<FormulaIngestReport, ExcelError> {
9777        self.observe_function_semantic_epoch()?;
9778        let formula_cells_seen = batches.iter().map(|batch| batch.len() as u64).sum();
9779        #[cfg(feature = "benchmark_internal")]
9780        let benchmark_forced_replay =
9781            std::env::var_os("FORMUALIZER_BENCH_FORCE_FORMULA_FAMILY_REPLAY").is_some();
9782        #[cfg(not(feature = "benchmark_internal"))]
9783        let benchmark_forced_replay = false;
9784        let (mut report, materialize_batches, planned_materialize) = if benchmark_forced_replay
9785            && self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
9786        {
9787            let mut report =
9788                FormulaIngestReport::with_mode(FormulaPlaneMode::AuthoritativeExperimental);
9789            report
9790                .fallback_reasons
9791                .insert("ForcedReplay".to_string(), formula_cells_seen);
9792            (report, batches, BTreeMap::new())
9793        } else {
9794            match self.config.formula_plane_mode {
9795                FormulaPlaneMode::Off => (
9796                    FormulaIngestReport::with_mode(FormulaPlaneMode::Off),
9797                    batches,
9798                    BTreeMap::new(),
9799                ),
9800                FormulaPlaneMode::Shadow => (
9801                    if partitioned_families
9802                        .iter()
9803                        .all(|(_, families)| families.is_empty())
9804                    {
9805                        if compressed_families.is_empty() {
9806                            self.analyze_formula_plane_shadow_candidates(&batches)
9807                        } else {
9808                            self.analyze_compressed_formula_plane_shadow(&compressed_families)
9809                        }
9810                    } else {
9811                        let mut partition_report =
9812                            self.analyze_partitioned_formula_plane_shadow(&partitioned_families);
9813                        if compressed_families
9814                            .iter()
9815                            .any(|(_, families)| !families.is_empty())
9816                        {
9817                            partition_report.accumulate(
9818                                &self.analyze_compressed_formula_plane_shadow(&compressed_families),
9819                            );
9820                        }
9821                        partition_report
9822                    },
9823                    batches,
9824                    BTreeMap::new(),
9825                ),
9826                FormulaPlaneMode::AuthoritativeExperimental => {
9827                    self.analyze_formula_plane_authoritative_ingest(&batches)
9828                }
9829            }
9830        };
9831        report.formula_cells_seen = formula_cells_seen;
9832        report.source_formula_events = source_counts[0];
9833        report.source_ordinary_events = source_counts[1];
9834        report.source_shared_anchor_events = source_counts[2];
9835        report.source_shared_descendant_events = source_counts[3];
9836        report.source_unknown_events = source_counts[4];
9837        report.source_formula_records_spooled = source_counts[5];
9838        report.source_spool_encoded_bytes = source_counts[6];
9839        report.source_spool_peak_memory_bytes = source_counts[7];
9840        report.source_spool_spilled_bytes = source_counts[8];
9841        report.source_spool_spill_files = source_counts[9];
9842        report.source_spool_replays = source_counts[10];
9843        if let Some(source) = source_report {
9844            report.source_families_seen = source.families_seen;
9845            report.source_family_cells_seen = source.family_cells_seen;
9846            report.source_family_shadow_eligible = source.source_clean_families;
9847            report.source_family_shadow_eligible_cells = source.source_clean_cells;
9848            report.source_partitioned_families_seen = source.source_fragmentable_families;
9849            report.source_partition_holes = source.source_hole_exclusions;
9850            report.source_partition_ordinary_exceptions = source.source_ordinary_exclusions;
9851            report.source_partition_failures = source.source_partition_failures;
9852            report.source_partition_surviving_cells = source.source_fragmentable_cells;
9853            report.source_family_fallback = report
9854                .source_family_fallback
9855                .saturating_add(source.replay_families);
9856            report.source_family_fallback_cells = report
9857                .source_family_fallback_cells
9858                .saturating_add(source.replay_cells);
9859            report.source_forward_descendants = source.forward_descendants;
9860            report.source_evidence_limit_fallbacks = source.evidence_limit_fallbacks;
9861            report.source_evidence_peak_bytes = source.evidence_peak_bytes;
9862            for (reason, count) in source.fallback_reasons {
9863                let total = report.fallback_reasons.entry(reason).or_default();
9864                *total = total.saturating_add(count);
9865            }
9866        }
9867
9868        if !materialize_batches.iter().all(FormulaIngestBatch::is_empty)
9869            || !planned_materialize.is_empty()
9870        {
9871            let mut prepared_by_sheet = planned_materialize;
9872            for batch in materialize_batches {
9873                if batch.is_empty() {
9874                    continue;
9875                }
9876                let sheet_id = self.graph.sheet_id(&batch.sheet_name).ok_or_else(|| {
9877                    ExcelError::new(ExcelErrorKind::Ref)
9878                        .with_message(format!("unknown ingest sheet: {}", batch.sheet_name))
9879                })?;
9880                let mut pipeline = self.ingest_pipeline();
9881                let ingested = pipeline.ingest_batch(batch.formulas.into_iter().map(|record| {
9882                    let placement = CellRef::new(
9883                        sheet_id,
9884                        Coord::from_excel(record.row, record.col, true, true),
9885                    );
9886                    (
9887                        FormulaAstInput::RawArena(record.ast_id),
9888                        placement,
9889                        record.formula_text,
9890                    )
9891                }))?;
9892                prepared_by_sheet
9893                    .entry(batch.sheet_name)
9894                    .or_default()
9895                    .extend(ingested.into_iter().map(|formula| {
9896                        (
9897                            formula.placement.coord.row() + 1,
9898                            formula.placement.coord.col() + 1,
9899                            formula.ast_id,
9900                            formula.dep_plan,
9901                        )
9902                    }));
9903            }
9904            let admission_preflighted = self.graph_admission_enabled();
9905            if admission_preflighted {
9906                let mut preview = Vec::new();
9907                for (sheet_name, formulas) in &prepared_by_sheet {
9908                    let sheet_id = self.graph.sheet_id(sheet_name).ok_or_else(|| {
9909                        ExcelError::new(ExcelErrorKind::Ref)
9910                            .with_message(format!("unknown ingest sheet: {sheet_name}"))
9911                    })?;
9912                    preview.extend(
9913                        formulas
9914                            .iter()
9915                            .map(|(row, col, _, plan)| (sheet_id, *row, *col, plan.clone())),
9916                    );
9917                }
9918                let admission = self.graph.preview_formula_mutations(&preview)?;
9919                self.preflight_graph_admission(admission)?;
9920            }
9921
9922            let mut builder = self.begin_bulk_ingest();
9923            if admission_preflighted {
9924                builder.mark_admission_preflighted();
9925            }
9926            for (sheet_name, formulas) in prepared_by_sheet {
9927                if formulas.is_empty() {
9928                    continue;
9929                }
9930                let sheet_id = builder.add_sheet(&sheet_name);
9931                builder.add_formula_plans(sheet_id, formulas);
9932            }
9933            let summary = builder.finish()?;
9934            report.graph_formula_cells_materialized = summary.formulas as u64;
9935            report.graph_vertices_created = summary.vertices as u64;
9936            report.graph_edges_created = summary.edges as u64;
9937        }
9938
9939        if publish_report {
9940            self.record_formula_ingest_report(report.clone());
9941        }
9942        Ok(report)
9943    }
9944
9945    fn dedup_formula_parse_diagnostics_since(&mut self, start: usize) {
9946        let mut unique = Vec::new();
9947        for diagnostic in self.formula_parse_diagnostics.drain(start..) {
9948            let duplicate = unique.iter().any(|prior: &FormulaParseDiagnostic| {
9949                prior.sheet == diagnostic.sheet
9950                    && prior.row == diagnostic.row
9951                    && prior.col == diagnostic.col
9952                    && prior.formula == diagnostic.formula
9953                    && prior.policy == diagnostic.policy
9954            });
9955            if !duplicate {
9956                unique.push(diagnostic);
9957            }
9958        }
9959        self.formula_parse_diagnostics.extend(unique);
9960    }
9961
9962    pub fn handle_formula_parse_error(
9963        &mut self,
9964        sheet: &str,
9965        row: u32,
9966        col: u32,
9967        formula: &str,
9968        message: String,
9969    ) -> Result<Option<ASTNode>, ExcelError> {
9970        let policy = self.config.formula_parse_policy;
9971
9972        if policy == FormulaParsePolicy::Strict {
9973            let col_a1 = col_letters_from_1based(col).unwrap_or_else(|_| "?".to_string());
9974            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(format!(
9975                "Formula parse error at {sheet}!{col_a1}{row}: {message}"
9976            )));
9977        }
9978
9979        self.formula_parse_diagnostics.push(FormulaParseDiagnostic {
9980            sheet: sheet.to_string(),
9981            row,
9982            col,
9983            formula: formula.to_string(),
9984            message: message.clone(),
9985            policy,
9986        });
9987
9988        match policy {
9989            FormulaParsePolicy::Strict => unreachable!(),
9990            FormulaParsePolicy::KeepCachedValue => Ok(None),
9991            FormulaParsePolicy::AsText => Ok(Some(ASTNode::new(
9992                ASTNodeType::Literal(LiteralValue::Text(formula.to_string())),
9993                None,
9994            ))),
9995            FormulaParsePolicy::CoerceToError => {
9996                let err = ExcelError::new(ExcelErrorKind::Error)
9997                    .with_message(format!("Malformed formula: {message}"));
9998                Ok(Some(ASTNode::new(
9999                    ASTNodeType::Literal(LiteralValue::Error(err)),
10000                    None,
10001                )))
10002            }
10003        }
10004    }
10005
10006    #[cfg(test)]
10007    fn target_preparation_fault(
10008        &mut self,
10009        seam: crate::engine::target_preparation::TargetPreparationFault,
10010    ) -> Result<(), ExcelError> {
10011        if self.target_preparation_fault_for_test == Some(seam) {
10012            self.target_preparation_fault_for_test = None;
10013            Err(ExcelError::new(ExcelErrorKind::Value)
10014                .with_message(format!("injected target preparation fault: {seam:?}")))
10015        } else {
10016            Ok(())
10017        }
10018    }
10019
10020    fn preparation_stale(
10021        reason: formualizer_common::PreparationStaleReason,
10022        message: impl Into<String>,
10023    ) -> ExcelError {
10024        ExcelError::new(ExcelErrorKind::Value)
10025            .with_message(message)
10026            .with_extra(formualizer_common::ExcelErrorExtra::PreparationStale { reason })
10027    }
10028
10029    fn preparation_revision_stale_reason(
10030        assumptions: &crate::engine::PreparationRevision,
10031        current: &crate::engine::PreparationRevision,
10032        planning_requests: &BTreeSet<(String, String, usize)>,
10033        staged_leases_match: bool,
10034    ) -> Option<formualizer_common::PreparationStaleReason> {
10035        if assumptions.graph != current.graph {
10036            Some(formualizer_common::PreparationStaleReason::Graph)
10037        } else if assumptions.authority != current.authority
10038            || assumptions.authority_indexes != current.authority_indexes
10039            || assumptions.authority_indexed_plane != current.authority_indexed_plane
10040        {
10041            Some(formualizer_common::PreparationStaleReason::Authority)
10042        } else if assumptions.staged != current.staged || !staged_leases_match {
10043            Some(formualizer_common::PreparationStaleReason::Staged)
10044        } else if assumptions.symbols != current.symbols {
10045            Some(formualizer_common::PreparationStaleReason::Symbols)
10046        } else if assumptions.provider != current.provider {
10047            Some(formualizer_common::PreparationStaleReason::Provider)
10048        } else if assumptions.semantic != current.semantic
10049            && crate::function_registry::semantic_changes_affect_requests_since(
10050                assumptions.semantic,
10051                planning_requests.iter().cloned(),
10052            )
10053        {
10054            Some(formualizer_common::PreparationStaleReason::Semantic)
10055        } else {
10056            None
10057        }
10058    }
10059
10060    fn preparation_revisions(&self) -> crate::engine::PreparationRevision {
10061        let (authority, authority_indexes, authority_indexed_plane) =
10062            self.graph.authority_revisions();
10063        crate::engine::PreparationRevision {
10064            graph: self.graph.topology_revision(),
10065            authority,
10066            authority_indexes,
10067            authority_indexed_plane,
10068            staged: self.staged_formula_index.revision(),
10069            symbols: self.graph.symbol_revision(),
10070            semantic: crate::function_registry::semantic_epoch(),
10071            provider: self.resolver.planning_semantic_revision(),
10072        }
10073    }
10074
10075    fn planning_revision_snapshot(&self) -> PlanningRevisionSnapshot {
10076        let registry_guard = crate::function_registry::semantic_epoch_read_guard();
10077        let provider = self.resolver.planning_semantic_revision();
10078        let semantic = registry_guard.epoch();
10079        let (authority, authority_indexes, authority_indexed_plane) =
10080            self.graph.authority_revisions();
10081        let mut span_refs = self.graph.formula_authority().active_span_refs();
10082        span_refs.sort_unstable_by_key(|span_ref| {
10083            (span_ref.id.0, span_ref.generation, span_ref.version)
10084        });
10085        PlanningRevisionSnapshot {
10086            engine_topology_epoch: self.topology_epoch,
10087            graph_topology_revision: self.graph.topology_revision(),
10088            authority,
10089            authority_indexes,
10090            authority_indexed_plane,
10091            staged: self.staged_formula_index.revision(),
10092            symbols: self.graph.symbol_revision(),
10093            semantic,
10094            provider,
10095            formula_plane_mode: self.config.formula_plane_mode,
10096            deterministic_mode: self.config.deterministic_mode.clone(),
10097            budgets: self.evaluation_resource_budgets.clone(),
10098            span_refs,
10099        }
10100    }
10101
10102    fn recalc_plan_key(&self) -> RecalcPlanKey {
10103        RecalcPlanKey {
10104            engine_token: Arc::clone(&self.recalc_plan_token),
10105            revisions: self.planning_revision_snapshot(),
10106        }
10107    }
10108
10109    fn plan_stale(reason: formualizer_common::PlanStaleReason) -> ExcelError {
10110        ExcelError::new(ExcelErrorKind::Value)
10111            .with_message(format!("recalculation plan is stale: {}", reason.as_str()))
10112            .with_extra(formualizer_common::ExcelErrorExtra::PlanStale { reason })
10113    }
10114
10115    fn validate_recalc_plan_key(&self, key: &RecalcPlanKey) -> Result<(), ExcelError> {
10116        use formualizer_common::PlanStaleReason;
10117
10118        if !Arc::ptr_eq(&key.engine_token, &self.recalc_plan_token) {
10119            return Err(Self::plan_stale(PlanStaleReason::Engine));
10120        }
10121
10122        let current = self.planning_revision_snapshot();
10123        let expected = &key.revisions;
10124        let stale = if expected.provider != current.provider {
10125            Some(PlanStaleReason::Provider)
10126        } else if expected.semantic != current.semantic {
10127            Some(PlanStaleReason::Semantic)
10128        } else if expected.budgets != current.budgets
10129            || expected.deterministic_mode != current.deterministic_mode
10130        {
10131            Some(PlanStaleReason::Budget)
10132        } else if expected.staged != current.staged {
10133            Some(PlanStaleReason::Staged)
10134        } else if expected.symbols != current.symbols {
10135            Some(PlanStaleReason::Symbols)
10136        } else if expected.formula_plane_mode != current.formula_plane_mode
10137            || expected.authority != current.authority
10138            || expected.authority_indexes != current.authority_indexes
10139            || expected.authority_indexed_plane != current.authority_indexed_plane
10140        {
10141            Some(PlanStaleReason::Authority)
10142        } else if expected.span_refs != current.span_refs {
10143            Some(PlanStaleReason::SpanGeneration)
10144        } else if expected.graph_topology_revision != current.graph_topology_revision
10145            || expected.engine_topology_epoch != current.engine_topology_epoch
10146        {
10147            Some(PlanStaleReason::Graph)
10148        } else {
10149            None
10150        };
10151        stale.map_or(Ok(()), |reason| Err(Self::plan_stale(reason)))
10152    }
10153
10154    fn target_preparation_checkpoint(
10155        &mut self,
10156        deadline: Option<std::time::Instant>,
10157        work: u64,
10158    ) -> Result<(), ExcelError> {
10159        if self
10160            .active_cancel_flag
10161            .as_ref()
10162            .is_some_and(|cancel| cancel.is_cancelled())
10163        {
10164            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
10165                .with_message("target graph preparation cancelled"));
10166        }
10167        if deadline.is_some_and(|deadline| std::time::Instant::now() >= deadline) {
10168            return Err(crate::engine::ResourceLedgerError::Exhausted(
10169                formualizer_common::ResourceExhaustionDetail {
10170                    reason: formualizer_common::ResourceExhaustionReason::Deadline,
10171                    limit: 0,
10172                    observed: 1,
10173                    request_id: self
10174                        .active_evaluation_resource_request
10175                        .as_ref()
10176                        .map(|stats| stats.request_id),
10177                },
10178            )
10179            .into_excel_error());
10180        }
10181        self.charge_bounded_work(work)
10182    }
10183
10184    fn opaque_reason_in_ast(
10185        &self,
10186        ast: &ASTNode,
10187        provider: &dyn crate::traits::FunctionProvider,
10188    ) -> Option<crate::engine::OpaqueReason> {
10189        match &ast.node_type {
10190            ASTNodeType::Function { name, args } => {
10191                let canonical = name.rsplit('.').next().unwrap_or(name).to_ascii_uppercase();
10192                if canonical == "INDIRECT" {
10193                    return Some(crate::engine::OpaqueReason::RuntimeTextReference);
10194                }
10195                let Some(function) = provider.get_function_for_planning("", name) else {
10196                    return Some(crate::engine::OpaqueReason::UnknownFunction);
10197                };
10198                let caps = function.caps();
10199                if caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
10200                    || caps.contains(FnCaps::RETURNS_REFERENCE)
10201                {
10202                    return Some(crate::engine::OpaqueReason::DynamicReference);
10203                }
10204                args.iter()
10205                    .find_map(|arg| self.opaque_reason_in_ast(arg, provider))
10206            }
10207            ASTNodeType::Call { .. } => Some(crate::engine::OpaqueReason::UnknownCustomFunction),
10208            ASTNodeType::UnaryOp { expr, .. } => self.opaque_reason_in_ast(expr, provider),
10209            ASTNodeType::BinaryOp { left, right, .. } => self
10210                .opaque_reason_in_ast(left, provider)
10211                .or_else(|| self.opaque_reason_in_ast(right, provider)),
10212            ASTNodeType::Array(rows) => rows
10213                .iter()
10214                .flat_map(|row| row.iter())
10215                .find_map(|item| self.opaque_reason_in_ast(item, provider)),
10216            ASTNodeType::Reference {
10217                reference:
10218                    ReferenceType::Cell {
10219                        sheet: Some(sheet), ..
10220                    }
10221                    | ReferenceType::Range {
10222                        sheet: Some(sheet), ..
10223                    },
10224                ..
10225            } if self.graph.sheet_id(sheet).is_none() => {
10226                Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding)
10227            }
10228            ASTNodeType::Reference {
10229                reference:
10230                    ReferenceType::External(_)
10231                    | ReferenceType::Cell3D { .. }
10232                    | ReferenceType::Range3D { .. },
10233                ..
10234            } => Some(crate::engine::OpaqueReason::UnresolvedCrossSheetBinding),
10235            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => None,
10236        }
10237    }
10238
10239    fn target_planning_snapshot(
10240        &mut self,
10241        ast: &ASTNode,
10242        planning_requests: &mut BTreeSet<(String, String, usize)>,
10243    ) -> Result<crate::function_registry::RegistryPlanningSnapshot, ExcelError> {
10244        #[cfg(test)]
10245        if std::mem::take(&mut self.inject_target_semantic_stale_once_for_test) {
10246            return Err(Self::preparation_stale(
10247                formualizer_common::PreparationStaleReason::Semantic,
10248                "injected target semantic preparation movement",
10249            ));
10250        }
10251        #[cfg(test)]
10252        if let Some(hook) = self.before_target_planning_snapshot_hook.take() {
10253            hook();
10254        }
10255        let mut requests = Vec::new();
10256        Self::collect_planning_function_requests(ast, &mut requests);
10257        requests.sort();
10258        requests.dedup();
10259        planning_requests.extend(requests.iter().cloned());
10260        crate::function_registry::RegistryPlanningSnapshot::capture_for_requests(
10261            &self.resolver,
10262            requests,
10263        )
10264        .map_err(|error| {
10265            ExcelError::new(ExcelErrorKind::Value)
10266                .with_message(format!("target planning snapshot unavailable: {error:?}"))
10267        })
10268    }
10269
10270    fn target_planning_snapshot_stale_reason(
10271        snapshot: &crate::function_registry::RegistryPlanningSnapshot,
10272        assumptions: &crate::engine::PreparationRevision,
10273    ) -> Option<formualizer_common::PreparationStaleReason> {
10274        if snapshot
10275            .provider_revision()
10276            .is_some_and(|revision| Some(revision) != assumptions.provider)
10277        {
10278            Some(formualizer_common::PreparationStaleReason::Provider)
10279        } else if snapshot.epoch() != assumptions.semantic
10280            && snapshot.semantic_changes_affect_requests_since(assumptions.semantic)
10281        {
10282            Some(formualizer_common::PreparationStaleReason::Semantic)
10283        } else {
10284            None
10285        }
10286    }
10287
10288    fn widen_target_preparation(
10289        policy: crate::engine::OpaquePreparePolicy,
10290        scope: &mut crate::engine::PrepareScope,
10291        reasons: &mut Vec<crate::engine::OpaqueReason>,
10292        reason: crate::engine::OpaqueReason,
10293    ) -> Result<bool, ExcelError> {
10294        if policy == crate::engine::OpaquePreparePolicy::Error {
10295            return Err(ExcelError::new(ExcelErrorKind::NImpl)
10296                .with_message(format!("opaque target preparation semantics: {reason:?}")));
10297        }
10298        if !reasons.contains(&reason) {
10299            reasons.push(reason);
10300        }
10301        if !matches!(scope, crate::engine::PrepareScope::Workbook) {
10302            *scope = crate::engine::PrepareScope::Workbook;
10303            Ok(true)
10304        } else {
10305            Ok(false)
10306        }
10307    }
10308
10309    fn widen_target_preparation_to_sheet(
10310        policy: crate::engine::OpaquePreparePolicy,
10311        scope: &mut crate::engine::PrepareScope,
10312        reasons: &mut Vec<crate::engine::OpaqueReason>,
10313        reason: crate::engine::OpaqueReason,
10314        sheet: &str,
10315    ) -> Result<bool, ExcelError> {
10316        if policy == crate::engine::OpaquePreparePolicy::Error {
10317            return Err(ExcelError::new(ExcelErrorKind::NImpl)
10318                .with_message(format!("opaque target preparation semantics: {reason:?}")));
10319        }
10320        if !reasons.contains(&reason) {
10321            reasons.push(reason);
10322        }
10323        match scope {
10324            crate::engine::PrepareScope::Exact => {
10325                *scope = crate::engine::PrepareScope::Sheets(vec![sheet.to_string()]);
10326                Ok(true)
10327            }
10328            crate::engine::PrepareScope::Sheets(sheets) => {
10329                if sheets.iter().any(|candidate| candidate == sheet) {
10330                    Ok(false)
10331                } else {
10332                    sheets.push(sheet.to_string());
10333                    sheets.sort();
10334                    Ok(true)
10335                }
10336            }
10337            crate::engine::PrepareScope::Workbook => Ok(false),
10338        }
10339    }
10340
10341    fn ast_has_proven_sheet_local_dynamic(
10342        ast: &ASTNode,
10343        provider: &dyn crate::traits::FunctionProvider,
10344    ) -> bool {
10345        fn classify(ast: &ASTNode, provider: &dyn crate::traits::FunctionProvider) -> (bool, bool) {
10346            match &ast.node_type {
10347                ASTNodeType::Function { name, args } => {
10348                    let Some(function) = provider.get_function_for_planning("", name) else {
10349                        return (false, false);
10350                    };
10351                    let caps = function.caps();
10352                    let dynamic = caps.contains(FnCaps::DYNAMIC_DEPENDENCY)
10353                        || caps.contains(FnCaps::RETURNS_REFERENCE);
10354                    let canonical = name.rsplit('.').next().unwrap_or(name);
10355                    if dynamic
10356                        && !canonical.eq_ignore_ascii_case("OFFSET")
10357                        && !canonical.eq_ignore_ascii_case("INDEX")
10358                    {
10359                        return (false, true);
10360                    }
10361                    let mut has_dynamic = dynamic;
10362                    for arg in args {
10363                        let (safe, child_dynamic) = classify(arg, provider);
10364                        if !safe {
10365                            return (false, has_dynamic || child_dynamic);
10366                        }
10367                        has_dynamic |= child_dynamic;
10368                    }
10369                    (true, has_dynamic)
10370                }
10371                ASTNodeType::UnaryOp { expr, .. } => classify(expr, provider),
10372                ASTNodeType::BinaryOp { left, right, .. } => {
10373                    let (left_safe, left_dynamic) = classify(left, provider);
10374                    let (right_safe, right_dynamic) = classify(right, provider);
10375                    (left_safe && right_safe, left_dynamic || right_dynamic)
10376                }
10377                ASTNodeType::Array(rows) => {
10378                    let mut has_dynamic = false;
10379                    for item in rows.iter().flatten() {
10380                        let (safe, child_dynamic) = classify(item, provider);
10381                        if !safe {
10382                            return (false, has_dynamic || child_dynamic);
10383                        }
10384                        has_dynamic |= child_dynamic;
10385                    }
10386                    (true, has_dynamic)
10387                }
10388                ASTNodeType::Reference {
10389                    reference:
10390                        ReferenceType::Cell { sheet: None, .. }
10391                        | ReferenceType::Range { sheet: None, .. },
10392                    ..
10393                }
10394                | ASTNodeType::Literal(_)
10395                | ASTNodeType::Omitted => (true, false),
10396                ASTNodeType::Call { .. } | ASTNodeType::Reference { .. } => (false, false),
10397            }
10398        }
10399
10400        let (safe, dynamic) = classify(ast, provider);
10401        safe && dynamic
10402    }
10403
10404    fn table_selection_region(
10405        &self,
10406        entry: &crate::engine::graph::TableEntry,
10407        selection: &crate::engine::TableSelection,
10408    ) -> Result<PreparationRegion, ExcelError> {
10409        let mut start_row = entry.range.start.coord.row() + 1;
10410        let mut end_row = entry.range.end.coord.row() + 1;
10411        let mut start_col = entry.range.start.coord.col() + 1;
10412        let mut end_col = entry.range.end.coord.col() + 1;
10413        match selection {
10414            crate::engine::TableSelection::Whole => {}
10415            crate::engine::TableSelection::Headers => {
10416                if !entry.header_row {
10417                    return Err(ExcelError::new(ExcelErrorKind::Value)
10418                        .with_message(format!("table {} has no header row", entry.name)));
10419                }
10420                end_row = start_row;
10421            }
10422            crate::engine::TableSelection::Data => {
10423                if entry.header_row {
10424                    start_row = start_row.saturating_add(1);
10425                }
10426                if entry.totals_row {
10427                    end_row = end_row.saturating_sub(1);
10428                }
10429            }
10430            crate::engine::TableSelection::Totals => {
10431                if !entry.totals_row {
10432                    return Err(ExcelError::new(ExcelErrorKind::Value)
10433                        .with_message(format!("table {} has no totals row", entry.name)));
10434                }
10435                start_row = end_row;
10436            }
10437            crate::engine::TableSelection::Column(column) => {
10438                let index = entry.col_index(column).ok_or_else(|| {
10439                    ExcelError::new(ExcelErrorKind::Name)
10440                        .with_message(format!("unknown table column: {column}"))
10441                })?;
10442                start_col = start_col.saturating_add(index as u32);
10443                end_col = start_col;
10444            }
10445            crate::engine::TableSelection::Columns { start, end } => {
10446                let first = entry.col_index(start).ok_or_else(|| {
10447                    ExcelError::new(ExcelErrorKind::Name)
10448                        .with_message(format!("unknown table column: {start}"))
10449                })?;
10450                let last = entry.col_index(end).ok_or_else(|| {
10451                    ExcelError::new(ExcelErrorKind::Name)
10452                        .with_message(format!("unknown table column: {end}"))
10453                })?;
10454                if first > last {
10455                    return Err(ExcelError::new(ExcelErrorKind::Value)
10456                        .with_message("table column selection is reversed"));
10457                }
10458                start_col = start_col.saturating_add(first as u32);
10459                end_col = entry
10460                    .range
10461                    .start
10462                    .coord
10463                    .col()
10464                    .saturating_add(last as u32)
10465                    .saturating_add(1);
10466            }
10467        }
10468        if start_row > end_row && matches!(selection, crate::engine::TableSelection::Data) {
10469            start_row = entry.range.start.coord.row() + 1;
10470            end_row = start_row;
10471        }
10472        if start_row > end_row || start_col > end_col {
10473            return Err(
10474                ExcelError::new(ExcelErrorKind::Value).with_message("table selection is empty")
10475            );
10476        }
10477        Ok(PreparationRegion {
10478            sheet: self.graph.sheet_name(entry.sheet_id()).to_string(),
10479            sheet_id: entry.sheet_id(),
10480            start_row,
10481            start_col,
10482            end_row,
10483            end_col,
10484        })
10485    }
10486
10487    #[cfg(test)]
10488    pub(crate) fn reset_target_root_dedup_probes_for_test() {
10489        TARGET_ROOT_DEDUP_PROBES.with(|probes| probes.set(0));
10490    }
10491
10492    #[cfg(test)]
10493    pub(crate) fn target_root_dedup_probes_for_test() -> usize {
10494        TARGET_ROOT_DEDUP_PROBES.with(std::cell::Cell::get)
10495    }
10496
10497    pub(crate) fn resolve_target_producers(
10498        &mut self,
10499        targets: &[crate::engine::EvaluationTarget],
10500    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
10501        use crate::engine::target_preparation::TargetProducer;
10502
10503        let request_id = self
10504            .active_evaluation_resource_request
10505            .as_ref()
10506            .map(|request| request.request_id);
10507        let mut roots = OrderedTargetProducers::with_capacity(targets.len())
10508            .map_err(|_| target_root_allocation_error(targets.len(), request_id))?;
10509        let active_span_refs = self
10510            .graph
10511            .formula_authority()
10512            .active_span_refs()
10513            .into_iter()
10514            .map(|span_ref| (span_ref.id, span_ref))
10515            .collect::<BTreeMap<_, _>>();
10516        let resolve_region = |engine: &mut Self,
10517                              region: Region,
10518                              value_only: Option<CellRef>,
10519                              roots: &mut OrderedTargetProducers|
10520         -> Result<(), ExcelError> {
10521            let before = roots.len();
10522            let authority = engine.graph.formula_authority();
10523            for matched in authority.producer_results.query(region).matches {
10524                if let FormulaProducerId::Span(span_id) = matched.value.producer
10525                    && let Some(span_ref) = active_span_refs.get(&span_id).copied()
10526                    && let Some(demanded) = matched.value.result_region.intersection(region)
10527                {
10528                    roots
10529                        .push(TargetProducer::Span { span_ref, demanded })
10530                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10531                }
10532            }
10533            for anchor in engine.graph.spill_anchors_in_region(
10534                region.sheet_id(),
10535                region.axis_ranges().0.query_bounds().0,
10536                region.axis_ranges().1.query_bounds().0,
10537                region.axis_ranges().0.query_bounds().1,
10538                region.axis_ranges().1.query_bounds().1,
10539            ) {
10540                roots
10541                    .push(TargetProducer::Legacy(anchor))
10542                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10543            }
10544            for vertex in engine.graph.vertices_in_region(
10545                region.sheet_id(),
10546                region.axis_ranges().0.query_bounds().0,
10547                region.axis_ranges().0.query_bounds().1,
10548                region.axis_ranges().1.query_bounds().0,
10549                region.axis_ranges().1.query_bounds().1,
10550            ) {
10551                let vertex = engine
10552                    .graph
10553                    .get_cell_ref(vertex)
10554                    .and_then(|cell| engine.graph.spill_registry_anchor_for_cell(cell))
10555                    .unwrap_or(vertex);
10556                // `vertices_in_region` is a sheet-index query and a sheet index holds only
10557                // grid-addressed vertices, so a region can never yield a symbol: names,
10558                // tables and external sources have no position for a region to cover.
10559                // Symbol roots come from the by-name lookups below instead.
10560                if matches!(
10561                    engine.graph.get_vertex_kind(vertex),
10562                    VertexKind::FormulaScalar | VertexKind::FormulaArray
10563                ) {
10564                    roots
10565                        .push(TargetProducer::Legacy(vertex))
10566                        .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10567                }
10568            }
10569            if roots.len() == before
10570                && let Some(cell) = value_only
10571            {
10572                roots
10573                    .push(TargetProducer::ValueOnly(cell))
10574                    .map_err(|_| target_root_allocation_error(roots.len() + 1, request_id))?;
10575            }
10576            Ok(())
10577        };
10578
10579        for target in targets {
10580            match target {
10581                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10582                    if *row == 0 || *col == 0 {
10583                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10584                            .with_message("target cell coordinates are one-based"));
10585                    }
10586                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10587                        ExcelError::new(ExcelErrorKind::Ref)
10588                            .with_message(format!("target sheet not found: {sheet}"))
10589                    })?;
10590                    let cell = CellRef::new(sheet_id, Coord::from_excel(*row, *col, true, true));
10591                    resolve_region(
10592                        self,
10593                        Region::point(sheet_id, *row - 1, *col - 1),
10594                        Some(cell),
10595                        &mut roots,
10596                    )?;
10597                }
10598                crate::engine::EvaluationTarget::Range(range) => {
10599                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10600                        ExcelError::new(ExcelErrorKind::Ref)
10601                            .with_message(format!("target sheet not found: {}", range.sheet))
10602                    })?;
10603                    resolve_region(
10604                        self,
10605                        Region::rect(
10606                            sheet_id,
10607                            range.start_row - 1,
10608                            range.end_row - 1,
10609                            range.start_col - 1,
10610                            range.end_col - 1,
10611                        ),
10612                        None,
10613                        &mut roots,
10614                    )?;
10615                }
10616                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10617                    let scope = self.name_query_scope(scope_sheet.as_deref())?;
10618                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, scope) {
10619                        roots
10620                            .push(TargetProducer::Symbol(entry.vertex))
10621                            .map_err(|_| {
10622                                target_root_allocation_error(roots.len() + 1, request_id)
10623                            })?;
10624                    }
10625                }
10626                crate::engine::EvaluationTarget::Table { name, .. } => {
10627                    if let Some(entry) = self.graph.resolve_table_entry(name) {
10628                        roots
10629                            .push(TargetProducer::Symbol(entry.vertex))
10630                            .map_err(|_| {
10631                                target_root_allocation_error(roots.len() + 1, request_id)
10632                            })?;
10633                    }
10634                }
10635            }
10636        }
10637        Ok(roots.into_vec())
10638    }
10639
10640    fn resolve_target_producers_from_existing_roots(
10641        &mut self,
10642        roots: &[crate::engine::target_preparation::TargetProducer],
10643    ) -> Result<Vec<crate::engine::target_preparation::TargetProducer>, ExcelError> {
10644        use crate::engine::target_preparation::TargetProducer;
10645        let request_id = self
10646            .active_evaluation_resource_request
10647            .as_ref()
10648            .map(|request| request.request_id);
10649        let mut refreshed = OrderedTargetProducers::with_capacity(roots.len())
10650            .map_err(|_| target_root_allocation_error(roots.len(), request_id))?;
10651        for root in roots {
10652            match *root {
10653                TargetProducer::Span { demanded, .. } => {
10654                    let (rows, cols) = demanded.axis_ranges();
10655                    for vertex in self.graph.vertices_in_region(
10656                        demanded.sheet_id(),
10657                        rows.query_bounds().0,
10658                        rows.query_bounds().1,
10659                        cols.query_bounds().0,
10660                        cols.query_bounds().1,
10661                    ) {
10662                        if matches!(
10663                            self.graph.get_vertex_kind(vertex),
10664                            VertexKind::FormulaScalar | VertexKind::FormulaArray
10665                        ) {
10666                            refreshed
10667                                .push(TargetProducer::Legacy(vertex))
10668                                .map_err(|_| {
10669                                    target_root_allocation_error(refreshed.len() + 1, request_id)
10670                                })?;
10671                        }
10672                    }
10673                }
10674                other => {
10675                    refreshed.push(other).map_err(|_| {
10676                        target_root_allocation_error(refreshed.len() + 1, request_id)
10677                    })?;
10678                }
10679            }
10680        }
10681        Ok(refreshed.into_vec())
10682    }
10683
10684    /// Transactionally prepare the complete ordinary staged demand closure for typed targets.
10685    ///
10686    /// This method prepares graph topology only. It does not evaluate target values.
10687    pub fn prepare_graph_for_targets(
10688        &mut self,
10689        targets: &[crate::engine::EvaluationTarget],
10690        options: crate::engine::TargetEvalOptions<'_>,
10691    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10692        let previous_budgets = self.evaluation_resource_budgets.clone();
10693        let diagnostics_len = self.formula_parse_diagnostics.len();
10694        let previous_report = self.last_formula_ingest_report.clone();
10695        if let Some(budgets) = options.budgets {
10696            self.evaluation_resource_budgets = budgets.clone();
10697        }
10698        let previous_graph_budget_override = self
10699            .graph
10700            .set_admission_budget_override(Some(self.evaluation_resource_budgets.clone()));
10701        // Hoist the call's cancellation onto the engine for its duration, so the
10702        // preparation checkpoints observe it. This is a standalone entry point, so
10703        // the previous value is restored rather than cleared.
10704        let previous_cancel = self.active_cancel_flag.take();
10705        self.active_cancel_flag = options.cancel.clone();
10706        let result = self.observe_evaluation_resource_request(
10707            EvaluationRequestKind::TargetPreparation,
10708            |engine| engine.prepare_graph_for_targets_unobserved(targets, &options),
10709        );
10710        self.active_cancel_flag = previous_cancel;
10711        self.graph
10712            .set_admission_budget_override(previous_graph_budget_override);
10713        self.evaluation_resource_budgets = previous_budgets;
10714        if result.is_err() {
10715            self.formula_parse_diagnostics.truncate(diagnostics_len);
10716            self.last_formula_ingest_report = previous_report;
10717        }
10718        result
10719    }
10720
10721    fn prepare_graph_for_targets_unobserved(
10722        &mut self,
10723        targets: &[crate::engine::EvaluationTarget],
10724        options: &crate::engine::TargetEvalOptions<'_>,
10725    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10726        let scratch_checkpoint = self
10727            .active_resource_ledger
10728            .as_ref()
10729            .map_or(0, crate::engine::ResourceLedger::scratch_checkpoint);
10730        let result = self.prepare_graph_for_targets_transaction(targets, options);
10731        let release = self
10732            .active_resource_ledger
10733            .as_mut()
10734            .map_or(Ok(()), |ledger| {
10735                ledger.release_scratch_to(scratch_checkpoint)
10736            });
10737        match (result, release) {
10738            (result, Ok(())) => result,
10739            (Ok(_), Err(error)) | (Err(_), Err(error)) => Err(error.into_excel_error()),
10740        }
10741    }
10742
10743    fn prepare_graph_for_targets_transaction(
10744        &mut self,
10745        targets: &[crate::engine::EvaluationTarget],
10746        options: &crate::engine::TargetEvalOptions<'_>,
10747    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
10748        use crate::engine::{
10749            OpaqueReason, PreparationOutcome, PrepareScope, PreparedTargetGraphReport,
10750            TableSelection,
10751        };
10752
10753        self.target_preparation_checkpoint(options.deadline, 0)?;
10754        self.observe_function_semantic_epoch()?;
10755        let assumptions = self.preparation_revisions();
10756        let ledger_at_start = self
10757            .active_resource_ledger
10758            .as_ref()
10759            .map(|ledger| ledger.snapshot());
10760        let diagnostics_len = self.formula_parse_diagnostics.len();
10761        let report_len = self.last_formula_ingest_report.clone();
10762        let request_id = options.request_id.or_else(|| {
10763            self.active_evaluation_resource_request
10764                .as_ref()
10765                .map(|stats| stats.request_id)
10766        });
10767
10768        let mut scope = PrepareScope::Exact;
10769        let mut reasons = Vec::new();
10770        let mut regions = VecDeque::new();
10771        let mut normalized = Vec::with_capacity(targets.len());
10772        let mut symbol_vertices = VecDeque::new();
10773        for target in targets {
10774            self.target_preparation_checkpoint(options.deadline, 1)?;
10775            match target {
10776                crate::engine::EvaluationTarget::Cell { sheet, row, col } => {
10777                    if *row == 0 || *col == 0 {
10778                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10779                            .with_message("target cell coordinates are one-based"));
10780                    }
10781                    let sheet_id = self.graph.sheet_id(sheet).ok_or_else(|| {
10782                        ExcelError::new(ExcelErrorKind::Ref)
10783                            .with_message(format!("target sheet not found: {sheet}"))
10784                    })?;
10785                    regions.push_back(PreparationRegion {
10786                        sheet: sheet.clone(),
10787                        sheet_id,
10788                        start_row: *row,
10789                        start_col: *col,
10790                        end_row: *row,
10791                        end_col: *col,
10792                    });
10793                    normalized.push(target.clone());
10794                }
10795                crate::engine::EvaluationTarget::Range(range) => {
10796                    if range.start_row == 0
10797                        || range.start_col == 0
10798                        || range.end_row < range.start_row
10799                        || range.end_col < range.start_col
10800                    {
10801                        return Err(ExcelError::new(ExcelErrorKind::Ref)
10802                            .with_message("invalid target range"));
10803                    }
10804                    let sheet_id = self.graph.sheet_id(&range.sheet).ok_or_else(|| {
10805                        ExcelError::new(ExcelErrorKind::Ref)
10806                            .with_message(format!("target sheet not found: {}", range.sheet))
10807                    })?;
10808                    regions.push_back(PreparationRegion {
10809                        sheet: range.sheet.clone(),
10810                        sheet_id,
10811                        start_row: range.start_row,
10812                        start_col: range.start_col,
10813                        end_row: range.end_row,
10814                        end_col: range.end_col,
10815                    });
10816                    normalized.push(target.clone());
10817                }
10818                crate::engine::EvaluationTarget::Name { name, scope_sheet } => {
10819                    let name_scope = self.name_query_scope(scope_sheet.as_deref())?;
10820                    if let Some(entry) = self.graph.resolve_name_entry_in_scope(name, name_scope) {
10821                        symbol_vertices.push_back(entry.vertex);
10822                    } else {
10823                        Self::widen_target_preparation(
10824                            options.opaque_policy,
10825                            &mut scope,
10826                            &mut reasons,
10827                            OpaqueReason::UnresolvedName,
10828                        )?;
10829                    }
10830                    normalized.push(target.clone());
10831                }
10832                crate::engine::EvaluationTarget::Table { name, selection } => {
10833                    if let Some(entry) = self.graph.resolve_table_entry(name) {
10834                        let region = self.table_selection_region(entry, selection)?;
10835                        symbol_vertices.push_back(entry.vertex);
10836                        regions.push_back(region);
10837                    } else {
10838                        Self::widen_target_preparation(
10839                            options.opaque_policy,
10840                            &mut scope,
10841                            &mut reasons,
10842                            OpaqueReason::UnresolvedTable,
10843                        )?;
10844                    }
10845                    normalized.push(target.clone());
10846                }
10847            }
10848        }
10849
10850        let mut visited_regions = FxHashSet::default();
10851        let mut visited_vertices = FxHashSet::default();
10852        let mut selected = FxHashSet::default();
10853        let mut prepared = Vec::new();
10854        let mut pending_diagnostics = Vec::new();
10855        let mut planning_requests = BTreeSet::new();
10856        let mut selected_cells = Vec::new();
10857        let mut workbook_seeded = false;
10858        let mut sheet_scope_seeded = BTreeSet::new();
10859        let mut indexed_query_sheets = FxHashSet::default();
10860        let mut discovery_scratch_reserved = 0u64;
10861        let mut package_encountered = false;
10862        let mut selected_package_sheets = FxHashSet::default();
10863        let mut prepared_packages = Vec::new();
10864        let authoritative_with_ordinary = self.config.formula_plane_mode
10865            == FormulaPlaneMode::AuthoritativeExperimental
10866            && self.staged_formula_index.ordinary_count() != 0;
10867        let has_unknown_package_sheet = self
10868            .staged_formula_index
10869            .package_sheets()
10870            .any(|sheet| self.graph.sheet_id(sheet).is_none());
10871
10872        loop {
10873            if let PrepareScope::Sheets(sheets) = &scope {
10874                for sheet in sheets.clone() {
10875                    if !sheet_scope_seeded.insert(sheet.clone()) {
10876                        continue;
10877                    }
10878                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10879                        package_encountered = true;
10880                        Self::widen_target_preparation(
10881                            options.opaque_policy,
10882                            &mut scope,
10883                            &mut reasons,
10884                            OpaqueReason::UnsupportedSourceSemantics,
10885                        )?;
10886                        continue;
10887                    };
10888                    for lease in self.staged_formula_index.leases_for_sheet(&sheet) {
10889                        self.target_preparation_checkpoint(options.deadline, 1)?;
10890                        regions.push_back(PreparationRegion {
10891                            sheet: sheet.clone(),
10892                            sheet_id,
10893                            start_row: lease.row,
10894                            start_col: lease.col,
10895                            end_row: lease.row,
10896                            end_col: lease.col,
10897                        });
10898                    }
10899                    if self
10900                        .staged_formula_index
10901                        .package_lease_for_sheet(&sheet)
10902                        .is_some()
10903                    {
10904                        regions.push_back(PreparationRegion {
10905                            sheet: sheet.clone(),
10906                            sheet_id,
10907                            start_row: 1,
10908                            start_col: 1,
10909                            end_row: self.workbook_load_limits.max_sheet_rows,
10910                            end_col: self.workbook_load_limits.max_sheet_cols,
10911                        });
10912                    }
10913                }
10914            }
10915            if matches!(scope, PrepareScope::Workbook) && !workbook_seeded {
10916                workbook_seeded = true;
10917                for (sheet, lease) in self.staged_formula_index.all_leases() {
10918                    self.target_preparation_checkpoint(options.deadline, 1)?;
10919                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10920                        package_encountered = true;
10921                        Self::widen_target_preparation(
10922                            options.opaque_policy,
10923                            &mut scope,
10924                            &mut reasons,
10925                            OpaqueReason::UnsupportedSourceSemantics,
10926                        )?;
10927                        continue;
10928                    };
10929                    regions.push_back(PreparationRegion {
10930                        sheet,
10931                        sheet_id,
10932                        start_row: lease.row,
10933                        start_col: lease.col,
10934                        end_row: lease.row,
10935                        end_col: lease.col,
10936                    });
10937                }
10938                let package_sheets = self
10939                    .staged_formula_index
10940                    .package_sheets()
10941                    .map(str::to_string)
10942                    .collect::<Vec<_>>();
10943                for sheet in package_sheets {
10944                    let Some(sheet_id) = self.graph.sheet_id(&sheet) else {
10945                        package_encountered = true;
10946                        Self::widen_target_preparation(
10947                            options.opaque_policy,
10948                            &mut scope,
10949                            &mut reasons,
10950                            OpaqueReason::UnsupportedSourceSemantics,
10951                        )?;
10952                        continue;
10953                    };
10954                    regions.push_back(PreparationRegion {
10955                        sheet,
10956                        sheet_id,
10957                        start_row: 1,
10958                        start_col: 1,
10959                        end_row: self.workbook_load_limits.max_sheet_rows,
10960                        end_col: self.workbook_load_limits.max_sheet_cols,
10961                    });
10962                }
10963            }
10964
10965            let Some(region) = regions.pop_front() else {
10966                if let Some(vertex) = symbol_vertices.pop_front() {
10967                    self.target_preparation_checkpoint(options.deadline, 1)?;
10968                    if !visited_vertices.insert(vertex) || !self.graph.vertex_exists(vertex) {
10969                        continue;
10970                    }
10971                    let vertex_is_dynamic = self.graph.is_dynamic(vertex);
10972                    if let Some(ast) = self.graph.get_formula(vertex) {
10973                        let snapshot =
10974                            self.target_planning_snapshot(&ast, &mut planning_requests)?;
10975                        if let Some(reason) =
10976                            Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
10977                        {
10978                            return Err(Self::preparation_stale(
10979                                reason,
10980                                "target planning snapshot became stale during discovery",
10981                            ));
10982                        }
10983                        let opaque = self.opaque_reason_in_ast(&ast, &snapshot);
10984                        if let Some(reason) =
10985                            opaque.or(vertex_is_dynamic.then_some(OpaqueReason::DynamicReference))
10986                        {
10987                            if reason == OpaqueReason::DynamicReference
10988                                && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
10989                            {
10990                                let sheet = self.graph.get_vertex_sheet_id(vertex);
10991                                let sheet = self.graph.sheet_name(sheet).to_string();
10992                                Self::widen_target_preparation_to_sheet(
10993                                    options.opaque_policy,
10994                                    &mut scope,
10995                                    &mut reasons,
10996                                    reason,
10997                                    &sheet,
10998                                )?;
10999                            } else {
11000                                Self::widen_target_preparation(
11001                                    options.opaque_policy,
11002                                    &mut scope,
11003                                    &mut reasons,
11004                                    reason,
11005                                )?;
11006                            }
11007                        }
11008                    } else if vertex_is_dynamic {
11009                        Self::widen_target_preparation(
11010                            options.opaque_policy,
11011                            &mut scope,
11012                            &mut reasons,
11013                            OpaqueReason::DynamicReference,
11014                        )?;
11015                    }
11016                    if let Some(anchor) = self
11017                        .graph
11018                        .get_cell_ref(vertex)
11019                        .and_then(|cell| self.graph.spill_registry_anchor_for_cell(cell))
11020                    {
11021                        symbol_vertices.push_back(anchor);
11022                    }
11023                    if let Some(cell) = self.graph.get_cell_ref(vertex) {
11024                        let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
11025                        regions.push_back(PreparationRegion {
11026                            sheet,
11027                            sheet_id: cell.sheet_id,
11028                            start_row: cell.coord.row() + 1,
11029                            start_col: cell.coord.col() + 1,
11030                            end_row: cell.coord.row() + 1,
11031                            end_col: cell.coord.col() + 1,
11032                        });
11033                    }
11034                    for dependency in self.graph.get_dependencies(vertex) {
11035                        self.target_preparation_checkpoint(options.deadline, 1)?;
11036                        symbol_vertices.push_back(dependency);
11037                    }
11038                    if let Some(range_dependencies) = self
11039                        .graph
11040                        .formula_range_dependencies(vertex)
11041                        .map(<[_]>::to_vec)
11042                    {
11043                        for range in range_dependencies {
11044                            self.target_preparation_checkpoint(options.deadline, 1)?;
11045                            // `Current` is the sheet the formula lives on.
11046                            let context_sheet = self.graph.get_vertex_sheet_id(vertex);
11047                            let Ok(sheet_id) =
11048                                self.resolve_sheet_locator(&range.sheet, context_sheet)
11049                            else {
11050                                if Self::widen_target_preparation(
11051                                    options.opaque_policy,
11052                                    &mut scope,
11053                                    &mut reasons,
11054                                    OpaqueReason::UnresolvedCrossSheetBinding,
11055                                )? {
11056                                    break;
11057                                }
11058                                continue;
11059                            };
11060                            let sheet = self.graph.sheet_name(sheet_id).to_string();
11061                            regions.push_back(PreparationRegion {
11062                                sheet,
11063                                sheet_id,
11064                                start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11065                                start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11066                                end_row: range
11067                                    .end_row
11068                                    .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11069                                        bound.index + 1
11070                                    }),
11071                                end_col: range
11072                                    .end_col
11073                                    .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11074                                        bound.index + 1
11075                                    }),
11076                            });
11077                        }
11078                    }
11079                    if let Some(name) = self.graph.named_range_by_vertex(vertex).cloned() {
11080                        match &name.definition {
11081                            NamedDefinition::Cell(cell) => regions.push_back(PreparationRegion {
11082                                sheet: self.graph.sheet_name(cell.sheet_id).to_string(),
11083                                sheet_id: cell.sheet_id,
11084                                start_row: cell.coord.row() + 1,
11085                                start_col: cell.coord.col() + 1,
11086                                end_row: cell.coord.row() + 1,
11087                                end_col: cell.coord.col() + 1,
11088                            }),
11089                            NamedDefinition::Range(range) => regions.push_back(PreparationRegion {
11090                                sheet: self.graph.sheet_name(range.start.sheet_id).to_string(),
11091                                sheet_id: range.start.sheet_id,
11092                                start_row: range.start.coord.row() + 1,
11093                                start_col: range.start.coord.col() + 1,
11094                                end_row: range.end.coord.row() + 1,
11095                                end_col: range.end.coord.col() + 1,
11096                            }),
11097                            NamedDefinition::Formula {
11098                                ast,
11099                                dependencies,
11100                                range_deps,
11101                            } => {
11102                                let snapshot =
11103                                    self.target_planning_snapshot(ast, &mut planning_requests)?;
11104                                if let Some(reason) = Self::target_planning_snapshot_stale_reason(
11105                                    &snapshot,
11106                                    &assumptions,
11107                                ) {
11108                                    return Err(Self::preparation_stale(
11109                                        reason,
11110                                        "target planning snapshot became stale during discovery",
11111                                    ));
11112                                }
11113                                if let Some(reason) = self.opaque_reason_in_ast(ast, &snapshot) {
11114                                    Self::widen_target_preparation(
11115                                        options.opaque_policy,
11116                                        &mut scope,
11117                                        &mut reasons,
11118                                        reason,
11119                                    )?;
11120                                }
11121                                for dependency in dependencies {
11122                                    self.target_preparation_checkpoint(options.deadline, 1)?;
11123                                    symbol_vertices.push_back(*dependency);
11124                                }
11125                                for range in range_deps {
11126                                    self.target_preparation_checkpoint(options.deadline, 1)?;
11127                                    // `Current` is the sheet this name's formula
11128                                    // was interpreted on, which is the sheet its
11129                                    // vertex is placed on -- the same derivation
11130                                    // the formula-vertex arm above uses. It is
11131                                    // never the workbook default sheet, and an
11132                                    // unresolvable `Name` widens instead of
11133                                    // silently landing on some other sheet.
11134                                    let context_sheet = self.graph.get_vertex_sheet_id(vertex);
11135                                    let Ok(sheet_id) =
11136                                        self.resolve_sheet_locator(&range.sheet, context_sheet)
11137                                    else {
11138                                        if Self::widen_target_preparation(
11139                                            options.opaque_policy,
11140                                            &mut scope,
11141                                            &mut reasons,
11142                                            OpaqueReason::UnresolvedCrossSheetBinding,
11143                                        )? {
11144                                            break;
11145                                        }
11146                                        continue;
11147                                    };
11148                                    regions.push_back(PreparationRegion {
11149                                        sheet: self.graph.sheet_name(sheet_id).to_string(),
11150                                        sheet_id,
11151                                        start_row: range
11152                                            .start_row
11153                                            .map_or(1, |bound| bound.index + 1),
11154                                        start_col: range
11155                                            .start_col
11156                                            .map_or(1, |bound| bound.index + 1),
11157                                        end_row: range.end_row.map_or(
11158                                            self.workbook_load_limits.max_sheet_rows,
11159                                            |bound| bound.index + 1,
11160                                        ),
11161                                        end_col: range.end_col.map_or(
11162                                            self.workbook_load_limits.max_sheet_cols,
11163                                            |bound| bound.index + 1,
11164                                        ),
11165                                    });
11166                                }
11167                            }
11168                            NamedDefinition::Literal(_) => {}
11169                        }
11170                    }
11171                    if let Some(table) = self.graph.table_by_vertex(vertex) {
11172                        regions
11173                            .push_back(self.table_selection_region(table, &TableSelection::Whole)?);
11174                    }
11175                    continue;
11176                }
11177                break;
11178            };
11179
11180            self.target_preparation_checkpoint(options.deadline, 1)?;
11181            if !visited_regions.insert(region.clone()) {
11182                continue;
11183            }
11184            if let PrepareScope::Sheets(sheets) = &mut scope
11185                && !sheets.iter().any(|sheet| sheet == &region.sheet)
11186            {
11187                sheets.push(region.sheet.clone());
11188                sheets.sort();
11189            }
11190            let package_match = self.staged_formula_index.package_for_region(
11191                &region.sheet,
11192                region.start_row,
11193                region.start_col,
11194                region.end_row,
11195                region.end_col,
11196            );
11197            let package_lease = match package_match {
11198                Some(Ok(lease)) => Some(lease),
11199                Some(Err(()))
11200                    if region.start_row == 1
11201                        && region.start_col == 1
11202                        && region.end_row == self.workbook_load_limits.max_sheet_rows
11203                        && region.end_col == self.workbook_load_limits.max_sheet_cols =>
11204                {
11205                    self.staged_formula_index
11206                        .package_lease_for_sheet(&region.sheet)
11207                }
11208                Some(Err(())) => {
11209                    Self::widen_target_preparation(
11210                        options.opaque_policy,
11211                        &mut scope,
11212                        &mut reasons,
11213                        OpaqueReason::DeferredSourcePackage,
11214                    )?;
11215                    None
11216                }
11217                None => None,
11218            };
11219            let compatibility_before_package_replay = package_lease.is_some()
11220                && (authoritative_with_ordinary || has_unknown_package_sheet);
11221            let package_lease = if compatibility_before_package_replay {
11222                package_encountered = true;
11223                Self::widen_target_preparation(
11224                    options.opaque_policy,
11225                    &mut scope,
11226                    &mut reasons,
11227                    OpaqueReason::UnsupportedSourceSemantics,
11228                )?;
11229                None
11230            } else {
11231                package_lease
11232            };
11233            if let Some(package_lease) = package_lease
11234                && selected_package_sheets.insert(region.sheet.clone())
11235            {
11236                self.target_preparation_checkpoint(options.deadline, 1)?;
11237                let mut package = self.prepare_target_source_package(
11238                    &region.sheet,
11239                    package_lease,
11240                    options.deadline,
11241                )?;
11242                for placement in &package.placements {
11243                    self.target_preparation_checkpoint(options.deadline, 1)?;
11244                    for dependency in &placement.fragment_dependency_proof().1.dependencies {
11245                        self.target_preparation_checkpoint(options.deadline, 1)?;
11246                        let (rows, cols) = dependency.read_region.axis_ranges();
11247                        let (start_row, end_row) = rows.query_bounds();
11248                        let (start_col, end_col) = cols.query_bounds();
11249                        let dependency_sheet = dependency.read_region.sheet_id();
11250                        regions.push_back(PreparationRegion {
11251                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11252                            sheet_id: dependency_sheet,
11253                            start_row: start_row.saturating_add(1),
11254                            start_col: start_col.saturating_add(1),
11255                            end_row: end_row
11256                                .min(self.workbook_load_limits.max_sheet_rows.saturating_sub(1))
11257                                .saturating_add(1),
11258                            end_col: end_col
11259                                .min(self.workbook_load_limits.max_sheet_cols.saturating_sub(1))
11260                                .saturating_add(1),
11261                        });
11262                    }
11263                }
11264
11265                let mut final_fallback = BTreeMap::new();
11266                for record in package.fallback_records() {
11267                    final_fallback.insert((record.row, record.col), record.clone());
11268                }
11269                let batch = self
11270                    .formula_batch_from_exact_replay(&region.sheet, final_fallback.into_values())?;
11271                for record in batch.formulas {
11272                    self.target_preparation_checkpoint(options.deadline, 1)?;
11273                    let ast = self
11274                        .graph
11275                        .data_store()
11276                        .retrieve_ast(record.ast_id, self.graph.sheet_reg())
11277                        .ok_or_else(|| {
11278                            ExcelError::new(ExcelErrorKind::Value)
11279                                .with_message("target fallback AST is unavailable")
11280                        })?;
11281                    let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
11282                    if let Some(reason) =
11283                        Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
11284                    {
11285                        return Err(Self::preparation_stale(
11286                            reason,
11287                            "target fallback planning snapshot became stale during discovery",
11288                        ));
11289                    }
11290                    let proven_sheet_local_dynamic =
11291                        Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
11292                    if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
11293                        if reason == OpaqueReason::DynamicReference && proven_sheet_local_dynamic {
11294                            Self::widen_target_preparation_to_sheet(
11295                                options.opaque_policy,
11296                                &mut scope,
11297                                &mut reasons,
11298                                reason,
11299                                &region.sheet,
11300                            )?;
11301                        } else {
11302                            Self::widen_target_preparation(
11303                                options.opaque_policy,
11304                                &mut scope,
11305                                &mut reasons,
11306                                reason,
11307                            )?;
11308                        }
11309                    }
11310                    let placement = CellRef::new(
11311                        package.sheet_id,
11312                        Coord::from_excel(record.row, record.col, true, true),
11313                    );
11314                    let ingested = self
11315                        .graph
11316                        .ingest_pipeline(&snapshot)
11317                        .enable_function_semantics()
11318                        .ingest_formula(
11319                            FormulaAstInput::RawArena(record.ast_id),
11320                            placement,
11321                            record.formula_text,
11322                        )?;
11323                    if ingested.dep_plan.dynamic {
11324                        if proven_sheet_local_dynamic {
11325                            Self::widen_target_preparation_to_sheet(
11326                                options.opaque_policy,
11327                                &mut scope,
11328                                &mut reasons,
11329                                OpaqueReason::DynamicReference,
11330                                &region.sheet,
11331                            )?;
11332                        } else {
11333                            Self::widen_target_preparation(
11334                                options.opaque_policy,
11335                                &mut scope,
11336                                &mut reasons,
11337                                OpaqueReason::DynamicReference,
11338                            )?;
11339                        }
11340                    }
11341                    for dep in &ingested.dep_plan.direct_cell_deps {
11342                        self.target_preparation_checkpoint(options.deadline, 1)?;
11343                        regions.push_back(PreparationRegion {
11344                            sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
11345                            sheet_id: dep.sheet_id,
11346                            start_row: dep.coord.row().saturating_add(1),
11347                            start_col: dep.coord.col().saturating_add(1),
11348                            end_row: dep.coord.row().saturating_add(1),
11349                            end_col: dep.coord.col().saturating_add(1),
11350                        });
11351                    }
11352                    for range in &ingested.dep_plan.range_deps {
11353                        self.target_preparation_checkpoint(options.deadline, 1)?;
11354                        // `Current` is the sheet the staged package's formula
11355                        // lives on.
11356                        let Ok(dependency_sheet) =
11357                            self.resolve_sheet_locator(&range.sheet, package.sheet_id)
11358                        else {
11359                            Self::widen_target_preparation(
11360                                options.opaque_policy,
11361                                &mut scope,
11362                                &mut reasons,
11363                                OpaqueReason::UnresolvedCrossSheetBinding,
11364                            )?;
11365                            continue;
11366                        };
11367                        regions.push_back(PreparationRegion {
11368                            sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11369                            sheet_id: dependency_sheet,
11370                            start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11371                            start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11372                            end_row: range
11373                                .end_row
11374                                .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11375                                    bound.index + 1
11376                                }),
11377                            end_col: range
11378                                .end_col
11379                                .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11380                                    bound.index + 1
11381                                }),
11382                        });
11383                    }
11384                    for name in ingested
11385                        .dep_plan
11386                        .resolved_named_refs
11387                        .iter()
11388                        .chain(&ingested.dep_plan.named_refs)
11389                    {
11390                        self.target_preparation_checkpoint(options.deadline, 1)?;
11391                        if let Some(entry) = self.graph.resolve_name_entry(name, package.sheet_id) {
11392                            symbol_vertices.push_back(entry.vertex);
11393                        } else if self.graph.resolve_source_scalar_entry(name).is_none()
11394                            && self.graph.resolve_source_table_entry(name).is_none()
11395                        {
11396                            Self::widen_target_preparation(
11397                                options.opaque_policy,
11398                                &mut scope,
11399                                &mut reasons,
11400                                OpaqueReason::UnresolvedName,
11401                            )?;
11402                        }
11403                    }
11404                    for table in &ingested.dep_plan.table_refs {
11405                        self.target_preparation_checkpoint(options.deadline, 1)?;
11406                        if let Some(entry) = self.graph.resolve_table_entry(table) {
11407                            symbol_vertices.push_back(entry.vertex);
11408                        } else if self.graph.resolve_source_table_entry(table).is_none() {
11409                            Self::widen_target_preparation(
11410                                options.opaque_policy,
11411                                &mut scope,
11412                                &mut reasons,
11413                                OpaqueReason::UnresolvedTable,
11414                            )?;
11415                        }
11416                    }
11417                    package.legacy.push((
11418                        record.row,
11419                        record.col,
11420                        ingested.ast_id,
11421                        ingested.dep_plan,
11422                    ));
11423                }
11424                prepared_packages.push(package);
11425            }
11426            let leases = self.staged_formula_index.leases_in_region(
11427                &region.sheet,
11428                region.start_row,
11429                region.start_col,
11430                region.end_row,
11431                region.end_col,
11432            );
11433            for lease in leases {
11434                self.target_preparation_checkpoint(options.deadline, 1)?;
11435                let sheet_id = self.graph.sheet_id(&region.sheet).ok_or_else(|| {
11436                    ExcelError::new(ExcelErrorKind::Ref)
11437                        .with_message(format!("staged formula sheet not found: {}", region.sheet))
11438                })?;
11439                let key = (region.sheet.clone(), lease.row, lease.col, lease.generation);
11440                if !selected.insert(key) {
11441                    continue;
11442                }
11443                let text = self
11444                    .staged_formulas
11445                    .get(&region.sheet)
11446                    .and_then(|sheet| sheet.get_ordinary(lease.row, lease.col))
11447                    .ok_or_else(|| {
11448                        ExcelError::new(ExcelErrorKind::Value)
11449                            .with_message("staged formula index is stale")
11450                    })?
11451                    .to_string();
11452                let formula = if text.starts_with('=') {
11453                    text.clone()
11454                } else {
11455                    format!("={text}")
11456                };
11457                self.target_preparation_checkpoint(options.deadline, 1)?;
11458                let ast = match formualizer_parse::parser::parse(&formula) {
11459                    Ok(ast) => ast,
11460                    Err(error) => {
11461                        if self.config.formula_parse_policy == FormulaParsePolicy::Strict {
11462                            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
11463                                format!(
11464                                    "Formula parse error at {}!{}{}: {error}",
11465                                    region.sheet,
11466                                    col_letters_from_1based(lease.col)
11467                                        .unwrap_or_else(|_| "?".to_string()),
11468                                    lease.row
11469                                ),
11470                            ));
11471                        }
11472                        pending_diagnostics.push(FormulaParseDiagnostic {
11473                            sheet: region.sheet.clone(),
11474                            row: lease.row,
11475                            col: lease.col,
11476                            formula: formula.clone(),
11477                            message: error.to_string(),
11478                            policy: self.config.formula_parse_policy,
11479                        });
11480                        match self.config.formula_parse_policy {
11481                            FormulaParsePolicy::KeepCachedValue => {
11482                                selected_cells.push(
11483                                    formualizer_common::RangeAddress::new(
11484                                        region.sheet.clone(),
11485                                        lease.row,
11486                                        lease.col,
11487                                        lease.row,
11488                                        lease.col,
11489                                    )
11490                                    .expect("selected staged coordinates are valid"),
11491                                );
11492                                prepared.push(PreparedOrdinaryStagedFormula {
11493                                    sheet: region.sheet.clone(),
11494                                    sheet_id,
11495                                    lease,
11496                                    ast_id: None,
11497                                    plan: None,
11498                                });
11499                                continue;
11500                            }
11501                            FormulaParsePolicy::AsText => ASTNode::new(
11502                                ASTNodeType::Literal(LiteralValue::Text(formula.clone())),
11503                                None,
11504                            ),
11505                            FormulaParsePolicy::CoerceToError => ASTNode::new(
11506                                ASTNodeType::Literal(LiteralValue::Error(
11507                                    ExcelError::new(ExcelErrorKind::Error)
11508                                        .with_message(format!("Malformed formula: {error}")),
11509                                )),
11510                                None,
11511                            ),
11512                            FormulaParsePolicy::Strict => unreachable!(),
11513                        }
11514                    }
11515                };
11516                self.target_preparation_checkpoint(options.deadline, 1)?;
11517                let snapshot = self.target_planning_snapshot(&ast, &mut planning_requests)?;
11518                self.target_preparation_checkpoint(options.deadline, 1)?;
11519                if let Some(reason) =
11520                    Self::target_planning_snapshot_stale_reason(&snapshot, &assumptions)
11521                {
11522                    return Err(Self::preparation_stale(
11523                        reason,
11524                        "target planning snapshot became stale during discovery",
11525                    ));
11526                }
11527                if let Some(reason) = self.opaque_reason_in_ast(&ast, &snapshot) {
11528                    if reason == OpaqueReason::DynamicReference
11529                        && Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot)
11530                    {
11531                        Self::widen_target_preparation_to_sheet(
11532                            options.opaque_policy,
11533                            &mut scope,
11534                            &mut reasons,
11535                            reason,
11536                            &region.sheet,
11537                        )?;
11538                    } else {
11539                        Self::widen_target_preparation(
11540                            options.opaque_policy,
11541                            &mut scope,
11542                            &mut reasons,
11543                            reason,
11544                        )?;
11545                    }
11546                }
11547                let proven_sheet_local_dynamic =
11548                    Self::ast_has_proven_sheet_local_dynamic(&ast, &snapshot);
11549                let placement = CellRef::new(
11550                    sheet_id,
11551                    Coord::from_excel(lease.row, lease.col, true, true),
11552                );
11553                let ingested = self.graph.ingest_pipeline(&snapshot).ingest_formula(
11554                    FormulaAstInput::Tree(ast),
11555                    placement,
11556                    Some(Arc::from(formula)),
11557                )?;
11558                self.target_preparation_checkpoint(options.deadline, 1)?;
11559                if ingested.dep_plan.dynamic {
11560                    if proven_sheet_local_dynamic {
11561                        Self::widen_target_preparation_to_sheet(
11562                            options.opaque_policy,
11563                            &mut scope,
11564                            &mut reasons,
11565                            OpaqueReason::DynamicReference,
11566                            &region.sheet,
11567                        )?;
11568                    } else {
11569                        Self::widen_target_preparation(
11570                            options.opaque_policy,
11571                            &mut scope,
11572                            &mut reasons,
11573                            OpaqueReason::DynamicReference,
11574                        )?;
11575                    }
11576                }
11577                for dep in &ingested.dep_plan.direct_cell_deps {
11578                    self.target_preparation_checkpoint(options.deadline, 1)?;
11579                    regions.push_back(PreparationRegion {
11580                        sheet: self.graph.sheet_name(dep.sheet_id).to_string(),
11581                        sheet_id: dep.sheet_id,
11582                        start_row: dep.coord.row() + 1,
11583                        start_col: dep.coord.col() + 1,
11584                        end_row: dep.coord.row() + 1,
11585                        end_col: dep.coord.col() + 1,
11586                    });
11587                }
11588                for range in &ingested.dep_plan.range_deps {
11589                    self.target_preparation_checkpoint(options.deadline, 1)?;
11590                    // `Current` is the sheet the staged formula lives on.
11591                    let Ok(dependency_sheet) = self.resolve_sheet_locator(&range.sheet, sheet_id)
11592                    else {
11593                        Self::widen_target_preparation(
11594                            options.opaque_policy,
11595                            &mut scope,
11596                            &mut reasons,
11597                            OpaqueReason::UnresolvedCrossSheetBinding,
11598                        )?;
11599                        continue;
11600                    };
11601                    regions.push_back(PreparationRegion {
11602                        sheet: self.graph.sheet_name(dependency_sheet).to_string(),
11603                        sheet_id: dependency_sheet,
11604                        start_row: range.start_row.map_or(1, |bound| bound.index + 1),
11605                        start_col: range.start_col.map_or(1, |bound| bound.index + 1),
11606                        end_row: range
11607                            .end_row
11608                            .map_or(self.workbook_load_limits.max_sheet_rows, |bound| {
11609                                bound.index + 1
11610                            }),
11611                        end_col: range
11612                            .end_col
11613                            .map_or(self.workbook_load_limits.max_sheet_cols, |bound| {
11614                                bound.index + 1
11615                            }),
11616                    });
11617                }
11618                for name in ingested
11619                    .dep_plan
11620                    .resolved_named_refs
11621                    .iter()
11622                    .chain(&ingested.dep_plan.named_refs)
11623                {
11624                    self.target_preparation_checkpoint(options.deadline, 1)?;
11625                    if let Some(entry) = self.graph.resolve_name_entry(name, sheet_id) {
11626                        symbol_vertices.push_back(entry.vertex);
11627                    } else if self.graph.resolve_source_scalar_entry(name).is_none()
11628                        && self.graph.resolve_source_table_entry(name).is_none()
11629                    {
11630                        Self::widen_target_preparation(
11631                            options.opaque_policy,
11632                            &mut scope,
11633                            &mut reasons,
11634                            OpaqueReason::UnresolvedName,
11635                        )?;
11636                    }
11637                }
11638                for table in &ingested.dep_plan.table_refs {
11639                    self.target_preparation_checkpoint(options.deadline, 1)?;
11640                    if let Some(entry) = self.graph.resolve_table_entry(table) {
11641                        symbol_vertices.push_back(entry.vertex);
11642                    } else if self.graph.resolve_source_table_entry(table).is_none() {
11643                        Self::widen_target_preparation(
11644                            options.opaque_policy,
11645                            &mut scope,
11646                            &mut reasons,
11647                            OpaqueReason::UnresolvedTable,
11648                        )?;
11649                    }
11650                }
11651                selected_cells.push(
11652                    formualizer_common::RangeAddress::new(
11653                        region.sheet.clone(),
11654                        lease.row,
11655                        lease.col,
11656                        lease.row,
11657                        lease.col,
11658                    )
11659                    .expect("selected staged coordinates are valid"),
11660                );
11661                prepared.push(PreparedOrdinaryStagedFormula {
11662                    sheet: region.sheet.clone(),
11663                    sheet_id,
11664                    lease,
11665                    ast_id: Some(ingested.ast_id),
11666                    plan: Some(ingested.dep_plan),
11667                });
11668            }
11669
11670            if indexed_query_sheets.insert(region.sheet_id) {
11671                self.graph.prepare_sheet_index_for_query(region.sheet_id);
11672                let bytes = (self.graph.sheet_index_vertex_count(region.sheet_id) as u64)
11673                    .saturating_mul(32);
11674                self.reserve_graph_source_scratch(bytes)?;
11675                discovery_scratch_reserved = discovery_scratch_reserved.saturating_add(bytes);
11676            }
11677            let spill_anchors = self.graph.spill_anchors_in_region(
11678                region.sheet_id,
11679                region.start_row - 1,
11680                region.start_col - 1,
11681                region.end_row - 1,
11682                region.end_col - 1,
11683            );
11684            for anchor in spill_anchors {
11685                self.target_preparation_checkpoint(options.deadline, 1)?;
11686                symbol_vertices.push_back(anchor);
11687            }
11688            let vertices = self.graph.vertices_in_region(
11689                region.sheet_id,
11690                region.start_row - 1,
11691                region.end_row - 1,
11692                region.start_col - 1,
11693                region.end_col - 1,
11694            );
11695            for vertex in vertices {
11696                self.target_preparation_checkpoint(options.deadline, 1)?;
11697                symbol_vertices.push_back(vertex);
11698            }
11699        }
11700
11701        #[cfg(test)]
11702        self.target_preparation_fault(
11703            crate::engine::target_preparation::TargetPreparationFault::AfterDiscovery,
11704        )?;
11705
11706        if package_encountered
11707            || (self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
11708                && !prepared.is_empty())
11709        {
11710            Self::widen_target_preparation(
11711                options.opaque_policy,
11712                &mut scope,
11713                &mut reasons,
11714                OpaqueReason::UnsupportedSourceSemantics,
11715            )?;
11716            self.target_preparation_checkpoint(options.deadline, 0)?;
11717            let selected_count = self.staged_formula_count();
11718            let selected_packages = self
11719                .staged_formulas
11720                .values()
11721                .filter_map(|staged| staged.deferred_package.as_ref())
11722                .map(|package| package.families.len() + package.partitioned_families.len())
11723                .sum();
11724            self.formula_parse_diagnostics.truncate(diagnostics_len);
11725            self.last_formula_ingest_report = report_len;
11726            #[cfg(test)]
11727            if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11728                hook();
11729            }
11730            self.target_preparation_checkpoint(options.deadline, 0)?;
11731            #[cfg(test)]
11732            self.target_preparation_fault(
11733                crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11734            )?;
11735            let current_revisions = self.preparation_revisions();
11736            if let Some(reason) = Self::preparation_revision_stale_reason(
11737                &assumptions,
11738                &current_revisions,
11739                &planning_requests,
11740                true,
11741            ) {
11742                return Err(Self::preparation_stale(
11743                    reason,
11744                    "target compatibility preparation plan is stale",
11745                ));
11746            }
11747            #[cfg(test)]
11748            self.target_preparation_fault(
11749                crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11750            )?;
11751            let commit_work_before = self
11752                .active_resource_ledger
11753                .as_ref()
11754                .map_or(0, |ledger| ledger.snapshot().work_charged);
11755            let commit_started = crate::instant::FzInstant::now();
11756            self.build_graph_all_unobserved()?;
11757            let commit_window = commit_started.elapsed();
11758            let ledger_after = self
11759                .active_resource_ledger
11760                .as_ref()
11761                .map(|ledger| ledger.snapshot());
11762            let actual_commit_work = ledger_after
11763                .map_or(0, |snapshot| snapshot.work_charged)
11764                .saturating_sub(commit_work_before);
11765            let observed_scratch_bytes = ledger_after
11766                .map_or(0, |snapshot| snapshot.scratch_peak)
11767                .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
11768            let revisions = assumptions.clone();
11769            let report = PreparedTargetGraphReport {
11770                request_id: request_id.unwrap_or_default(),
11771                requested_targets: targets.len(),
11772                normalized_regions: visited_regions.len(),
11773                normalized_target_list: normalized,
11774                selected_staged_cells: selected_count,
11775                selected_source_families: selected_packages,
11776                retained_staged_cells: self.staged_formula_count(),
11777                selected_cells,
11778                retained_cells: Vec::new(),
11779                widened_scope: PrepareScope::Workbook,
11780                widening_reasons: reasons,
11781                revisions,
11782                commit_window,
11783                estimated_scratch_bytes: discovery_scratch_reserved
11784                    .saturating_add((selected_count as u64).saturating_mul(256)),
11785                observed_scratch_bytes,
11786                estimated_commit_work: selected_count as u64,
11787                actual_commit_work,
11788                outcome: PreparationOutcome::CompatibilityPrepared,
11789            };
11790            self.observe_target_preparation_report(&report);
11791            return Ok(report);
11792        }
11793
11794        prepared.sort_by_key(|formula| formula.lease.insertion_order);
11795        for package in &prepared_packages {
11796            selected_cells.extend(package.replay_records.iter().filter_map(|record| {
11797                formualizer_common::RangeAddress::new(
11798                    package.sheet.clone(),
11799                    record.row,
11800                    record.col,
11801                    record.row,
11802                    record.col,
11803                )
11804                .ok()
11805            }));
11806        }
11807        let (mut legacy_graph, mut planned_formula_count) =
11808            self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11809        let placements = prepared_packages
11810            .iter_mut()
11811            .flat_map(|package| std::mem::take(&mut package.placements))
11812            .collect::<Vec<_>>();
11813        let formula_plane = if placements.is_empty() {
11814            None
11815        } else {
11816            match self.graph.formula_authority().prepare_formula_plane_append(
11817                placements,
11818                self.graph.data_store(),
11819                self.graph.sheet_reg(),
11820            ) {
11821                Ok(append) => Some(append),
11822                Err(error) => {
11823                    let reason = format!("TargetFormulaPlaneAppend:{error}");
11824                    for package in &mut prepared_packages {
11825                        *package
11826                            .source_report
11827                            .fallback_reasons
11828                            .entry(reason.clone())
11829                            .or_default() += package.direct_families as u64;
11830                        if self.config.formula_plane_mode
11831                            == FormulaPlaneMode::AuthoritativeExperimental
11832                        {
11833                            self.materialize_target_package_direct_records(
11834                                package,
11835                                &assumptions,
11836                                &mut planning_requests,
11837                                options.deadline,
11838                            )?;
11839                        } else {
11840                            package.direct_families = 0;
11841                            package.direct_cells = 0;
11842                            package.direct_fragments = 0;
11843                            package.direct_complete_families = 0;
11844                            package.direct_complete_cells = 0;
11845                            package.direct_partition_families = 0;
11846                            package.direct_partition_cells = 0;
11847                        }
11848                    }
11849                    (legacy_graph, planned_formula_count) =
11850                        self.prepare_target_combined_legacy_graph(&prepared_packages, &prepared)?;
11851                    None
11852                }
11853            }
11854        };
11855        let new_vertices = legacy_graph.new_vertex_count();
11856        let new_edges = legacy_graph.planned_edge_count().ok_or_else(|| {
11857            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11858        })?;
11859        let removed_edges = legacy_graph.removed_edge_count().ok_or_else(|| {
11860            ExcelError::new(ExcelErrorKind::NImpl).with_message("target graph edge count overflow")
11861        })?;
11862        let current = self.graph.baseline_stats();
11863        let final_vertices = current
11864            .graph_vertex_count
11865            .checked_add(new_vertices)
11866            .ok_or_else(|| {
11867                crate::engine::ResourceLedgerError::Exhausted(
11868                    formualizer_common::ResourceExhaustionDetail {
11869                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11870                        limit: u64::MAX,
11871                        observed: u64::MAX,
11872                        request_id,
11873                    },
11874                )
11875                .into_excel_error()
11876            })?;
11877        let final_edges = current
11878            .graph_edge_count
11879            .checked_sub(removed_edges)
11880            .and_then(|count| count.checked_add(new_edges))
11881            .ok_or_else(|| {
11882                crate::engine::ResourceLedgerError::Exhausted(
11883                    formualizer_common::ResourceExhaustionDetail {
11884                        reason: formualizer_common::ResourceExhaustionReason::ArithmeticOverflow,
11885                        limit: u64::MAX,
11886                        observed: u64::MAX,
11887                        request_id,
11888                    },
11889                )
11890                .into_excel_error()
11891            })?;
11892        #[cfg(test)]
11893        self.target_preparation_fault(
11894            crate::engine::target_preparation::TargetPreparationFault::Admission,
11895        )?;
11896        let resource = |reason, limit: u64, observed: u64| {
11897            crate::engine::ResourceLedgerError::Exhausted(
11898                formualizer_common::ResourceExhaustionDetail {
11899                    reason,
11900                    limit,
11901                    observed,
11902                    request_id,
11903                },
11904            )
11905            .into_excel_error()
11906        };
11907        let admission = crate::engine::resource_ledger::GraphAdmission {
11908            final_vertices,
11909            final_edges,
11910            materialization_cells: planned_formula_count as u64,
11911            added_vertices: new_vertices,
11912            added_edges: new_edges,
11913        };
11914        let materialized_bytes = admission
11915            .materialized_graph_bytes()
11916            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
11917        if let Err(error) = self.preflight_graph_admission(admission) {
11918            if let formualizer_common::ExcelErrorExtra::Resource { detail } = &error.extra {
11919                self.observe_target_admission_failure(detail.reason);
11920            }
11921            return Err(error);
11922        }
11923        let selected_package_records = prepared_packages
11924            .iter()
11925            .map(|package| package.replay_records.len() as u64)
11926            .sum::<u64>();
11927        let planned_working_bytes = (prepared.len() as u64)
11928            .saturating_add(selected_package_records)
11929            .saturating_mul(256)
11930            .saturating_add((visited_regions.len() as u64).saturating_mul(128))
11931            .saturating_add((visited_vertices.len() as u64).saturating_mul(32))
11932            .saturating_add((selected.len() as u64).saturating_mul(96))
11933            .saturating_add(materialized_bytes);
11934        let scratch_bytes = discovery_scratch_reserved.saturating_add(planned_working_bytes);
11935        let remaining_scratch = scratch_bytes.saturating_sub(discovery_scratch_reserved);
11936        if let Err(error) = self.reserve_graph_source_scratch(remaining_scratch) {
11937            self.observe_target_admission_failure(
11938                formualizer_common::ResourceExhaustionReason::ScratchMemory,
11939            );
11940            return Err(error);
11941        }
11942
11943        let estimated_commit_duration = std::time::Duration::from_nanos(
11944            (new_vertices as u64)
11945                .saturating_add(new_edges as u64)
11946                .saturating_add(prepared.len() as u64)
11947                .saturating_add(selected_package_records)
11948                .max(1)
11949                .saturating_mul(100),
11950        );
11951        if options.deadline.is_some_and(|deadline| {
11952            std::time::Instant::now()
11953                .checked_add(estimated_commit_duration)
11954                .is_none_or(|finish| finish > deadline)
11955        }) {
11956            self.observe_target_admission_failure(
11957                formualizer_common::ResourceExhaustionReason::Deadline,
11958            );
11959            return Err(resource(
11960                formualizer_common::ResourceExhaustionReason::Deadline,
11961                0,
11962                1,
11963            ));
11964        }
11965        #[cfg(test)]
11966        if let Some(hook) = self.before_target_preparation_commit_hook.take() {
11967            hook();
11968        }
11969        self.target_preparation_checkpoint(options.deadline, 0)?;
11970        #[cfg(test)]
11971        self.target_preparation_fault(
11972            crate::engine::target_preparation::TargetPreparationFault::FinalRevisionValidation,
11973        )?;
11974        let current_revisions = self.preparation_revisions();
11975        let staged_leases_match = prepared.iter().all(|formula| {
11976            self.staged_formula_index
11977                .lease_matches(&formula.sheet, formula.lease)
11978        }) && prepared_packages.iter().all(|package| {
11979            self.staged_formula_index
11980                .package_lease_matches(&package.sheet, package.lease)
11981        });
11982        let stale_reason = Self::preparation_revision_stale_reason(
11983            &assumptions,
11984            &current_revisions,
11985            &planning_requests,
11986            staged_leases_match,
11987        );
11988        if let Some(reason) = stale_reason {
11989            return Err(Self::preparation_stale(
11990                reason,
11991                "target graph preparation plan is stale",
11992            ));
11993        }
11994        #[cfg(test)]
11995        self.target_preparation_fault(
11996            crate::engine::target_preparation::TargetPreparationFault::FinalGraphValidation,
11997        )?;
11998        self.graph
11999            .validate_prepared_legacy_graph_plan(&legacy_graph)
12000            .map_err(|error| {
12001                Self::preparation_stale(
12002                    formualizer_common::PreparationStaleReason::Graph,
12003                    format!("target graph preparation plan is stale: {error}"),
12004                )
12005            })?;
12006        if let Some(append) = formula_plane.as_ref() {
12007            self.graph
12008                .formula_authority()
12009                .validate_prepared_formula_plane_append(
12010                    append,
12011                    self.graph.data_store(),
12012                    self.graph.sheet_reg(),
12013                )
12014                .map_err(|error| {
12015                    Self::preparation_stale(
12016                        formualizer_common::PreparationStaleReason::Authority,
12017                        format!("target FormulaPlane preparation is stale: {error}"),
12018                    )
12019                })?;
12020        }
12021        #[cfg(test)]
12022        self.target_preparation_fault(
12023            crate::engine::target_preparation::TargetPreparationFault::Reservation,
12024        )?;
12025        self.graph.reserve_prepared_legacy_graph_plan(&legacy_graph);
12026        self.formula_parse_diagnostics
12027            .try_reserve(pending_diagnostics.len())
12028            .map_err(|_| {
12029                resource(
12030                    formualizer_common::ResourceExhaustionReason::Admission,
12031                    pending_diagnostics.len() as u64,
12032                    pending_diagnostics.len() as u64,
12033                )
12034            })?;
12035        self.target_preparation_checkpoint(options.deadline, 0)?;
12036        #[cfg(test)]
12037        self.target_preparation_fault(
12038            crate::engine::target_preparation::TargetPreparationFault::BeforeFirstMutation,
12039        )?;
12040
12041        let commit_started = crate::instant::FzInstant::now();
12042        let committed = self
12043            .graph
12044            .apply_prevalidated_legacy_graph_plan(legacy_graph);
12045        let plane_report =
12046            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
12047                formula_plane.map(|append| {
12048                    self.graph
12049                        .formula_authority_mut()
12050                        .apply_prevalidated_formula_plane_append(append)
12051                })
12052            } else {
12053                None
12054            };
12055        if let Some(report) = plane_report.as_ref() {
12056            self.graph.mark_formula_spans_dirty(
12057                report.spans.iter().copied(),
12058                WholeSpanDirtyReason::NewSpan,
12059            );
12060        }
12061        for formula in &prepared {
12062            let removed = self
12063                .staged_formulas
12064                .get_mut(&formula.sheet)
12065                .and_then(|sheet| sheet.remove_ordinary(formula.lease.row, formula.lease.col));
12066            debug_assert!(removed.is_some());
12067            let index_removed = self.staged_formula_index.remove(
12068                &formula.sheet,
12069                formula.lease.row,
12070                formula.lease.col,
12071            );
12072            debug_assert!(index_removed);
12073        }
12074        for package in &prepared_packages {
12075            let removed = self
12076                .staged_formulas
12077                .get_mut(&package.sheet)
12078                .and_then(|staged| staged.deferred_package.take());
12079            debug_assert!(removed.is_some());
12080            self.staged_formula_index.set_package(&package.sheet, None);
12081        }
12082        let empty_sheets = self
12083            .staged_formulas
12084            .iter()
12085            .filter_map(|(sheet, staged)| staged.is_empty().then_some(sheet.clone()))
12086            .collect::<Vec<_>>();
12087        for sheet in empty_sheets {
12088            self.staged_formulas.remove(&sheet);
12089        }
12090        if committed > 0 || plane_report.is_some() {
12091            self.mark_topology_edited();
12092        }
12093        self.formula_parse_diagnostics.extend(pending_diagnostics);
12094        if !prepared.is_empty() || !prepared_packages.is_empty() {
12095            let mut ingest_delta = FormulaIngestReport::with_mode(self.config.formula_plane_mode);
12096            ingest_delta.formula_cells_seen = (prepared.len() as u64).saturating_add(
12097                prepared_packages
12098                    .iter()
12099                    .map(|package| package.replay_records.len() as u64)
12100                    .sum::<u64>(),
12101            );
12102            ingest_delta.graph_formula_cells_materialized = committed as u64;
12103            ingest_delta.graph_vertices_created = new_vertices as u64;
12104            ingest_delta.graph_edges_created = new_edges as u64;
12105            for package in &prepared_packages {
12106                let source = &package.source_report;
12107                ingest_delta.source_formula_events = ingest_delta
12108                    .source_formula_events
12109                    .saturating_add(source.source_formula_events);
12110                ingest_delta.source_formula_records_spooled = ingest_delta
12111                    .source_formula_records_spooled
12112                    .saturating_add(source.source_formula_records_spooled);
12113                ingest_delta.source_spool_encoded_bytes = ingest_delta
12114                    .source_spool_encoded_bytes
12115                    .saturating_add(source.source_spool_encoded_bytes);
12116                ingest_delta.source_spool_peak_memory_bytes = ingest_delta
12117                    .source_spool_peak_memory_bytes
12118                    .max(source.source_spool_peak_memory_bytes);
12119                ingest_delta.source_spool_spilled_bytes = ingest_delta
12120                    .source_spool_spilled_bytes
12121                    .saturating_add(source.source_spool_spilled_bytes);
12122                ingest_delta.source_spool_spill_files = ingest_delta
12123                    .source_spool_spill_files
12124                    .saturating_add(source.source_spool_spill_files);
12125                ingest_delta.source_spool_replays = ingest_delta
12126                    .source_spool_replays
12127                    .saturating_add(source.source_spool_replays)
12128                    .saturating_add(1);
12129                ingest_delta.source_families_seen = ingest_delta
12130                    .source_families_seen
12131                    .saturating_add(source.families_seen);
12132                ingest_delta.source_family_cells_seen = ingest_delta
12133                    .source_family_cells_seen
12134                    .saturating_add(source.family_cells_seen);
12135                ingest_delta.source_family_shadow_eligible = ingest_delta
12136                    .source_family_shadow_eligible
12137                    .saturating_add(source.source_clean_families);
12138                ingest_delta.source_family_shadow_eligible_cells = ingest_delta
12139                    .source_family_shadow_eligible_cells
12140                    .saturating_add(source.source_clean_cells);
12141                ingest_delta.source_partitioned_families_seen = ingest_delta
12142                    .source_partitioned_families_seen
12143                    .saturating_add(source.source_fragmentable_families);
12144                ingest_delta.source_partition_holes = ingest_delta
12145                    .source_partition_holes
12146                    .saturating_add(source.source_hole_exclusions);
12147                ingest_delta.source_partition_ordinary_exceptions = ingest_delta
12148                    .source_partition_ordinary_exceptions
12149                    .saturating_add(source.source_ordinary_exclusions);
12150                ingest_delta.source_partition_surviving_cells = ingest_delta
12151                    .source_partition_surviving_cells
12152                    .saturating_add(source.source_fragmentable_cells);
12153                for (reason, count) in &source.fallback_reasons {
12154                    let total = ingest_delta
12155                        .fallback_reasons
12156                        .entry(reason.clone())
12157                        .or_default();
12158                    *total = total.saturating_add(*count);
12159                }
12160                if self.config.formula_plane_mode == FormulaPlaneMode::Off {
12161                    ingest_delta.source_family_fallback = ingest_delta
12162                        .source_family_fallback
12163                        .saturating_add(source.families_seen);
12164                    ingest_delta.source_family_fallback_cells = ingest_delta
12165                        .source_family_fallback_cells
12166                        .saturating_add(source.family_cells_seen);
12167                } else {
12168                    ingest_delta.shadow_candidate_cells = ingest_delta
12169                        .shadow_candidate_cells
12170                        .saturating_add(source.family_cells_seen);
12171                    ingest_delta.shadow_accepted_span_cells = ingest_delta
12172                        .shadow_accepted_span_cells
12173                        .saturating_add(package.direct_cells);
12174                    ingest_delta.shadow_fallback_cells =
12175                        ingest_delta.shadow_fallback_cells.saturating_add(
12176                            source
12177                                .family_cells_seen
12178                                .saturating_sub(package.direct_cells),
12179                        );
12180                    ingest_delta.source_anchor_parses = ingest_delta
12181                        .source_anchor_parses
12182                        .saturating_add(package.anchor_parses);
12183                    ingest_delta.source_anchor_asts = ingest_delta
12184                        .source_anchor_asts
12185                        .saturating_add(package.anchor_asts);
12186                    ingest_delta.source_anchor_analyses = ingest_delta
12187                        .source_anchor_analyses
12188                        .saturating_add(package.anchor_analyses);
12189                    ingest_delta.source_compressed_families_prepared = ingest_delta
12190                        .source_compressed_families_prepared
12191                        .saturating_add(package.direct_complete_families);
12192                    ingest_delta.source_compressed_cells_prepared = ingest_delta
12193                        .source_compressed_cells_prepared
12194                        .saturating_add(package.direct_complete_cells);
12195                    ingest_delta.source_partitioned_families_prepared = ingest_delta
12196                        .source_partitioned_families_prepared
12197                        .saturating_add(package.direct_partition_families);
12198                    ingest_delta.source_partition_fragments_prepared = ingest_delta
12199                        .source_partition_fragments_prepared
12200                        .saturating_add(package.direct_fragments);
12201                    ingest_delta.source_partition_span_cells_prepared = ingest_delta
12202                        .source_partition_span_cells_prepared
12203                        .saturating_add(package.direct_partition_cells);
12204                    ingest_delta.source_partition_analyses_reused = ingest_delta
12205                        .source_partition_analyses_reused
12206                        .saturating_add(
12207                            package
12208                                .direct_fragments
12209                                .saturating_sub(package.direct_partition_families),
12210                        );
12211                    ingest_delta.graph_formula_vertices_avoided_shadow = ingest_delta
12212                        .graph_formula_vertices_avoided_shadow
12213                        .saturating_add(package.direct_cells);
12214                    ingest_delta.ast_roots_avoided_shadow =
12215                        ingest_delta.ast_roots_avoided_shadow.saturating_add(
12216                            package
12217                                .direct_complete_cells
12218                                .saturating_sub(package.direct_complete_families)
12219                                .saturating_add(
12220                                    package
12221                                        .direct_partition_cells
12222                                        .saturating_sub(package.direct_fragments),
12223                                ),
12224                        );
12225                    ingest_delta.edge_rows_avoided_shadow = ingest_delta
12226                        .edge_rows_avoided_shadow
12227                        .saturating_add(package.direct_cells);
12228                    if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
12229                    {
12230                        ingest_delta.source_family_promoted = ingest_delta
12231                            .source_family_promoted
12232                            .saturating_add(package.direct_families as u64);
12233                        ingest_delta.source_family_promoted_cells = ingest_delta
12234                            .source_family_promoted_cells
12235                            .saturating_add(package.direct_cells);
12236                        let descendants = package
12237                            .direct_complete_cells
12238                            .saturating_sub(package.direct_complete_families)
12239                            .saturating_add(
12240                                package
12241                                    .direct_partition_cells
12242                                    .saturating_sub(package.direct_partition_families),
12243                            );
12244                        ingest_delta.source_descendant_strings_avoided = ingest_delta
12245                            .source_descendant_strings_avoided
12246                            .saturating_add(descendants);
12247                        ingest_delta.source_descendant_events_avoided = ingest_delta
12248                            .source_descendant_events_avoided
12249                            .saturating_add(descendants);
12250                        ingest_delta.source_descendant_analyses_avoided = ingest_delta
12251                            .source_descendant_analyses_avoided
12252                            .saturating_add(descendants);
12253                    }
12254                    ingest_delta.source_family_fallback =
12255                        ingest_delta.source_family_fallback.saturating_add(
12256                            source
12257                                .families_seen
12258                                .saturating_sub(package.direct_families as u64),
12259                        );
12260                    ingest_delta.source_family_fallback_cells =
12261                        ingest_delta.source_family_fallback_cells.saturating_add(
12262                            source
12263                                .family_cells_seen
12264                                .saturating_sub(package.direct_cells),
12265                        );
12266                }
12267            }
12268            if let Some(report) = plane_report.as_ref() {
12269                ingest_delta.shadow_templates_interned = report.work.templates_to_append as u64;
12270                ingest_delta.shadow_spans_created = report.spans.len() as u64;
12271            }
12272            self.record_formula_ingest_report(ingest_delta);
12273        }
12274        let commit_window = commit_started.elapsed();
12275        let retained_cells = self
12276            .staged_formula_index
12277            .all_leases()
12278            .into_iter()
12279            .filter_map(|(sheet, lease)| {
12280                formualizer_common::RangeAddress::new(
12281                    sheet, lease.row, lease.col, lease.row, lease.col,
12282                )
12283                .ok()
12284            })
12285            .collect::<Vec<_>>();
12286        let observed_scratch_bytes = self
12287            .active_resource_ledger
12288            .as_ref()
12289            .map_or(0, |ledger| ledger.snapshot().scratch_peak)
12290            .saturating_sub(ledger_at_start.map_or(0, |snapshot| snapshot.scratch_current));
12291        let committed_spans = plane_report
12292            .as_ref()
12293            .map_or(0, |report| report.spans.len() as u64);
12294        let selected_source_families = prepared_packages
12295            .iter()
12296            .map(|package| package.lease.family_count)
12297            .sum::<usize>();
12298        let selected_staged_cells = prepared
12299            .len()
12300            .saturating_add(usize::try_from(selected_package_records).unwrap_or(usize::MAX));
12301        let actual_commit_work = (new_vertices as u64)
12302            .saturating_add(new_edges as u64)
12303            .saturating_add(committed as u64)
12304            .saturating_add(committed_spans)
12305            .saturating_add(prepared.len() as u64)
12306            .saturating_add(prepared_packages.len() as u64);
12307        let report = PreparedTargetGraphReport {
12308            request_id: request_id.unwrap_or_default(),
12309            requested_targets: targets.len(),
12310            normalized_regions: visited_regions.len(),
12311            normalized_target_list: normalized,
12312            selected_staged_cells,
12313            selected_source_families,
12314            retained_staged_cells: self.staged_formula_count(),
12315            selected_cells,
12316            retained_cells,
12317            widened_scope: scope,
12318            widening_reasons: reasons,
12319            revisions: assumptions,
12320            commit_window,
12321            estimated_scratch_bytes: scratch_bytes,
12322            observed_scratch_bytes,
12323            estimated_commit_work: (new_vertices as u64)
12324                .saturating_add(new_edges as u64)
12325                .saturating_add(prepared.len() as u64)
12326                .saturating_add(selected_package_records),
12327            actual_commit_work,
12328            outcome: PreparationOutcome::Prepared,
12329        };
12330        self.observe_target_preparation_report(&report);
12331        Ok(report)
12332    }
12333
12334    /// Build graph for all staged formulas.
12335    pub fn build_graph_all(&mut self) -> Result<(), formualizer_parse::ExcelError> {
12336        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
12337            engine.build_graph_all_unobserved()
12338        })
12339    }
12340
12341    fn build_graph_all_unobserved(&mut self) -> Result<(), formualizer_parse::ExcelError> {
12342        let selected = self.staged_formula_count();
12343        let started = crate::instant::FzInstant::now();
12344        self.resource_checkpoint(selected as u64)?;
12345        let scratch_bytes = (selected as u64).saturating_mul(256);
12346        let result = self.with_request_scratch(scratch_bytes, |engine| {
12347            let index_snapshot = engine.staged_formula_index.clone();
12348            let collected = std::mem::take(&mut engine.staged_formulas)
12349                .into_iter()
12350                .collect();
12351            engine.staged_formula_index.clear_all();
12352            engine.build_graph_from_staged_batches(collected, false, index_snapshot)
12353        });
12354        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
12355        result
12356    }
12357
12358    /// Build graph for specific sheets (consuming only those staged entries).
12359    pub fn build_graph_for_sheets<'a, I: IntoIterator<Item = &'a str>>(
12360        &mut self,
12361        sheets: I,
12362    ) -> Result<(), formualizer_parse::ExcelError> {
12363        let mut sheets = sheets.into_iter();
12364        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, move |engine| {
12365            let name_scratch = (sheets.size_hint().0 as u64).saturating_mul(64);
12366            engine.with_request_scratch(name_scratch, |engine| {
12367                // Allocation failure follows the baseline process-fatal policy; it is not a
12368                // recoverable resource error or a new staged-preparation route.
12369                let names = sheets.by_ref().map(str::to_string).collect::<Vec<_>>();
12370                engine.charge_bounded_work(names.len() as u64)?;
12371                engine.build_graph_for_sheet_names_unobserved(names)
12372            })
12373        })
12374    }
12375
12376    fn build_graph_for_sheet_names_unobserved(
12377        &mut self,
12378        sheets: Vec<String>,
12379    ) -> Result<(), formualizer_parse::ExcelError> {
12380        let started = crate::instant::FzInstant::now();
12381        let selected = sheets
12382            .iter()
12383            .filter_map(|sheet| self.staged_formulas.get(sheet))
12384            .map(StagedSheet::len)
12385            .sum::<usize>();
12386        self.resource_checkpoint(selected as u64)?;
12387        let scratch_bytes = (selected as u64).saturating_mul(256);
12388        self.reserve_request_scratch(scratch_bytes)?;
12389        let index_snapshot = self.staged_formula_index.clone();
12390        let mut collected = Vec::new();
12391        for sheet in sheets {
12392            if let Some(staged) = self.staged_formulas.remove(&sheet) {
12393                self.index_removed_staged_sheet(&sheet, &staged);
12394                collected.push((sheet, staged));
12395            }
12396        }
12397        let result = self.build_graph_from_staged_batches(collected, true, index_snapshot);
12398        self.release_request_scratch(scratch_bytes);
12399        self.observe_staged_preparation(selected, self.staged_formula_count(), started.elapsed());
12400        result
12401    }
12402
12403    fn build_graph_from_staged_batches(
12404        &mut self,
12405        collected: StagedFormulaBatches,
12406        share_parse_cache_across_sheets: bool,
12407        staged_index_snapshot: StagedFormulaIndex,
12408    ) -> Result<(), formualizer_parse::ExcelError> {
12409        if collected.is_empty() {
12410            return Ok(());
12411        }
12412        for (sheet, _) in &collected {
12413            let _ = self.add_sheet(sheet);
12414        }
12415
12416        let diagnostics_len = self.formula_parse_diagnostics.len();
12417        let mut collected = collected;
12418        let prepared = match self
12419            .prepare_staged_formula_batches(&mut collected, share_parse_cache_across_sheets)
12420        {
12421            Ok(prepared) => prepared,
12422            Err(error) => {
12423                self.formula_parse_diagnostics.truncate(diagnostics_len);
12424                for (sheet, staged) in collected {
12425                    self.restore_staged_sheet(sheet, staged);
12426                }
12427                self.staged_formula_index = staged_index_snapshot;
12428                return Err(error);
12429            }
12430        };
12431        let (ordinary, compressed, direct) = prepared;
12432
12433        // Deferred packages are consumed exactly once once authority commit starts.
12434        // Pre-commit lock, replay, and parse failures restore them above for retry.
12435        if !ordinary.is_empty()
12436            && let Err(error) = self.ingest_formula_batches(ordinary)
12437        {
12438            self.formula_parse_diagnostics.truncate(diagnostics_len);
12439            for (sheet, staged) in collected {
12440                self.restore_staged_sheet(sheet, staged);
12441            }
12442            self.staged_formula_index = staged_index_snapshot;
12443            return Err(error);
12444        }
12445        if !compressed.is_empty() {
12446            let _ = self.ingest_compressed_formula_source_batches(compressed)?;
12447        }
12448        if !direct.is_empty() {
12449            let _ = self.finish_compressed_formula_sources(direct)?;
12450        }
12451        self.dedup_formula_parse_diagnostics_since(diagnostics_len);
12452        Ok(())
12453    }
12454
12455    fn prepare_staged_formula_batches(
12456        &mut self,
12457        collected: &mut StagedFormulaBatches,
12458        share_parse_cache_across_sheets: bool,
12459    ) -> Result<PreparedStagedFormulaBatches, formualizer_parse::ExcelError> {
12460        let mut ordinary = Vec::new();
12461        let mut compressed = Vec::new();
12462        let mut direct = Vec::new();
12463        let mut cache: rustc_hash::FxHashMap<String, Option<crate::engine::arena::AstNodeId>> =
12464            rustc_hash::FxHashMap::default();
12465        cache.reserve(4096);
12466
12467        for (sheet, staged) in collected {
12468            if !share_parse_cache_across_sheets {
12469                cache.clear();
12470            }
12471            let mut entries: Vec<_> = staged
12472                .entries
12473                .iter()
12474                .cloned()
12475                .map(|(row, col, text)| (row, col, text, None))
12476                .collect();
12477            let mut deferred_source = None;
12478            let mut deferred_fallback = None;
12479            if let Some(package) = staged.deferred_package.as_mut() {
12480                if package.sheet_name != *sheet {
12481                    return Err(ExcelError::new(ExcelErrorKind::Value)
12482                        .with_message("deferred formula package sheet mismatch"));
12483                }
12484                let eligible: Vec<_> = package
12485                    .families
12486                    .iter()
12487                    .filter(|family| !package.invalidated.contains(&family.source_id))
12488                    .cloned()
12489                    .collect();
12490                let eligible_partitions: Vec<_> = package
12491                    .partitioned_families
12492                    .iter()
12493                    .filter(|family| !package.invalidated.contains(&family.source_id))
12494                    .cloned()
12495                    .collect();
12496                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
12497                    let mut preparation = self.prepare_source_formula_proposals(
12498                        sheet,
12499                        &eligible,
12500                        &eligible_partitions,
12501                        &package.partitioned_families,
12502                        package.report.source_formula_records_spooled,
12503                        Arc::clone(&package.replay),
12504                        &package.suppressed,
12505                    )?;
12506                    let replay_records = std::mem::take(&mut preparation.eager_replay);
12507                    let (fragment_legacy, ordered_fallback): (Vec<_>, Vec<_>) = {
12508                        let accepted: BTreeMap<_, _> = preparation
12509                            .fragmented
12510                            .iter()
12511                            .map(|fragment| (fragment.source.source_id, &fragment.source))
12512                            .collect();
12513                        replay_records.into_iter().partition(|record| {
12514                            record.partition_owner.is_some_and(|owner| {
12515                                accepted.get(&owner).is_some_and(|source| {
12516                                    Self::exact_replay_record_is_prepared_partition_legacy(
12517                                        source, record,
12518                                    )
12519                                })
12520                            })
12521                        })
12522                    };
12523                    preparation.eager_replay = fragment_legacy;
12524                    entries.extend(ordered_fallback.into_iter().map(|record| {
12525                        (
12526                            record.row,
12527                            record.col,
12528                            record.text,
12529                            Some((record.source_order, record.family, record.partition_owner)),
12530                        )
12531                    }));
12532                    deferred_source = Some((package.report.clone(), preparation));
12533                } else {
12534                    let mut replay_disposition = crate::engine::FormulaReplayDisposition::default();
12535                    for partition in &eligible_partitions {
12536                        replay_disposition
12537                            .register_partition(partition, false)
12538                            .map_err(|reason| {
12539                                ExcelError::new(ExcelErrorKind::Value).with_message(reason)
12540                            })?;
12541                    }
12542                    replay_disposition
12543                        .extend_suppressed_excel_coords(package.suppressed.iter().copied());
12544                    let replayed = package
12545                        .replay
12546                        .lock()
12547                        .map_err(|_| {
12548                            ExcelError::new(ExcelErrorKind::Value)
12549                                .with_message("deferred formula spool lock poisoned")
12550                        })?
12551                        .replay(&replay_disposition)
12552                        .map_err(|message| {
12553                            ExcelError::new(ExcelErrorKind::Value).with_message(message)
12554                        })?;
12555                    entries.extend(replayed.into_iter().map(|record| {
12556                        (
12557                            record.row,
12558                            record.col,
12559                            record.text,
12560                            Some((record.source_order, record.family, record.partition_owner)),
12561                        )
12562                    }));
12563                    let mut report = package.report.clone();
12564                    report.source_spool_replays = report.source_spool_replays.saturating_add(1);
12565                    deferred_fallback =
12566                        Some((report, package.families.clone(), eligible_partitions));
12567                }
12568            }
12569
12570            let mut formulas = Vec::new();
12571            let staged_order_base = u64::MAX.saturating_sub(entries.len() as u64);
12572            for (entry_index, (row, col, txt, source_proof)) in entries.into_iter().enumerate() {
12573                let key = if txt.starts_with('=') {
12574                    txt
12575                } else {
12576                    format!("={txt}")
12577                };
12578                let ast_id = if let Some(cached) = cache.get(&key) {
12579                    *cached
12580                } else {
12581                    let parsed = match formualizer_parse::parser::parse(&key) {
12582                        Ok(parsed) => Some(parsed),
12583                        Err(error) => self.handle_formula_parse_error(
12584                            sheet,
12585                            row,
12586                            col,
12587                            &key,
12588                            error.to_string(),
12589                        )?,
12590                    };
12591                    let ast_id = parsed.as_ref().map(|ast| self.intern_formula_ast(ast));
12592                    cache.insert(key.clone(), ast_id);
12593                    ast_id
12594                };
12595
12596                if let Some(ast_id) = ast_id {
12597                    let mut formula = FormulaIngestRecord::new(
12598                        row,
12599                        col,
12600                        ast_id,
12601                        Some(Arc::<str>::from(key.clone())),
12602                    );
12603                    if let Some((order, family, owner)) = source_proof {
12604                        formula = formula.with_source_proof(order, family, owner);
12605                    } else if deferred_source.is_some() {
12606                        formula = formula.with_source_proof(
12607                            crate::engine::SourceFormulaOrder::new(
12608                                staged_order_base.saturating_add(entry_index as u64),
12609                            ),
12610                            None,
12611                            None,
12612                        );
12613                    }
12614                    formulas.push(formula);
12615                }
12616            }
12617
12618            let batch = FormulaIngestBatch::new(sheet.clone(), formulas);
12619            if let Some((report, preparation)) = deferred_source {
12620                direct.push((batch, report, preparation));
12621            } else if let Some((report, families, partitions)) = deferred_fallback {
12622                let source_batch = crate::engine::FormulaCompressedSourceBatch::with_proposals(
12623                    batch.sheet_name.clone(),
12624                    report,
12625                    families,
12626                    partitions,
12627                );
12628                compressed.push((batch, source_batch));
12629            } else if !batch.is_empty() {
12630                ordinary.push(batch);
12631            }
12632        }
12633        Ok((ordinary, compressed, direct))
12634    }
12635
12636    /// Begin bulk Arrow ingest for base values (Phase A)
12637    pub fn begin_bulk_ingest_arrow(
12638        &mut self,
12639    ) -> crate::engine::arrow_ingest::ArrowBulkIngestBuilder<'_, R> {
12640        crate::engine::arrow_ingest::ArrowBulkIngestBuilder::new(self)
12641    }
12642
12643    /// Begin bulk updates to Arrow store (Phase C)
12644    pub fn begin_bulk_update_arrow(
12645        &mut self,
12646    ) -> crate::engine::arrow_ingest::ArrowBulkUpdateBuilder<'_, R> {
12647        crate::engine::arrow_ingest::ArrowBulkUpdateBuilder::new(self)
12648    }
12649
12650    fn ensure_known_sheet_id(&self, sheet: &str) -> Result<SheetId, crate::engine::EditorError> {
12651        self.graph.sheet_id(sheet).ok_or(
12652            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
12653                name: sheet.to_string(),
12654                reason: "Unknown sheet".to_string(),
12655            },
12656        )
12657    }
12658
12659    fn normalize_row_1based(row_1based: u32) -> Result<u32, crate::engine::EditorError> {
12660        if row_1based == 0 {
12661            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
12662        }
12663        Ok(row_1based - 1)
12664    }
12665
12666    fn normalize_row_range_1based(
12667        start_row_1based: u32,
12668        end_row_1based: u32,
12669    ) -> Result<(u32, u32), crate::engine::EditorError> {
12670        if start_row_1based == 0 || end_row_1based == 0 {
12671            return Err(crate::engine::EditorError::OutOfBounds { row: 0, col: 0 });
12672        }
12673        if start_row_1based > end_row_1based {
12674            return Err(crate::engine::EditorError::TransactionFailed {
12675                reason: "Row range start is greater than end".to_string(),
12676            });
12677        }
12678        Ok((start_row_1based - 1, end_row_1based - 1))
12679    }
12680
12681    fn invalidate_row_visibility_mask_cache(&self) {
12682        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12683            cache.clear();
12684        }
12685    }
12686
12687    fn set_row_hidden_by_sheet_id(
12688        &mut self,
12689        sheet_id: SheetId,
12690        row0: u32,
12691        hidden: bool,
12692        source: RowVisibilitySource,
12693    ) -> bool {
12694        let changed = {
12695            let state = self.row_visibility.entry(sheet_id).or_default();
12696            state.set_row_hidden(row0, hidden, source)
12697        };
12698
12699        let remove_entry = self
12700            .row_visibility
12701            .get(&sheet_id)
12702            .map(|state| state.is_empty())
12703            .unwrap_or(false);
12704        if remove_entry {
12705            self.row_visibility.remove(&sheet_id);
12706        }
12707
12708        if changed {
12709            self.invalidate_row_visibility_mask_cache();
12710        }
12711
12712        changed
12713    }
12714
12715    fn set_rows_hidden_by_sheet_id(
12716        &mut self,
12717        sheet_id: SheetId,
12718        start_row0: u32,
12719        end_row0: u32,
12720        hidden: bool,
12721        source: RowVisibilitySource,
12722    ) -> bool {
12723        let changed = {
12724            let state = self.row_visibility.entry(sheet_id).or_default();
12725            state.set_rows_hidden(start_row0, end_row0, hidden, source)
12726        };
12727
12728        let remove_entry = self
12729            .row_visibility
12730            .get(&sheet_id)
12731            .map(|state| state.is_empty())
12732            .unwrap_or(false);
12733        if remove_entry {
12734            self.row_visibility.remove(&sheet_id);
12735        }
12736
12737        if changed {
12738            self.invalidate_row_visibility_mask_cache();
12739        }
12740
12741        changed
12742    }
12743
12744    fn shift_row_visibility_insert(&mut self, sheet_id: SheetId, before0: u32, count: u32) {
12745        if count == 0 {
12746            return;
12747        }
12748        let mut changed = false;
12749        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12750            changed = state.insert_rows(before0, count);
12751            state.is_empty()
12752        } else {
12753            false
12754        };
12755        if remove_entry {
12756            self.row_visibility.remove(&sheet_id);
12757        }
12758        if changed {
12759            self.invalidate_row_visibility_mask_cache();
12760        }
12761    }
12762
12763    fn shift_row_visibility_delete(&mut self, sheet_id: SheetId, start0: u32, count: u32) {
12764        if count == 0 {
12765            return;
12766        }
12767        let mut changed = false;
12768        let remove_entry = if let Some(state) = self.row_visibility.get_mut(&sheet_id) {
12769            changed = state.delete_rows(start0, count);
12770            state.is_empty()
12771        } else {
12772            false
12773        };
12774        if remove_entry {
12775            self.row_visibility.remove(&sheet_id);
12776        }
12777        if changed {
12778            self.invalidate_row_visibility_mask_cache();
12779        }
12780    }
12781
12782    fn apply_inverse_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12783        if let crate::engine::ChangeEvent::SetRowVisibility {
12784            sheet_id,
12785            row0,
12786            source,
12787            old_hidden,
12788            ..
12789        } = event
12790        {
12791            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *old_hidden, *source);
12792        }
12793    }
12794
12795    fn apply_forward_row_visibility_event(&mut self, event: &crate::engine::ChangeEvent) {
12796        if let crate::engine::ChangeEvent::SetRowVisibility {
12797            sheet_id,
12798            row0,
12799            source,
12800            new_hidden,
12801            ..
12802        } = event
12803        {
12804            let _ = self.set_row_hidden_by_sheet_id(*sheet_id, *row0, *new_hidden, *source);
12805        }
12806    }
12807
12808    fn apply_inverse_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12809        for event in events.iter().rev() {
12810            self.apply_inverse_row_visibility_event(event);
12811        }
12812    }
12813
12814    fn apply_forward_row_visibility_events(&mut self, events: &[crate::engine::ChangeEvent]) {
12815        for event in events {
12816            self.apply_forward_row_visibility_event(event);
12817        }
12818    }
12819
12820    fn apply_inverse_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12821        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12822            sheet,
12823            row,
12824            col,
12825            old,
12826            ..
12827        } = event
12828        {
12829            self.apply_staged_formula_cell(sheet, *row, *col, old.as_deref());
12830        }
12831    }
12832
12833    fn apply_forward_staged_formula_event(&mut self, event: &crate::engine::ChangeEvent) {
12834        if let crate::engine::ChangeEvent::StagedFormulaCellChanged {
12835            sheet,
12836            row,
12837            col,
12838            new,
12839            ..
12840        } = event
12841        {
12842            self.apply_staged_formula_cell(sheet, *row, *col, new.as_deref());
12843        }
12844    }
12845
12846    /// Set a single cell's staged formula text to `target` (clearing it when
12847    /// `None`). Used by undo/redo replay of per-cell staged-formula deltas.
12848    fn apply_staged_formula_cell(&mut self, sheet: &str, row: u32, col: u32, target: Option<&str>) {
12849        match target {
12850            Some(text) => self.stage_formula_text(sheet, row, col, text.to_string()),
12851            None => {
12852                self.clear_staged_formula_text(sheet, row, col);
12853            }
12854        }
12855    }
12856
12857    pub fn set_row_hidden(
12858        &mut self,
12859        sheet: &str,
12860        row_1based: u32,
12861        hidden: bool,
12862        source: RowVisibilitySource,
12863    ) -> Result<(), crate::engine::EditorError> {
12864        self.observe_function_semantic_epoch()
12865            .map_err(crate::engine::EditorError::Excel)?;
12866        self.observe_function_semantic_epoch()
12867            .map_err(crate::engine::EditorError::Excel)?;
12868        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12869        let row0 = Self::normalize_row_1based(row_1based)?;
12870        if self.set_row_hidden_by_sheet_id(sheet_id, row0, hidden, source) {
12871            self.record_formula_plane_structural_change(StructuralScope::Region(
12872                Region::whole_row(sheet_id, row0),
12873            ));
12874            self.mark_data_edited();
12875        }
12876        Ok(())
12877    }
12878
12879    pub fn set_rows_hidden(
12880        &mut self,
12881        sheet: &str,
12882        start_row_1based: u32,
12883        end_row_1based: u32,
12884        hidden: bool,
12885        source: RowVisibilitySource,
12886    ) -> Result<(), crate::engine::EditorError> {
12887        let sheet_id = self.ensure_known_sheet_id(sheet)?;
12888        let (start_row0, end_row0) =
12889            Self::normalize_row_range_1based(start_row_1based, end_row_1based)?;
12890        if self.set_rows_hidden_by_sheet_id(sheet_id, start_row0, end_row0, hidden, source) {
12891            if start_row0 == end_row0 {
12892                self.record_formula_plane_structural_change(StructuralScope::Region(
12893                    Region::whole_row(sheet_id, start_row0),
12894                ));
12895            } else {
12896                self.record_formula_plane_structural_change(StructuralScope::Sheet(sheet_id));
12897            }
12898            self.mark_data_edited();
12899        }
12900        Ok(())
12901    }
12902
12903    pub fn is_row_hidden(
12904        &self,
12905        sheet: &str,
12906        row_1based: u32,
12907        source: Option<RowVisibilitySource>,
12908    ) -> Option<bool> {
12909        let sheet_id = self.graph.sheet_id(sheet)?;
12910        let row0 = row_1based.checked_sub(1)?;
12911        Some(
12912            self.row_visibility
12913                .get(&sheet_id)
12914                .map(|state| state.is_row_hidden(row0, source))
12915                .unwrap_or(false),
12916        )
12917    }
12918
12919    pub fn row_visibility_version(&self, sheet: &str) -> Option<u64> {
12920        let sheet_id = self.graph.sheet_id(sheet)?;
12921        Some(
12922            self.row_visibility
12923                .get(&sheet_id)
12924                .map(|state| state.version())
12925                .unwrap_or(0),
12926        )
12927    }
12928
12929    fn build_row_visibility_mask_for_view(
12930        &self,
12931        view: &RangeView<'_>,
12932        mode: VisibilityMaskMode,
12933    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
12934        let sheet_rows = view.sheet().nrows as usize;
12935        if sheet_rows == 0 || view.start_row() >= sheet_rows {
12936            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
12937        }
12938
12939        let sheet_id = self.graph.sheet_id(view.sheet_name())?;
12940        let start_row0 = view.start_row() as u32;
12941        let end_row0 = view.end_row().min(sheet_rows.saturating_sub(1)) as u32;
12942        let version = self
12943            .row_visibility
12944            .get(&sheet_id)
12945            .map(|state| state.version())
12946            .unwrap_or(0);
12947        let key = VisibilityMaskCacheKey {
12948            sheet_id,
12949            start_row0,
12950            end_row0,
12951            mode,
12952            version,
12953        };
12954
12955        if let Ok(cache) = self.row_visibility_mask_cache.read()
12956            && let Some(mask) = cache.get(&key)
12957        {
12958            #[cfg(test)]
12959            visibility_mask_test_hooks::inc_hit();
12960            return Some(mask.clone());
12961        }
12962
12963        #[cfg(test)]
12964        visibility_mask_test_hooks::inc_miss();
12965
12966        let state = self.row_visibility.get(&sheet_id);
12967        let mut out = Vec::with_capacity((end_row0 - start_row0 + 1) as usize);
12968        for row0 in start_row0..=end_row0 {
12969            let manual_hidden = state
12970                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Manual)))
12971                .unwrap_or(false);
12972            let filter_hidden = state
12973                .map(|s| s.is_row_hidden(row0, Some(RowVisibilitySource::Filter)))
12974                .unwrap_or(false);
12975
12976            let include = match mode {
12977                VisibilityMaskMode::IncludeAll => true,
12978                VisibilityMaskMode::ExcludeManualHidden => !manual_hidden,
12979                VisibilityMaskMode::ExcludeFilterHidden => !filter_hidden,
12980                VisibilityMaskMode::ExcludeManualOrFilterHidden => {
12981                    !(manual_hidden || filter_hidden)
12982                }
12983            };
12984            out.push(include);
12985        }
12986
12987        let mask = std::sync::Arc::new(arrow_array::BooleanArray::from(out));
12988        if let Ok(mut cache) = self.row_visibility_mask_cache.write() {
12989            const MAX_CACHE_ENTRIES: usize = 4096;
12990            if cache.len() >= MAX_CACHE_ENTRIES {
12991                cache.clear();
12992                #[cfg(test)]
12993                visibility_mask_test_hooks::inc_eviction();
12994            }
12995            cache.insert(key, mask.clone());
12996        }
12997
12998        Some(mask)
12999    }
13000
13001    fn editor_error_to_excel(error: crate::engine::EditorError) -> ExcelError {
13002        match error {
13003            crate::engine::EditorError::Excel(error) => error,
13004            other => ExcelError::new(ExcelErrorKind::Value).with_message(other.to_string()),
13005        }
13006    }
13007
13008    fn observe_function_semantic_epoch(&mut self) -> Result<bool, ExcelError> {
13009        let changes =
13010            crate::function_registry::semantic_changes_since(self.function_semantic_epoch_seen);
13011        let global_changed = changes.epoch != self.function_semantic_epoch_seen;
13012        let provider_revision = self.resolver.planning_semantic_revision();
13013        let provider_changed = provider_revision != self.function_provider_revision_seen;
13014        if !global_changed && !provider_changed {
13015            return Ok(false);
13016        }
13017
13018        let changed = changes.keys.into_iter().collect::<BTreeSet<_>>();
13019        let sheets: BTreeSet<_> = self
13020            .graph
13021            .formula_authority()
13022            .plane
13023            .spans
13024            .active_spans()
13025            .filter_map(|span| {
13026                let Some(template) = self
13027                    .graph
13028                    .formula_authority()
13029                    .plane
13030                    .templates
13031                    .get(span.template_id)
13032                else {
13033                    return (provider_changed || global_changed && !changes.complete)
13034                        .then_some(span.domain.sheet_id());
13035                };
13036                let Some(ast) = self
13037                    .graph
13038                    .data_store()
13039                    .reconstruct_ast_node(template.ast_id, self.graph.sheet_reg())
13040                else {
13041                    return (provider_changed || global_changed && !changes.complete)
13042                        .then_some(span.domain.sheet_id());
13043                };
13044                let global_affected = global_changed
13045                    && (!changes.complete || Self::ast_uses_changed_function(&ast, &changed));
13046                let provider_affected = provider_changed && Self::ast_contains_function(&ast);
13047                (global_affected || provider_affected).then_some(span.domain.sheet_id())
13048            })
13049            .collect();
13050        let invalidated = !sheets.is_empty();
13051        for sheet_id in sheets {
13052            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
13053                .map_err(Self::editor_error_to_excel)?;
13054        }
13055        if global_changed && !changed.is_empty() || provider_changed {
13056            self.cached_static_schedule = None;
13057        }
13058        self.function_semantic_epoch_seen = changes.epoch;
13059        self.function_provider_revision_seen = provider_revision;
13060        Ok(invalidated)
13061    }
13062
13063    pub(crate) fn ast_uses_changed_function(
13064        ast: &ASTNode,
13065        changed: &BTreeSet<(String, String)>,
13066    ) -> bool {
13067        match &ast.node_type {
13068            ASTNodeType::Function { name, args } => {
13069                let normalized = name.to_uppercase();
13070                let mut spellings = vec![(String::new(), normalized.clone())];
13071                let mut stripped = normalized.as_str();
13072                loop {
13073                    let Some(rest) = ["_XLFN.", "_XLL.", "_XLWS."]
13074                        .iter()
13075                        .find_map(|prefix| stripped.strip_prefix(prefix))
13076                    else {
13077                        break;
13078                    };
13079                    stripped = rest;
13080                    spellings.push((String::new(), stripped.to_string()));
13081                }
13082                let resolved = crate::function_registry::resolve("", name);
13083                let directly_changed = spellings.iter().any(|spelling| changed.contains(spelling))
13084                    || resolved.as_ref().is_some_and(|resolved| {
13085                        changed.contains(&(
13086                            resolved.namespace.clone(),
13087                            resolved.canonical_name.clone(),
13088                        ))
13089                    });
13090                directly_changed
13091                    || resolved.is_none()
13092                    || args
13093                        .iter()
13094                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
13095            }
13096            ASTNodeType::Call { callee, args } => {
13097                Self::ast_uses_changed_function(callee, changed)
13098                    || args
13099                        .iter()
13100                        .any(|arg| Self::ast_uses_changed_function(arg, changed))
13101            }
13102            ASTNodeType::UnaryOp { expr, .. } => Self::ast_uses_changed_function(expr, changed),
13103            ASTNodeType::BinaryOp { left, right, .. } => {
13104                Self::ast_uses_changed_function(left, changed)
13105                    || Self::ast_uses_changed_function(right, changed)
13106            }
13107            ASTNodeType::Array(rows) => rows
13108                .iter()
13109                .flatten()
13110                .any(|node| Self::ast_uses_changed_function(node, changed)),
13111            _ => false,
13112        }
13113    }
13114
13115    fn ast_contains_function(ast: &ASTNode) -> bool {
13116        match &ast.node_type {
13117            ASTNodeType::Function { .. } => true,
13118            ASTNodeType::Call { callee, args } => {
13119                Self::ast_contains_function(callee) || args.iter().any(Self::ast_contains_function)
13120            }
13121            ASTNodeType::UnaryOp { expr, .. } => Self::ast_contains_function(expr),
13122            ASTNodeType::BinaryOp { left, right, .. } => {
13123                Self::ast_contains_function(left) || Self::ast_contains_function(right)
13124            }
13125            ASTNodeType::Array(rows) => rows.iter().flatten().any(Self::ast_contains_function),
13126            ASTNodeType::Literal(_) | ASTNodeType::Omitted | ASTNodeType::Reference { .. } => false,
13127        }
13128    }
13129
13130    fn demote_span_containing_cell_for_write(
13131        &mut self,
13132        sheet_id: SheetId,
13133        row0: u32,
13134        col0: u32,
13135    ) -> Result<(), crate::engine::EditorError> {
13136        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
13137            return Ok(());
13138        }
13139        let placement = PlacementCoord::new(sheet_id, row0, col0);
13140        let inside_active_span = self
13141            .graph
13142            .formula_authority()
13143            .plane
13144            .spans
13145            .find_at(placement)
13146            .is_some();
13147        if inside_active_span {
13148            self.demote_spans_preserving_computed_overlays(
13149                sheet_id,
13150                Region::point(sheet_id, row0, col0),
13151            )?;
13152        }
13153        Ok(())
13154    }
13155
13156    fn demote_spans_preserving_computed_overlays(
13157        &mut self,
13158        _sheet_id: SheetId,
13159        affected_region: Region,
13160    ) -> Result<(), crate::engine::EditorError> {
13161        // Per-cell write inside a span (or whole-sheet demote via remove_sheet):
13162        // not a structural axis shift. Demote every span whose result or read
13163        // region intersects `affected_region`; leave disjoint spans untouched.
13164        self.demote_spans_for_structural_op_impl(None, affected_region, false)
13165    }
13166
13167    fn structural_row_region(sheet_id: SheetId, start_row0: u32) -> Region {
13168        Region::rows_from(sheet_id, start_row0)
13169    }
13170
13171    fn structural_col_region(sheet_id: SheetId, start_col0: u32) -> Region {
13172        Region::cols_from(sheet_id, start_col0)
13173    }
13174
13175    fn structural_dirty_region_for_domain(
13176        domain: &PlacementDomain,
13177        op: StructuralOp,
13178    ) -> Option<Region> {
13179        let (sheet_id, row_start, row_end, col_start, col_end) = match domain {
13180            PlacementDomain::RowRun {
13181                sheet_id,
13182                row_start,
13183                row_end,
13184                col,
13185            } => (*sheet_id, *row_start, *row_end, *col, *col),
13186            PlacementDomain::ColRun {
13187                sheet_id,
13188                row,
13189                col_start,
13190                col_end,
13191            } => (*sheet_id, *row, *row, *col_start, *col_end),
13192            PlacementDomain::Rect {
13193                sheet_id,
13194                row_start,
13195                row_end,
13196                col_start,
13197                col_end,
13198            } => (*sheet_id, *row_start, *row_end, *col_start, *col_end),
13199        };
13200        match op {
13201            StructuralOp::InsertRows {
13202                sheet_id: edited,
13203                before,
13204                ..
13205            } if sheet_id == edited && row_end >= before => Some(Region::rect(
13206                sheet_id,
13207                row_start.max(before),
13208                row_end,
13209                col_start,
13210                col_end,
13211            )),
13212            StructuralOp::DeleteRows {
13213                sheet_id: edited,
13214                start,
13215                ..
13216            } if sheet_id == edited && row_end >= start => Some(Region::rect(
13217                sheet_id,
13218                row_start.max(start),
13219                row_end,
13220                col_start,
13221                col_end,
13222            )),
13223            StructuralOp::InsertColumns {
13224                sheet_id: edited,
13225                before,
13226                ..
13227            } if sheet_id == edited && col_end >= before => Some(Region::rect(
13228                sheet_id,
13229                row_start,
13230                row_end,
13231                col_start.max(before),
13232                col_end,
13233            )),
13234            StructuralOp::DeleteColumns {
13235                sheet_id: edited,
13236                start,
13237                ..
13238            } if sheet_id == edited && col_end >= start => Some(Region::rect(
13239                sheet_id,
13240                row_start,
13241                row_end,
13242                col_start.max(start),
13243                col_end,
13244            )),
13245            _ => None,
13246        }
13247    }
13248
13249    fn span_result_region_intersects_affected(
13250        span: &crate::formula_plane::runtime::FormulaSpan,
13251        affected_region: &Region,
13252    ) -> bool {
13253        Region::from_domain(span.result_region.domain()).intersects(affected_region)
13254    }
13255
13256    fn span_any_read_region_intersects_affected(
13257        plane: &FormulaPlane,
13258        span: &crate::formula_plane::runtime::FormulaSpan,
13259        affected_region: &Region,
13260    ) -> bool {
13261        span.read_summary_id
13262            .and_then(|read_summary_id| plane.span_read_summaries.get(read_summary_id))
13263            .is_some_and(|summary| {
13264                summary
13265                    .dependencies
13266                    .iter()
13267                    .any(|dependency| dependency.read_region.intersects(affected_region))
13268            })
13269    }
13270
13271    fn insert_formula_plane_dirty_coords_for_span(
13272        &self,
13273        span_ref: FormulaSpanRef,
13274        dirty: ProducerDirtyDomain,
13275        out: &mut FxHashSet<(SheetId, u32, u32)>,
13276    ) -> Result<(), crate::engine::EditorError> {
13277        let authority = self.graph.formula_authority();
13278        let span = authority.plane.spans.get(span_ref).ok_or_else(|| {
13279            ExcelError::new(ExcelErrorKind::NImpl)
13280                .with_message("FormulaPlane dirty transfer referenced a stale span")
13281        })?;
13282        match dirty {
13283            ProducerDirtyDomain::Whole => {
13284                out.extend(
13285                    span.domain
13286                        .iter()
13287                        .map(|coord| (coord.sheet_id, coord.row, coord.col)),
13288                );
13289            }
13290            ProducerDirtyDomain::Cells(cells) => {
13291                out.extend(cells.into_iter().filter_map(|key| {
13292                    let coord = PlacementCoord::new(key.sheet_id, key.row, key.col);
13293                    span.domain
13294                        .contains(coord)
13295                        .then_some((coord.sheet_id, coord.row, coord.col))
13296                }));
13297            }
13298            ProducerDirtyDomain::Regions(regions) => {
13299                out.extend(span.domain.iter().filter_map(|coord| {
13300                    let key = crate::formula_plane::region_index::RegionKey::from(coord);
13301                    regions
13302                        .iter()
13303                        .any(|region| region.contains_key(key))
13304                        .then_some((coord.sheet_id, coord.row, coord.col))
13305                }));
13306            }
13307        }
13308        Ok(())
13309    }
13310
13311    fn compute_current_formula_plane_dirty_result_coords(
13312        &self,
13313    ) -> Result<FxHashSet<(SheetId, u32, u32)>, crate::engine::EditorError> {
13314        use crate::formula_plane::producer::compute_dirty_closure;
13315
13316        let authority = self.graph.formula_authority();
13317        let span_refs = authority.active_span_refs();
13318        let span_refs_by_id = span_refs
13319            .iter()
13320            .copied()
13321            .map(|span_ref| (span_ref.id, span_ref))
13322            .collect::<BTreeMap<_, _>>();
13323        let mut dirty_coords = FxHashSet::default();
13324
13325        for span_ref in self.graph.pending_formula_dirty_whole_spans() {
13326            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
13327                self.insert_formula_plane_dirty_coords_for_span(
13328                    span_ref,
13329                    ProducerDirtyDomain::Whole,
13330                    &mut dirty_coords,
13331                )?;
13332            }
13333        }
13334        for (span_ref, region) in self.graph.pending_formula_dirty_span_regions() {
13335            if span_refs_by_id.get(&span_ref.id) == Some(&span_ref) {
13336                self.insert_formula_plane_dirty_coords_for_span(
13337                    span_ref,
13338                    ProducerDirtyDomain::Regions(vec![region]),
13339                    &mut dirty_coords,
13340                )?;
13341            }
13342        }
13343
13344        let pending_changed_regions = self
13345            .graph
13346            .pending_formula_dirty_regions()
13347            .collect::<Vec<_>>();
13348        if pending_changed_regions.is_empty() {
13349            return Ok(dirty_coords);
13350        }
13351
13352        let closure = compute_dirty_closure(
13353            &authority.consumer_reads,
13354            pending_changed_regions,
13355            |producer| authority.producer_results.producer_result_region(producer),
13356        );
13357        if closure.incomplete {
13358            for span_ref in span_refs {
13359                self.insert_formula_plane_dirty_coords_for_span(
13360                    span_ref,
13361                    ProducerDirtyDomain::Whole,
13362                    &mut dirty_coords,
13363                )?;
13364            }
13365            return Ok(dirty_coords);
13366        }
13367        for work in closure.work {
13368            let FormulaProducerId::Span(span_id) = work.producer else {
13369                continue;
13370            };
13371            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
13372                continue;
13373            };
13374            self.insert_formula_plane_dirty_coords_for_span(
13375                span_ref,
13376                work.dirty,
13377                &mut dirty_coords,
13378            )?;
13379        }
13380        for fallback in closure.fallbacks {
13381            let FormulaProducerId::Span(span_id) = fallback.consumer else {
13382                continue;
13383            };
13384            let Some(span_ref) = span_refs_by_id.get(&span_id).copied() else {
13385                continue;
13386            };
13387            self.insert_formula_plane_dirty_coords_for_span(
13388                span_ref,
13389                ProducerDirtyDomain::Whole,
13390                &mut dirty_coords,
13391            )?;
13392        }
13393
13394        Ok(dirty_coords)
13395    }
13396
13397    /// Demote active FormulaPlane spans affected by a structural edit on `sheet_id`.
13398    ///
13399    /// This is the conservative Option-A correctness path for structural edits: rather than
13400    /// attempting to transform FormulaPlane span domains/templates/indexes, materialize each span
13401    /// placement as an ordinary legacy graph formula at its current coordinate, remove the span,
13402    /// and let the existing VertexEditor structural machinery shift/delete those vertices and
13403    /// adjust their ASTs.  Spans whose formula domain is on `sheet_id` are affected directly; spans
13404    /// on other sheets are also affected when one of their retained read regions targets
13405    /// `sheet_id`, because those read-region coordinates become stale after row/column shifts.
13406    fn demote_spans_for_structural_op(
13407        &mut self,
13408        op: StructuralOp,
13409        affected_region: Region,
13410    ) -> Result<(), crate::engine::EditorError> {
13411        if op.count() == 0 {
13412            return Ok(());
13413        }
13414        // #171: after an axis edit, retained read-summary relocation cannot be
13415        // used as a proof of disconnection. Fail closed for this engine.
13416        self.legacy_island_structural_summaries_trusted = false;
13417        self.demote_spans_for_structural_op_impl(Some(op), affected_region, true)
13418    }
13419
13420    fn indexed_structural_candidate_span_refs(
13421        &self,
13422        affected_region: Region,
13423    ) -> Result<Vec<FormulaSpanRef>, crate::engine::EditorError> {
13424        use crate::formula_plane::region_index::BoundedRegionQueryResult;
13425
13426        let authority = self.graph.formula_authority();
13427        if authority.indexed_plane_epoch() != authority.plane.epoch().0 {
13428            return Err(ExcelError::new(ExcelErrorKind::NImpl)
13429                .with_message("FormulaPlane structural candidate indexes are stale")
13430                .into());
13431        }
13432        let indexed_region = match affected_region.axis_ranges() {
13433            (
13434                crate::formula_plane::region_index::AxisRange::From(start),
13435                crate::formula_plane::region_index::AxisRange::All,
13436            ) => Region {
13437                sheet_id: affected_region.sheet_id(),
13438                rows: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
13439                cols: crate::formula_plane::region_index::AxisRange::All,
13440            },
13441            (
13442                crate::formula_plane::region_index::AxisRange::All,
13443                crate::formula_plane::region_index::AxisRange::From(start),
13444            ) => Region {
13445                sheet_id: affected_region.sheet_id(),
13446                rows: crate::formula_plane::region_index::AxisRange::All,
13447                cols: crate::formula_plane::region_index::AxisRange::Span(start, u32::MAX),
13448            },
13449            _ => affected_region,
13450        };
13451        let max_candidates = self.config.max_formula_plane_cache_candidates;
13452        let result_query = match authority
13453            .producer_results
13454            .query_bounded(indexed_region, max_candidates)
13455        {
13456            BoundedRegionQueryResult::Complete(result) => result,
13457            BoundedRegionQueryResult::Incomplete {
13458                observed_candidates,
13459            } => {
13460                return Err(ExcelError::new(ExcelErrorKind::NImpl)
13461                    .with_message(format!(
13462                        "FormulaPlane structural result candidate limit exceeded: observed {observed_candidates}, limit {max_candidates}"
13463                    ))
13464                    .into());
13465            }
13466        };
13467        let remaining = max_candidates.saturating_sub(result_query.matches.len());
13468        let read_query = match authority
13469            .consumer_reads
13470            .query_changed_region_bounded(indexed_region, remaining)
13471        {
13472            BoundedRegionQueryResult::Complete(result) => result,
13473            BoundedRegionQueryResult::Incomplete { .. } => {
13474                return Err(ExcelError::new(ExcelErrorKind::NImpl)
13475                    .with_message("FormulaPlane structural read candidate limit exceeded")
13476                    .into());
13477            }
13478        };
13479
13480        let mut ids = BTreeSet::new();
13481        for matched in result_query.matches {
13482            if let FormulaProducerId::Span(id) = matched.value.producer {
13483                ids.insert(id);
13484            }
13485        }
13486        for matched in read_query.matches {
13487            if let FormulaProducerId::Span(id) = matched.value.consumer {
13488                ids.insert(id);
13489            }
13490        }
13491
13492        ids.into_iter()
13493            .map(|id| {
13494                authority.plane.spans.current_ref(id).ok_or_else(|| {
13495                    crate::engine::EditorError::Excel(
13496                        ExcelError::new(ExcelErrorKind::NImpl)
13497                            .with_message("FormulaPlane structural index referenced a stale span"),
13498                    )
13499                })
13500            })
13501            .collect()
13502    }
13503
13504    fn demote_spans_for_structural_op_impl(
13505        &mut self,
13506        op: Option<StructuralOp>,
13507        affected_region: Region,
13508        clear_computed_overlays: bool,
13509    ) -> Result<(), crate::engine::EditorError> {
13510        struct SpanPlan {
13511            span_ref: FormulaSpanRef,
13512            sheet_id: SheetId,
13513            ast: ASTNode,
13514            origin_row: u32,
13515            origin_col: u32,
13516            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13517            placements: Vec<(u32, u32)>,
13518        }
13519
13520        fn substitute_literal_slots_for_template_placement(
13521            ast: &ASTNode,
13522            binding: &[LiteralValue],
13523        ) -> ASTNode {
13524            fn clone_with_slots(
13525                ast: &ASTNode,
13526                binding: &[LiteralValue],
13527                next: &mut usize,
13528                in_array: bool,
13529            ) -> ASTNode {
13530                let node_type = match &ast.node_type {
13531                    ASTNodeType::Literal(_) if !in_array => {
13532                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
13533                        *next = next.saturating_add(1);
13534                        ASTNodeType::Literal(value)
13535                    }
13536                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
13537                    ASTNodeType::Omitted => ASTNodeType::Omitted,
13538                    ASTNodeType::Reference {
13539                        original,
13540                        reference,
13541                    } => ASTNodeType::Reference {
13542                        original: original.clone(),
13543                        reference: reference.clone(),
13544                    },
13545                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
13546                        op: op.clone(),
13547                        expr: Box::new(clone_with_slots(expr, binding, next, in_array)),
13548                    },
13549                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
13550                        op: op.clone(),
13551                        left: Box::new(clone_with_slots(left, binding, next, in_array)),
13552                        right: Box::new(clone_with_slots(right, binding, next, in_array)),
13553                    },
13554                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
13555                        name: name.clone(),
13556                        args: args
13557                            .iter()
13558                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
13559                            .collect(),
13560                    },
13561                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
13562                        callee: Box::new(clone_with_slots(callee, binding, next, in_array)),
13563                        args: args
13564                            .iter()
13565                            .map(|arg| clone_with_slots(arg, binding, next, in_array))
13566                            .collect(),
13567                    },
13568                    ASTNodeType::Array(rows) => ASTNodeType::Array(
13569                        rows.iter()
13570                            .map(|row| {
13571                                row.iter()
13572                                    .map(|cell| clone_with_slots(cell, binding, next, true))
13573                                    .collect()
13574                            })
13575                            .collect(),
13576                    ),
13577                };
13578                ASTNode::new(node_type, ast.source_token.clone())
13579            }
13580            let mut next = 0usize;
13581            clone_with_slots(ast, binding, &mut next, false)
13582        }
13583
13584        let span_refs = self.indexed_structural_candidate_span_refs(affected_region)?;
13585        self.formula_plane_structural_span_candidates = self
13586            .formula_plane_structural_span_candidates
13587            .saturating_add(span_refs.len() as u64);
13588        if span_refs.is_empty() {
13589            return Ok(());
13590        }
13591        let dirty_span_coords = if clear_computed_overlays {
13592            FxHashSet::default()
13593        } else {
13594            self.compute_current_formula_plane_dirty_result_coords()?
13595        };
13596
13597        /// A template AST rewrite accompanying a span shift: the op displaced
13598        /// the target of an absolute read, which origin relocation cannot
13599        /// repoint (issue #168). The adjusted AST is re-interned as a fresh
13600        /// arena AST + template record at the apply site; keys are re-derived
13601        /// from the rewritten AST so the new record can never collide with
13602        /// the (immutable, possibly shared) pre-rewrite template.
13603        struct SpanTemplateRewrite {
13604            adjusted_ast: ASTNode,
13605            exact_canonical_key: Arc<str>,
13606            parameterized_canonical_key: Arc<str>,
13607            formula_text: Option<Arc<str>>,
13608            /// Canonical form of the rewritten AST at the new origin, used to
13609            /// rebuild the binding set's template-derived slot data (the
13610            /// literal slot map is keyed by arena node ids of the template
13611            /// AST; value-ref slot patterns carry canonical reference
13612            /// coordinates — both go stale across a rewrite).
13613            canonical_expr: crate::formula_plane::template_canonical::CanonicalExpr,
13614            value_ref_slots: Arc<[crate::formula_plane::runtime::ValueRefSlotDescriptor]>,
13615        }
13616
13617        struct ShiftPlan {
13618            span_ref: FormulaSpanRef,
13619            template_id: crate::formula_plane::ids::FormulaTemplateId,
13620            new_origin_row: u32,
13621            new_origin_col: u32,
13622            new_domain: crate::formula_plane::runtime::PlacementDomain,
13623            new_read_summary: Option<SpanReadSummary>,
13624            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13625            force_binding_residual_axes: bool,
13626            rewrite: Option<SpanTemplateRewrite>,
13627            dirty_region: Option<Region>,
13628        }
13629
13630        /// StructuralOp and ShiftOperation are field-for-field identical
13631        /// (both 0-based); the adjuster type is the legacy shared one.
13632        fn shift_operation_for(
13633            op: StructuralOp,
13634        ) -> crate::engine::graph::editor::reference_adjuster::ShiftOperation {
13635            use crate::engine::graph::editor::reference_adjuster::ShiftOperation;
13636            match op {
13637                StructuralOp::InsertRows {
13638                    sheet_id,
13639                    before,
13640                    count,
13641                } => ShiftOperation::InsertRows {
13642                    sheet_id,
13643                    before,
13644                    count,
13645                },
13646                StructuralOp::DeleteRows {
13647                    sheet_id,
13648                    start,
13649                    count,
13650                } => ShiftOperation::DeleteRows {
13651                    sheet_id,
13652                    start,
13653                    count,
13654                },
13655                StructuralOp::InsertColumns {
13656                    sheet_id,
13657                    before,
13658                    count,
13659                } => ShiftOperation::InsertColumns {
13660                    sheet_id,
13661                    before,
13662                    count,
13663                },
13664                StructuralOp::DeleteColumns {
13665                    sheet_id,
13666                    start,
13667                    count,
13668                } => ShiftOperation::DeleteColumns {
13669                    sheet_id,
13670                    start,
13671                    count,
13672                },
13673            }
13674        }
13675
13676        /// Split a straddled span into an unshifted upper half (which keeps the
13677        /// span id via `replace_span_geometry`) and a shifted lower half (a
13678        /// fresh span). Domains and origins here are POST-insert for the lower
13679        /// half, mirroring `ShiftPlan`.
13680        struct SplitPlan {
13681            span_ref: FormulaSpanRef,
13682            sheet_id: SheetId,
13683            template_id: crate::formula_plane::ids::FormulaTemplateId,
13684            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
13685            is_constant_result: bool,
13686            upper_domain: crate::formula_plane::runtime::PlacementDomain,
13687            upper_read_summary: Option<SpanReadSummary>,
13688            lower_new_origin_row: u32,
13689            lower_new_origin_col: u32,
13690            lower_new_domain: crate::formula_plane::runtime::PlacementDomain,
13691            lower_new_read_summary: Option<SpanReadSummary>,
13692            lower_force_binding_residual_axes: bool,
13693            lower_overlay_refs: Vec<crate::formula_plane::runtime::FormulaOverlayRef>,
13694            upper_dirty_region: Option<Region>,
13695            lower_dirty_region: Option<Region>,
13696        }
13697
13698        fn checked_shift_u32(value: u32, delta: i64) -> Option<u32> {
13699            u32::try_from(i64::from(value).checked_add(delta)?).ok()
13700        }
13701
13702        /// Structural transform of a read summary for a NO-REWRITE shift.
13703        /// Regions move with the op; projection rules keep their absolute
13704        /// bounds (the AST is untouched) but must shift their RELATIVE
13705        /// bounds by `-origin_delta`: rules store placement-relative
13706        /// offsets, which equal `authored_coordinate - template_origin`,
13707        /// and a moved origin over an unchanged AST changes that difference
13708        /// (otherwise incremental dirty projection silently goes stale —
13709        /// issue #168 family). Rewrite plans do NOT use this transform;
13710        /// their summaries are re-derived from the rewritten canonical
13711        /// template instead.
13712        fn shifted_read_summary(
13713            read_summary: &SpanReadSummary,
13714            new_result_region: Region,
13715            op: StructuralOp,
13716            row_delta: i64,
13717            col_delta: i64,
13718            origin_row_delta: i64,
13719            origin_col_delta: i64,
13720        ) -> Option<SpanReadSummary> {
13721            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13722            for dependency in &read_summary.dependencies {
13723                let read_region = match op.classify_region(dependency.read_region) {
13724                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13725                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13726                        dependency.read_region
13727                    }
13728                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13729                        ..
13730                    } => dependency
13731                        .read_region
13732                        .project_through_axis_shift(row_delta, col_delta)?,
13733                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles
13734                    | crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13735                        return None;
13736                    }
13737                };
13738                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13739                    read_region,
13740                    projection: dependency
13741                        .projection
13742                        .with_relative_offsets_shifted(-origin_row_delta, -origin_col_delta),
13743                });
13744            }
13745            Some(SpanReadSummary {
13746                result_region: new_result_region,
13747                dependencies,
13748            })
13749        }
13750
13751        fn compact_axis_through_delete(
13752            min: u32,
13753            max: u32,
13754            start: u32,
13755            count: u32,
13756        ) -> Option<(u32, u32)> {
13757            let end = start.saturating_add(count);
13758            if max < start || min >= end {
13759                return Some((min.saturating_sub(count), max.saturating_sub(count)));
13760            }
13761            let keeps_left = min < start;
13762            let keeps_right = max >= end;
13763            match (keeps_left, keeps_right) {
13764                (false, false) => None,
13765                (true, false) => Some((min, start.checked_sub(1)?)),
13766                (false, true) => Some((start, max.checked_sub(count)?)),
13767                (true, true) => Some((min, max.checked_sub(count)?)),
13768            }
13769        }
13770
13771        fn compact_domain_through_delete(
13772            domain: &PlacementDomain,
13773            op: StructuralOp,
13774        ) -> Option<PlacementDomain> {
13775            match (domain, op) {
13776                (
13777                    PlacementDomain::RowRun {
13778                        sheet_id,
13779                        row_start,
13780                        row_end,
13781                        col,
13782                    },
13783                    StructuralOp::DeleteRows { start, count, .. },
13784                ) => {
13785                    let (row_start, row_end) =
13786                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13787                    Some(PlacementDomain::row_run(
13788                        *sheet_id, row_start, row_end, *col,
13789                    ))
13790                }
13791                (
13792                    PlacementDomain::Rect {
13793                        sheet_id,
13794                        row_start,
13795                        row_end,
13796                        col_start,
13797                        col_end,
13798                    },
13799                    StructuralOp::DeleteRows { start, count, .. },
13800                ) => {
13801                    let (row_start, row_end) =
13802                        compact_axis_through_delete(*row_start, *row_end, start, count)?;
13803                    Some(PlacementDomain::rect(
13804                        *sheet_id, row_start, row_end, *col_start, *col_end,
13805                    ))
13806                }
13807                (
13808                    PlacementDomain::ColRun {
13809                        sheet_id,
13810                        row,
13811                        col_start,
13812                        col_end,
13813                    },
13814                    StructuralOp::DeleteColumns { start, count, .. },
13815                ) => {
13816                    let (col_start, col_end) =
13817                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13818                    Some(PlacementDomain::col_run(
13819                        *sheet_id, *row, col_start, col_end,
13820                    ))
13821                }
13822                (
13823                    PlacementDomain::Rect {
13824                        sheet_id,
13825                        row_start,
13826                        row_end,
13827                        col_start,
13828                        col_end,
13829                    },
13830                    StructuralOp::DeleteColumns { start, count, .. },
13831                ) => {
13832                    let (col_start, col_end) =
13833                        compact_axis_through_delete(*col_start, *col_end, start, count)?;
13834                    Some(PlacementDomain::rect(
13835                        *sheet_id, *row_start, *row_end, col_start, col_end,
13836                    ))
13837                }
13838                _ => None,
13839            }
13840        }
13841
13842        fn compact_axis_range_through_delete(
13843            axis: crate::formula_plane::region_index::AxisRange,
13844            start: u32,
13845            count: u32,
13846        ) -> Option<crate::formula_plane::region_index::AxisRange> {
13847            use crate::formula_plane::region_index::AxisRange;
13848            match axis {
13849                AxisRange::Point(point) => compact_axis_through_delete(point, point, start, count)
13850                    .map(|(point, _)| AxisRange::Point(point)),
13851                AxisRange::Span(min, max) => compact_axis_through_delete(min, max, start, count)
13852                    .map(|(min, max)| AxisRange::Span(min, max)),
13853                AxisRange::All => Some(AxisRange::All),
13854                AxisRange::From(_) | AxisRange::To(_) => None,
13855            }
13856        }
13857
13858        fn compact_region_through_delete(region: Region, op: StructuralOp) -> Option<Region> {
13859            let (rows, cols) = region.axis_ranges();
13860            match op {
13861                StructuralOp::DeleteRows {
13862                    sheet_id,
13863                    start,
13864                    count,
13865                } if region.sheet_id() == sheet_id => Some(Region {
13866                    sheet_id,
13867                    rows: compact_axis_range_through_delete(rows, start, count)?,
13868                    cols,
13869                }),
13870                StructuralOp::DeleteColumns {
13871                    sheet_id,
13872                    start,
13873                    count,
13874                } if region.sheet_id() == sheet_id => Some(Region {
13875                    sheet_id,
13876                    rows,
13877                    cols: compact_axis_range_through_delete(cols, start, count)?,
13878                }),
13879                _ => Some(region),
13880            }
13881        }
13882
13883        fn compact_read_summary_through_delete(
13884            read_summary: &SpanReadSummary,
13885            new_result_region: Region,
13886            op: StructuralOp,
13887        ) -> Option<SpanReadSummary> {
13888            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13889            for dependency in &read_summary.dependencies {
13890                let read_region = match op.classify_region(dependency.read_region) {
13891                    crate::formula_plane::structural_shift::AxisShiftCase::OtherSheet
13892                    | crate::formula_plane::structural_shift::AxisShiftCase::EntirelyBelow => {
13893                        dependency.read_region
13894                    }
13895                    crate::formula_plane::structural_shift::AxisShiftCase::EntirelyAboveShift {
13896                        ..
13897                    } => {
13898                        let (row_delta, col_delta) = op.axis_shift_delta();
13899                        dependency
13900                            .read_region
13901                            .project_through_axis_shift(row_delta, col_delta)?
13902                    }
13903                    crate::formula_plane::structural_shift::AxisShiftCase::Straddles => {
13904                        compact_region_through_delete(dependency.read_region, op)?
13905                    }
13906                    crate::formula_plane::structural_shift::AxisShiftCase::DeleteFullyContains => {
13907                        return None;
13908                    }
13909                };
13910                dependencies.push(crate::formula_plane::producer::SpanReadDependency {
13911                    read_region,
13912                    projection: dependency.projection,
13913                });
13914            }
13915            Some(SpanReadSummary {
13916                result_region: new_result_region,
13917                dependencies,
13918            })
13919        }
13920
13921        /// Re-derive a span read summary for a sub-domain of the original
13922        /// result region by replaying each retained projection rule against
13923        /// the half's result region — the same derivation ingest performs in
13924        /// `SpanReadSummary::from_formula_summary`, minus the dependency
13925        /// re-analysis. Returns None (caller demotes) when any rule cannot
13926        /// produce explicit read regions.
13927        fn rederive_read_summary_for_result(
13928            read_summary: &SpanReadSummary,
13929            result_region: Region,
13930        ) -> Option<SpanReadSummary> {
13931            let mut dependencies = Vec::with_capacity(read_summary.dependencies.len());
13932            for dependency in &read_summary.dependencies {
13933                let sheet_id = dependency.read_region.sheet_id();
13934                let read_regions = dependency
13935                    .projection
13936                    .read_regions_for_result(sheet_id, result_region)
13937                    .ok()?;
13938                for read_region in read_regions {
13939                    let dependency = crate::formula_plane::producer::SpanReadDependency {
13940                        read_region,
13941                        projection: dependency.projection,
13942                    };
13943                    if !dependencies.contains(&dependency) {
13944                        dependencies.push(dependency);
13945                    }
13946                }
13947            }
13948            Some(SpanReadSummary {
13949                result_region,
13950                dependencies,
13951            })
13952        }
13953
13954        /// Plan a conservative span split at a mid-domain op boundary: an
13955        /// insert that straddles the domain, or a delete that straddles it
13956        /// with placements surviving on both sides of the deleted band and no
13957        /// single relative-offset frame covering them (issue #171). Returns
13958        /// None whenever the split is not provably clean — missing template,
13959        /// masked span, non-trivial literal bindings, an unsplittable domain,
13960        /// a half that does not re-classify to NoOp (upper) / Shift (lower),
13961        /// or an overlay punch-out that straddles the boundary — in which case
13962        /// the caller falls back to demoting the whole span.
13963        fn plan_span_split(
13964            authority: &crate::formula_plane::authority::FormulaAuthority,
13965            span: &crate::formula_plane::runtime::FormulaSpan,
13966            span_ref: FormulaSpanRef,
13967            read_summary: Option<&SpanReadSummary>,
13968            op: StructuralOp,
13969        ) -> Option<SplitPlan> {
13970            use crate::formula_plane::structural_shift::{AxisShiftCase, split_domain_at};
13971
13972            if span.intrinsic_mask_id.is_some() {
13973                // v1 does not slice intrinsic masks.
13974                return None;
13975            }
13976            let binding_set = span
13977                .binding_set_id
13978                .and_then(|id| authority.plane.binding_sets.get(id));
13979            if binding_set.is_some_and(|binding_set| !binding_set.is_single_literal_binding()) {
13980                // Dictionary/affine per-placement literal bindings are keyed by
13981                // domain ordinal; splitting re-bases the lower half's ordinals.
13982                // Only trivially-uniform bindings are safe to carry across.
13983                return None;
13984            }
13985
13986            let (upper_domain, lower_domain) = split_domain_at(&span.domain, op)?;
13987            let upper_result_region = Region::from_domain(&upper_domain);
13988            let lower_result_region = Region::from_domain(&lower_domain);
13989            let (upper_read_summary, lower_read_summary) = match read_summary {
13990                Some(summary) => (
13991                    Some(rederive_read_summary_for_result(
13992                        summary,
13993                        upper_result_region,
13994                    )?),
13995                    Some(rederive_read_summary_for_result(
13996                        summary,
13997                        lower_result_region,
13998                    )?),
13999                ),
14000                None => (None, None),
14001            };
14002
14003            // Re-classify each half in the pre-insert frame. The upper half
14004            // must be a strict no-op (unshifted result, unshifted reads); the
14005            // lower half must be a clean whole-span shift.
14006            let upper_span = crate::formula_plane::runtime::FormulaSpan {
14007                domain: upper_domain.clone(),
14008                result_region: ResultRegion::scalar_cells(upper_domain.clone()),
14009                ..span.clone()
14010            };
14011            if classify_span_for_op(&upper_span, upper_read_summary.as_ref(), op)
14012                != SpanShiftPlan::NoOp
14013            {
14014                return None;
14015            }
14016            let lower_span = crate::formula_plane::runtime::FormulaSpan {
14017                domain: lower_domain.clone(),
14018                result_region: ResultRegion::scalar_cells(lower_domain.clone()),
14019                ..span.clone()
14020            };
14021            let SpanShiftPlan::Shift {
14022                row_delta,
14023                col_delta,
14024                origin_row_delta,
14025                origin_col_delta,
14026                rewrite_absolute_reads,
14027            } = classify_span_for_op(&lower_span, lower_read_summary.as_ref(), op)
14028            else {
14029                return None;
14030            };
14031            if rewrite_absolute_reads {
14032                // A displaced absolute read in the lower half implies the
14033                // upper half reads it too and cannot be a NoOp, so this is
14034                // unreachable when the upper check above passed; keep the
14035                // guard so a future classifier change cannot silently skip
14036                // the template rewrite.
14037                return None;
14038            }
14039
14040            // Span relocation is the ownership boundary for demotion and
14041            // formula lookup. Canonically shared templates may have a different
14042            // authoring origin, so never derive split provenance from them.
14043            let lower_new_origin_row =
14044                checked_shift_u32(span.ast_relocation.anchor_row, origin_row_delta)?;
14045            let lower_new_origin_col =
14046                checked_shift_u32(span.ast_relocation.anchor_col, origin_col_delta)?;
14047            let lower_new_domain = lower_domain.project_through_axis_shift(row_delta, col_delta)?;
14048            let lower_new_result_region = Region::from_domain(&lower_new_domain);
14049            let lower_new_read_summary = match lower_read_summary.as_ref() {
14050                Some(summary) => Some(shifted_read_summary(
14051                    summary,
14052                    lower_new_result_region,
14053                    op,
14054                    row_delta,
14055                    col_delta,
14056                    origin_row_delta,
14057                    origin_col_delta,
14058                )?),
14059                None => None,
14060            };
14061
14062            // Overlay punch-outs sourced from this span: entries entirely
14063            // before the boundary stay with the (id-retaining) upper half;
14064            // entries at/after the boundary transfer to the new lower span.
14065            // A straddling entry means the split is not clean.
14066            let mut lower_overlay_refs = Vec::new();
14067            for overlay_ref in authority
14068                .plane
14069                .formula_overlay
14070                .refs_for_source_span(span_ref)
14071            {
14072                let entry = authority.plane.formula_overlay.get(overlay_ref)?;
14073                match op.classify_region(Region::from_domain(&entry.domain)) {
14074                    AxisShiftCase::OtherSheet | AxisShiftCase::EntirelyBelow => {}
14075                    AxisShiftCase::EntirelyAboveShift { .. } => {
14076                        lower_overlay_refs.push(overlay_ref);
14077                    }
14078                    AxisShiftCase::Straddles | AxisShiftCase::DeleteFullyContains => {
14079                        return None;
14080                    }
14081                }
14082            }
14083
14084            let lower_force_binding_residual_axes = binding_set.is_some_and(|binding_set| {
14085                !binding_set.value_ref_slots.is_empty()
14086                    && (origin_row_delta != 0 || origin_col_delta != 0)
14087            });
14088
14089            Some(SplitPlan {
14090                span_ref,
14091                sheet_id: span.sheet_id,
14092                template_id: span.template_id,
14093                binding_set_id: span.binding_set_id,
14094                is_constant_result: span.is_constant_result,
14095                upper_domain,
14096                upper_read_summary,
14097                lower_new_origin_row,
14098                lower_new_origin_col,
14099                lower_new_domain,
14100                lower_new_read_summary,
14101                lower_force_binding_residual_axes,
14102                lower_overlay_refs,
14103                upper_dirty_region: None,
14104                lower_dirty_region: None,
14105            })
14106        }
14107
14108        /// Carry a 1-based template origin coordinate through a delete of
14109        /// `count` rows/columns starting at 0-based `start`. Returns None when
14110        /// the delete removes the origin coordinate itself: the template's
14111        /// relative reference deltas are anchored there, so the caller must
14112        /// demote rather than guess a new anchor.
14113        fn origin_coord_through_delete(origin: u32, start: u32, count: u32) -> Option<u32> {
14114            let origin0 = origin.checked_sub(1)?;
14115            let end = start.saturating_add(count);
14116            if origin0 < start {
14117                Some(origin)
14118            } else if origin0 >= end {
14119                Some(origin - count)
14120            } else {
14121                None
14122            }
14123        }
14124
14125        /// Carry the template origin through a delete instead of rebasing it
14126        /// to the compacted domain's start. Rebasing is only correct when the
14127        /// origin coincides with the domain start; split/shifted spans keep
14128        /// their original origin (Shift{origin_delta: 0} convention), and a
14129        /// head-overlapping delete moves the domain start away from the
14130        /// origin, so rebasing silently re-anchors every relative read.
14131        fn origin_through_delete(
14132            origin_row: u32,
14133            origin_col: u32,
14134            op: StructuralOp,
14135        ) -> Option<(u32, u32)> {
14136            match op {
14137                StructuralOp::DeleteRows { start, count, .. } => Some((
14138                    origin_coord_through_delete(origin_row, start, count)?,
14139                    origin_col,
14140                )),
14141                StructuralOp::DeleteColumns { start, count, .. } => Some((
14142                    origin_row,
14143                    origin_coord_through_delete(origin_col, start, count)?,
14144                )),
14145                StructuralOp::InsertRows { .. } | StructuralOp::InsertColumns { .. } => None,
14146            }
14147        }
14148
14149        let mut shift_plans = Vec::new();
14150        let mut split_plans = Vec::new();
14151        let mut remove_refs = Vec::new();
14152        let mut demote_refs = Vec::new();
14153        for span_ref in span_refs {
14154            let authority = self.graph.formula_authority();
14155            let Some(span) = authority.plane.spans.get(span_ref) else {
14156                continue;
14157            };
14158            let read_summary = span
14159                .read_summary_id
14160                .and_then(|id| authority.plane.span_read_summaries.get(id));
14161            let Some(op) = op else {
14162                // Non-structural demote path (per-cell write into span, or
14163                // remove_sheet's whole-sheet sweep). Only demote spans whose
14164                // result or read region intersects affected_region; leave
14165                // disjoint spans untouched.
14166                let result_region_affected =
14167                    Self::span_result_region_intersects_affected(span, &affected_region);
14168                let read_region_affected = Self::span_any_read_region_intersects_affected(
14169                    &authority.plane,
14170                    span,
14171                    &affected_region,
14172                );
14173                if result_region_affected || read_region_affected {
14174                    demote_refs.push(span_ref);
14175                }
14176                continue;
14177            };
14178            match classify_span_for_op(span, read_summary, op) {
14179                SpanShiftPlan::NoOp => {}
14180                SpanShiftPlan::Remove => {
14181                    remove_refs.push(span_ref);
14182                }
14183                SpanShiftPlan::Demote {
14184                    reason:
14185                        crate::formula_plane::structural_shift::SpanDemoteReason::DeletePartiallyOverlaps,
14186                } => {
14187                    let Some(template) = authority.plane.templates.get(span.template_id) else {
14188                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14189                            .with_message(
14190                                "FormulaPlane delete compaction found a span with a missing template",
14191                            )
14192                            .into());
14193                    };
14194                    let binding_compaction_safe = span
14195                        .binding_set_id
14196                        .and_then(|id| authority.plane.binding_sets.get(id))
14197                        .is_none_or(|binding_set| binding_set.is_single_literal_binding());
14198                    // Compaction projects read regions through the delete but
14199                    // keeps the template AST: an absolute read whose target
14200                    // the delete displaces would keep its stale coordinate
14201                    // (issue #168), so demote to the per-cell path instead.
14202                    let absolute_reads_compaction_safe = read_summary.is_none_or(|summary| {
14203                        !crate::formula_plane::structural_shift::summary_has_displaced_absolute_read(
14204                            summary, op,
14205                        )
14206                    });
14207                    // Compaction also keeps ONE relative-offset frame for the
14208                    // whole surviving domain. When the delete displaces
14209                    // placements and reads differently on either side of the
14210                    // band, no single frame is right (issue #171) — fall
14211                    // through to the split/demote path below.
14212                    let compaction_frame_sound = read_summary.is_none_or(|summary| {
14213                        crate::formula_plane::structural_shift::delete_compaction_frame_is_sound(
14214                            summary,
14215                            &span.domain,
14216                            template.origin_row,
14217                            template.origin_col,
14218                            op,
14219                        )
14220                    });
14221                    let mut compaction_plan = None;
14222                    if binding_compaction_safe
14223                        && absolute_reads_compaction_safe
14224                        && compaction_frame_sound
14225                        && let Some(new_domain) = compact_domain_through_delete(&span.domain, op)
14226                    {
14227                        let new_result_region = Region::from_domain(&new_domain);
14228                        let new_read_summary = if let Some(summary) = read_summary {
14229                            compact_read_summary_through_delete(summary, new_result_region, op)
14230                        } else {
14231                            None
14232                        };
14233                        let carried_origin =
14234                            origin_through_delete(template.origin_row, template.origin_col, op);
14235                        if (read_summary.is_none() || new_read_summary.is_some())
14236                            && let Some((new_origin_row, new_origin_col)) = carried_origin
14237                        {
14238                            let force_binding_residual_axes = span
14239                                .binding_set_id
14240                                .and_then(|id| authority.plane.binding_sets.get(id))
14241                                .is_some_and(|binding_set| {
14242                                    !binding_set.value_ref_slots.is_empty()
14243                                        && (new_origin_row != template.origin_row
14244                                            || new_origin_col != template.origin_col)
14245                                });
14246                            let dirty_region =
14247                                Self::structural_dirty_region_for_domain(&new_domain, op);
14248                            compaction_plan = Some(ShiftPlan {
14249                                span_ref,
14250                                template_id: span.template_id,
14251                                new_origin_row,
14252                                new_origin_col,
14253                                new_domain,
14254                                new_read_summary,
14255                                binding_set_id: span.binding_set_id,
14256                                force_binding_residual_axes,
14257                                rewrite: None,
14258                                dirty_region,
14259                            });
14260                        }
14261                    }
14262                    match compaction_plan {
14263                        Some(plan) => shift_plans.push(plan),
14264                        // Compaction cannot express the post-delete frame.
14265                        // Try the same split the insert path uses — an
14266                        // untouched upper half plus a shifted lower half,
14267                        // each re-classified in its own frame — and demote
14268                        // the whole span when the split is not provably
14269                        // clean (issue #171).
14270                        None => match plan_span_split(authority, span, span_ref, read_summary, op) {
14271                            Some(mut plan) => {
14272                                plan.upper_dirty_region =
14273                                    Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
14274                                plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
14275                                    &plan.lower_new_domain,
14276                                    op,
14277                                );
14278                                split_plans.push(plan);
14279                            }
14280                            None => demote_refs.push(span_ref),
14281                        },
14282                    }
14283                }
14284                SpanShiftPlan::Demote { .. } => {
14285                    demote_refs.push(span_ref);
14286                }
14287                SpanShiftPlan::Split => {
14288                    match plan_span_split(authority, span, span_ref, read_summary, op) {
14289                        Some(mut plan) => {
14290                            plan.upper_dirty_region =
14291                                Self::structural_dirty_region_for_domain(&plan.upper_domain, op);
14292                            plan.lower_dirty_region = Self::structural_dirty_region_for_domain(
14293                                &plan.lower_new_domain,
14294                                op,
14295                            );
14296                            split_plans.push(plan);
14297                        }
14298                        // Not provably clean: demote the whole span (the
14299                        // pre-split conservative path).
14300                        None => demote_refs.push(span_ref),
14301                    }
14302                }
14303                SpanShiftPlan::Shift {
14304                    row_delta,
14305                    col_delta,
14306                    origin_row_delta,
14307                    origin_col_delta,
14308                    rewrite_absolute_reads,
14309                } => {
14310                    let Some(template) = authority.plane.templates.get(span.template_id) else {
14311                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14312                            .with_message("FormulaPlane shift found a span with a missing template")
14313                            .into());
14314                    };
14315                    let Some(new_origin_row) =
14316                        checked_shift_u32(template.origin_row, origin_row_delta)
14317                    else {
14318                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14319                            .with_message("FormulaPlane shift overflowed template origin row")
14320                            .into());
14321                    };
14322                    let Some(new_origin_col) =
14323                        checked_shift_u32(template.origin_col, origin_col_delta)
14324                    else {
14325                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14326                            .with_message("FormulaPlane shift overflowed template origin column")
14327                            .into());
14328                    };
14329                    let Some(new_domain) =
14330                        span.domain.project_through_axis_shift(row_delta, col_delta)
14331                    else {
14332                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14333                            .with_message("FormulaPlane shift overflowed span domain")
14334                            .into());
14335                    };
14336                    let new_result_region = Region::from_domain(&new_domain);
14337                    let force_binding_residual_axes = span
14338                        .binding_set_id
14339                        .and_then(|id| authority.plane.binding_sets.get(id))
14340                        .is_some_and(|binding_set| {
14341                            !binding_set.value_ref_slots.is_empty()
14342                                && (origin_row_delta != 0 || origin_col_delta != 0)
14343                        });
14344                    let rewrite_and_summary = if rewrite_absolute_reads {
14345                        // Reify the (immutable, possibly shared) template
14346                        // AST and repoint its displaced references through
14347                        // the shared adjuster — the same relocation the
14348                        // per-cell path applies, so span-ON stays equal to
14349                        // span-OFF (issue #168). Canonical keys are
14350                        // re-derived from the rewritten AST at the new
14351                        // origin, so the fresh record cannot collide with
14352                        // the pre-rewrite template.
14353                        //
14354                        // FRAME GUARD: the adjuster relocates by AUTHORED
14355                        // coordinate, but a template's authoring frame may
14356                        // diverge from its domain (split lower halves keep
14357                        // the original anchor; pinned-origin shifts move the
14358                        // domain without the origin). The rewrite is only
14359                        // equivalent to per-cell adjustment when every
14360                        // relative read's authored coordinate classifies
14361                        // against the op boundary the same way its read
14362                        // region does — otherwise the rewrite shears
14363                        // relative reads by the op count. Demote instead;
14364                        // see rewrite_frame_is_sound for the full invariant.
14365                        let frame_sound = read_summary.is_some_and(|summary| {
14366                            crate::formula_plane::structural_shift::rewrite_frame_is_sound(
14367                                summary,
14368                                template.origin_row,
14369                                template.origin_col,
14370                                op,
14371                            )
14372                        });
14373                        if !frame_sound {
14374                            demote_refs.push(span_ref);
14375                            continue;
14376                        }
14377                        let Some(ast) = self
14378                            .graph
14379                            .data_store()
14380                            .retrieve_ast(template.ast_id, self.graph.sheet_reg())
14381                        else {
14382                            demote_refs.push(span_ref);
14383                            continue;
14384                        };
14385                        let shift_op = shift_operation_for(op);
14386                        let adjuster =
14387                            crate::engine::graph::editor::reference_adjuster::ReferenceAdjuster::new();
14388                        let context = crate::engine::graph::editor::reference_adjuster::ReferenceContext::new(
14389                            span.sheet_id,
14390                            self.graph.sheet_reg(),
14391                        );
14392                        let Some(adjusted_ast) =
14393                            adjuster.adjust_ast_if_changed_in_context(&ast, &shift_op, &context)
14394                        else {
14395                            // The classifier saw a displaced absolute read,
14396                            // so an unchanged AST is a contract violation;
14397                            // demote conservatively rather than shift with a
14398                            // stale template.
14399                            demote_refs.push(span_ref);
14400                            continue;
14401                        };
14402                        let canonical = crate::formula_plane::template_canonical::canonicalize_template(
14403                            &adjusted_ast,
14404                            new_origin_row,
14405                            new_origin_col,
14406                        );
14407                        if !canonical.labels.is_authority_supported() {
14408                            demote_refs.push(span_ref);
14409                            continue;
14410                        }
14411                        // The rewrite touches only references; the literal
14412                        // slot layout must survive unchanged or the binding
14413                        // set's per-placement literal bindings no longer
14414                        // line up. Demote defensively if it differs.
14415                        let literal_slots_compatible = span
14416                            .binding_set_id
14417                            .and_then(|id| authority.plane.binding_sets.get(id))
14418                            .is_none_or(|binding_set| {
14419                                binding_set.literal_slots.as_ref()
14420                                    == canonical.literal_slot_descriptors.as_ref()
14421                            });
14422                        if !literal_slots_compatible {
14423                            demote_refs.push(span_ref);
14424                            continue;
14425                        }
14426                        // Value-ref slot patterns are re-derived from the
14427                        // rewritten canonical template so memoized reads
14428                        // resolve the repointed coordinates (issue #168).
14429                        // Placement-relative memo KEYS are additionally
14430                        // protected by force_binding_residual_axes below
14431                        // (the rewrite always moves the origin, so the
14432                        // existing origin-delta condition fires).
14433                        let value_ref_slots = Arc::from(
14434                            crate::formula_plane::placement::value_ref_slot_descriptors(
14435                                &canonical.expr,
14436                            )
14437                            .into_boxed_slice(),
14438                        );
14439                        // The read summary is RE-DERIVED from the rewritten
14440                        // canonical template rather than structurally
14441                        // transformed: a structural transform would keep the
14442                        // old projection rules, whose absolute indices (and
14443                        // relative offsets, via the moved origin) no longer
14444                        // match the rewritten AST — incremental writes to
14445                        // the repointed targets would then silently fail to
14446                        // dirty the span. Re-derivation rebuilds rules in
14447                        // the (rewritten AST, new origin) frame, keeping
14448                        // `result_region == domain region` by construction.
14449                        let dependency_summary =
14450                            crate::formula_plane::dependency_summary::summarize_canonical_template(
14451                                &canonical,
14452                            );
14453                        let rewritten_result_region =
14454                            ResultRegion::scalar_cells(new_domain.clone());
14455                        let Ok(rederived_summary) = SpanReadSummary::from_formula_summary(
14456                            span.sheet_id,
14457                            &rewritten_result_region,
14458                            &dependency_summary,
14459                            self.graph.sheet_reg(),
14460                        ) else {
14461                            demote_refs.push(span_ref);
14462                            continue;
14463                        };
14464                        let formula_text = Some(Arc::<str>::from(
14465                            formualizer_parse::pretty::canonical_formula(&adjusted_ast),
14466                        ));
14467                        Some((
14468                            SpanTemplateRewrite {
14469                                exact_canonical_key: Arc::<str>::from(canonical.key.payload()),
14470                                parameterized_canonical_key: Arc::<str>::from(
14471                                    canonical.parameterized_key.payload(),
14472                                ),
14473                                formula_text,
14474                                canonical_expr: canonical.expr,
14475                                value_ref_slots,
14476                                adjusted_ast,
14477                            },
14478                            rederived_summary,
14479                        ))
14480                    } else {
14481                        None
14482                    };
14483                    let (rewrite, new_read_summary) = match rewrite_and_summary {
14484                        Some((rewrite, summary)) => (Some(rewrite), Some(summary)),
14485                        None => {
14486                            let new_read_summary = if let Some(summary) = read_summary {
14487                                Some(
14488                                    shifted_read_summary(
14489                                        summary,
14490                                        new_result_region,
14491                                        op,
14492                                        row_delta,
14493                                        col_delta,
14494                                        origin_row_delta,
14495                                        origin_col_delta,
14496                                    )
14497                                    .ok_or_else(|| {
14498                                        ExcelError::new(ExcelErrorKind::Ref).with_message(
14499                                            "FormulaPlane shift could not project read summary",
14500                                        )
14501                                    })?,
14502                                )
14503                            } else {
14504                                None
14505                            };
14506                            (None, new_read_summary)
14507                        }
14508                    };
14509                    let dirty_region = Self::structural_dirty_region_for_domain(&new_domain, op);
14510                    shift_plans.push(ShiftPlan {
14511                        span_ref,
14512                        template_id: span.template_id,
14513                        new_origin_row,
14514                        new_origin_col,
14515                        new_domain,
14516                        new_read_summary,
14517                        binding_set_id: span.binding_set_id,
14518                        force_binding_residual_axes,
14519                        rewrite,
14520                        dirty_region,
14521                    });
14522                }
14523            }
14524        }
14525        let span_geometry_changed =
14526            !shift_plans.is_empty() || !split_plans.is_empty() || !remove_refs.is_empty();
14527        let mut span_dirty_deltas = Vec::new();
14528        if span_geometry_changed {
14529            // Rewritten template ASTs must be interned into the graph's AST
14530            // arena before the authority is borrowed mutably; the literal
14531            // slot map is keyed by the freshly interned arena node ids.
14532            let mut prepared_shift_plans = Vec::with_capacity(shift_plans.len());
14533            for plan in shift_plans {
14534                let rewrite_prepared = plan.rewrite.as_ref().map(|rewrite| {
14535                    let ast_id = self.graph.store_ast(&rewrite.adjusted_ast);
14536                    let template_slot_map =
14537                        crate::formula_plane::placement::build_template_slot_map(
14538                            ast_id,
14539                            self.graph.data_store(),
14540                            &rewrite.canonical_expr,
14541                        );
14542                    (ast_id, template_slot_map)
14543                });
14544                prepared_shift_plans.push((plan, rewrite_prepared));
14545            }
14546            let authority = self.graph.formula_authority_mut();
14547            for span_ref in remove_refs {
14548                authority.plane.remove_overlays_for_source_span(span_ref);
14549                authority.plane.remove_span(span_ref);
14550            }
14551            for (plan, rewrite_prepared) in prepared_shift_plans {
14552                let dirty_region = plan.dirty_region;
14553                let (template_id, rewrite_slots) = if let Some(rewrite) = plan.rewrite {
14554                    let Some((ast_id, template_slot_map)) = rewrite_prepared else {
14555                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14556                            .with_message("FormulaPlane shift lost a rewritten template AST")
14557                            .into());
14558                    };
14559                    let Some(template_id) = authority.plane.intern_rewritten_template(
14560                        plan.template_id,
14561                        ast_id,
14562                        rewrite.exact_canonical_key,
14563                        rewrite.parameterized_canonical_key,
14564                        rewrite.formula_text,
14565                        plan.new_origin_row,
14566                        plan.new_origin_col,
14567                    ) else {
14568                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14569                            .with_message("FormulaPlane shift could not intern rewritten template")
14570                            .into());
14571                    };
14572                    (
14573                        template_id,
14574                        Some((template_slot_map, rewrite.value_ref_slots)),
14575                    )
14576                } else {
14577                    let Some(template_id) = authority.plane.intern_shifted_template_origin(
14578                        plan.template_id,
14579                        plan.new_origin_row,
14580                        plan.new_origin_col,
14581                    ) else {
14582                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14583                            .with_message("FormulaPlane shift could not clone template origin")
14584                            .into());
14585                    };
14586                    (template_id, None)
14587                };
14588                if let Some((template_slot_map, value_ref_slots)) = rewrite_slots
14589                    && let Some(binding_set_id) = plan.binding_set_id
14590                {
14591                    authority.plane.set_binding_template_slots(
14592                        binding_set_id,
14593                        template_slot_map,
14594                        value_ref_slots,
14595                    );
14596                }
14597                if let Some(binding_set_id) = plan.binding_set_id {
14598                    let Some(template) = authority.plane.templates.get(template_id) else {
14599                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14600                            .with_message("FormulaPlane shift could not find shifted template")
14601                            .into());
14602                    };
14603                    let (ast_id, origin_row, origin_col) =
14604                        (template.ast_id, template.origin_row, template.origin_col);
14605                    authority.plane.set_binding_template_anchor(
14606                        binding_set_id,
14607                        ast_id,
14608                        origin_row,
14609                        origin_col,
14610                    );
14611                }
14612                let read_summary_id = plan
14613                    .new_read_summary
14614                    .map(|summary| authority.plane.insert_span_read_summary(summary));
14615                let result_region = ResultRegion::scalar_cells(plan.new_domain.clone());
14616                if !authority.plane.replace_span_geometry(
14617                    plan.span_ref,
14618                    template_id,
14619                    plan.new_domain,
14620                    result_region,
14621                    read_summary_id,
14622                ) {
14623                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14624                        .with_message("FormulaPlane shift could not update span geometry")
14625                        .into());
14626                }
14627                if let Some(region) = dirty_region {
14628                    span_dirty_deltas.push((plan.span_ref, region));
14629                }
14630                if plan.force_binding_residual_axes
14631                    && let Some(binding_set_id) = plan.binding_set_id
14632                {
14633                    // Value-ref memoization keys are placement-relative. When a
14634                    // structural op moves the formula origin while keeping some
14635                    // precedents fixed (e.g. insert a column before a formula
14636                    // family that reads column A), those keys no longer name
14637                    // the same producer cells. Keep correctness by forcing
14638                    // placement offsets into the key so memoization falls back
14639                    // to per-placement work rather than broadcasting stale
14640                    // representative values.
14641                    authority.plane.force_binding_residual_axes(binding_set_id);
14642                }
14643            }
14644            for plan in split_plans {
14645                let upper_dirty_region = plan.upper_dirty_region;
14646                let lower_dirty_region = plan.lower_dirty_region;
14647                let Some(split_relocation) = authority
14648                    .plane
14649                    .spans
14650                    .get(plan.span_ref)
14651                    .map(|span| span.ast_relocation)
14652                else {
14653                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14654                        .with_message("FormulaPlane split lost its anchor AST state")
14655                        .into());
14656                };
14657                // Lower half: a fresh span at the POST-insert coordinates,
14658                // mirroring the Shift arm's template/binding handling.
14659                let Some(lower_template_id) = authority.plane.intern_shifted_template_origin(
14660                    plan.template_id,
14661                    plan.lower_new_origin_row,
14662                    plan.lower_new_origin_col,
14663                ) else {
14664                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14665                        .with_message("FormulaPlane split could not clone template origin")
14666                        .into());
14667                };
14668                let lower_binding_set_id = if let Some(binding_set_id) = plan.binding_set_id {
14669                    let Some(source) = authority.plane.binding_sets.get(binding_set_id) else {
14670                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14671                            .with_message(
14672                                "FormulaPlane split found a span with a missing binding set",
14673                            )
14674                            .into());
14675                    };
14676                    // Safe to clone wholesale: `plan_span_split` only accepts
14677                    // single-literal-binding sets, whose per-placement lookups
14678                    // are ordinal-independent.
14679                    let clone = source.clone();
14680                    let new_binding_set_id = authority.plane.insert_binding_set(clone);
14681                    let Some(template) = authority.plane.templates.get(lower_template_id) else {
14682                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14683                            .with_message("FormulaPlane split could not find shifted template")
14684                            .into());
14685                    };
14686                    let (ast_id, origin_row, origin_col) =
14687                        (template.ast_id, template.origin_row, template.origin_col);
14688                    authority.plane.set_binding_template_anchor(
14689                        new_binding_set_id,
14690                        ast_id,
14691                        origin_row,
14692                        origin_col,
14693                    );
14694                    Some(new_binding_set_id)
14695                } else {
14696                    None
14697                };
14698                let lower_read_summary_id = plan
14699                    .lower_new_read_summary
14700                    .map(|summary| authority.plane.insert_span_read_summary(summary));
14701                let lower_result_region = ResultRegion::scalar_cells(plan.lower_new_domain.clone());
14702                let lower_ref = authority.plane.insert_span_with_ast_relocation(
14703                    crate::formula_plane::runtime::NewFormulaSpan {
14704                        sheet_id: plan.sheet_id,
14705                        template_id: lower_template_id,
14706                        domain: plan.lower_new_domain,
14707                        result_region: lower_result_region,
14708                        intrinsic_mask_id: None,
14709                        read_summary_id: lower_read_summary_id,
14710                        binding_set_id: lower_binding_set_id,
14711                        is_constant_result: plan.is_constant_result,
14712                    },
14713                    crate::formula_plane::runtime::SpanAstRelocation {
14714                        ast_id: split_relocation.ast_id,
14715                        anchor_row: plan.lower_new_origin_row,
14716                        anchor_col: plan.lower_new_origin_col,
14717                    },
14718                );
14719                if let Some(region) = lower_dirty_region {
14720                    span_dirty_deltas.push((lower_ref, region));
14721                }
14722                if let Some(binding_set_id) = lower_binding_set_id {
14723                    authority
14724                        .plane
14725                        .set_binding_span_ref(binding_set_id, lower_ref);
14726                    if plan.lower_force_binding_residual_axes {
14727                        // Same contract as the Shift arm: a moved origin with
14728                        // stationary value-ref precedents invalidates
14729                        // placement-relative memoization keys.
14730                        authority.plane.force_binding_residual_axes(binding_set_id);
14731                    }
14732                }
14733                // The lower half inherits the parent's defined-name
14734                // invalidation registrations; the upper half keeps them by
14735                // retaining the span id.
14736                let name_keys = authority.plane.span_registered_name_keys(plan.span_ref.id);
14737                if !name_keys.is_empty() {
14738                    authority
14739                        .plane
14740                        .register_span_name_dependents(lower_ref, &name_keys);
14741                }
14742                for overlay_ref in plan.lower_overlay_refs {
14743                    if !authority
14744                        .plane
14745                        .set_overlay_source_span(overlay_ref, Some(lower_ref))
14746                    {
14747                        return Err(ExcelError::new(ExcelErrorKind::Ref)
14748                            .with_message(
14749                                "FormulaPlane split could not re-point overlay provenance",
14750                            )
14751                            .into());
14752                    }
14753                }
14754                // Upper half: keep the span id, truncate the domain, and swap
14755                // in a summary re-derived for the truncated result region.
14756                let upper_read_summary_id = plan
14757                    .upper_read_summary
14758                    .map(|summary| authority.plane.insert_span_read_summary(summary));
14759                let upper_result_region = ResultRegion::scalar_cells(plan.upper_domain.clone());
14760                if !authority.plane.replace_span_geometry_with_ast_relocation(
14761                    plan.span_ref,
14762                    plan.template_id,
14763                    split_relocation,
14764                    plan.upper_domain,
14765                    upper_result_region,
14766                    upper_read_summary_id,
14767                ) {
14768                    return Err(ExcelError::new(ExcelErrorKind::Ref)
14769                        .with_message("FormulaPlane split could not update span geometry")
14770                        .into());
14771                }
14772                if let Some(region) = upper_dirty_region {
14773                    span_dirty_deltas.push((plan.span_ref, region));
14774                }
14775            }
14776            authority.rebuild_indexes();
14777        }
14778        for (span_ref, region) in span_dirty_deltas {
14779            self.graph.mark_formula_span_region_dirty(span_ref, region);
14780        }
14781
14782        let mut span_plans = Vec::new();
14783        for span_ref in demote_refs {
14784            let authority = self.graph.formula_authority();
14785            let Some(span) = authority.plane.spans.get(span_ref) else {
14786                continue;
14787            };
14788            let relocation = span.ast_relocation;
14789            let ast = self
14790                .graph
14791                .data_store()
14792                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
14793                .ok_or_else(|| {
14794                    ExcelError::new(ExcelErrorKind::Ref).with_message(
14795                        "FormulaPlane demotion could not retrieve the span anchor AST",
14796                    )
14797                })?;
14798            let placements = span
14799                .domain
14800                .iter()
14801                .map(|placement| (placement.row + 1, placement.col + 1))
14802                .collect();
14803            span_plans.push(SpanPlan {
14804                span_ref,
14805                sheet_id: span.sheet_id,
14806                ast,
14807                origin_row: relocation.anchor_row,
14808                origin_col: relocation.anchor_col,
14809                binding_set_id: span.binding_set_id,
14810                placements,
14811            });
14812        }
14813        if span_plans.is_empty() {
14814            return Ok(());
14815        }
14816
14817        let mut relocated = Vec::new();
14818        let mut placement_cells = Vec::new();
14819        for plan in &span_plans {
14820            for &(row, col) in &plan.placements {
14821                let row_delta = i64::from(row) - i64::from(plan.origin_row);
14822                let col_delta = i64::from(col) - i64::from(plan.origin_col);
14823                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
14824                    let authority = self.graph.formula_authority();
14825                    if let Some(binding_set) = authority.plane.binding_sets.get(binding_set_id) {
14826                        if binding_set.is_single_literal_binding() {
14827                            plan.ast.clone()
14828                        } else {
14829                            let placement = crate::formula_plane::runtime::PlacementCoord::new(
14830                                plan.sheet_id,
14831                                row.saturating_sub(1),
14832                                col.saturating_sub(1),
14833                            );
14834                            let binding =
14835                                authority.plane.spans.get(plan.span_ref).and_then(|span| {
14836                                    binding_set
14837                                        .literal_bindings_for_placement(&span.domain, placement)
14838                                });
14839                            if let Some(binding) = binding {
14840                                substitute_literal_slots_for_template_placement(
14841                                    &plan.ast,
14842                                    binding.as_ref(),
14843                                )
14844                            } else {
14845                                plan.ast.clone()
14846                            }
14847                        }
14848                    } else {
14849                        plan.ast.clone()
14850                    }
14851                } else {
14852                    plan.ast.clone()
14853                };
14854                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)?;
14855                relocated.push((plan.sheet_id, row, col, ast));
14856                placement_cells.push((plan.sheet_id, row, col));
14857            }
14858        }
14859        let planned_by_sheet = {
14860            let mut pipeline = self.ingest_pipeline();
14861            let mut planned_by_sheet: BTreeMap<
14862                SheetId,
14863                Vec<(u32, u32, AstNodeId, DependencyPlanRow)>,
14864            > = BTreeMap::new();
14865            for (formula_sheet_id, row, col, ast) in relocated {
14866                let placement =
14867                    CellRef::new(formula_sheet_id, Coord::from_excel(row, col, true, true));
14868                let ingested =
14869                    pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?;
14870                planned_by_sheet.entry(formula_sheet_id).or_default().push((
14871                    row,
14872                    col,
14873                    ingested.ast_id,
14874                    ingested.dep_plan,
14875                ));
14876            }
14877            planned_by_sheet
14878        };
14879        {
14880            let authority = self.graph.formula_authority_mut();
14881            for plan in &span_plans {
14882                authority
14883                    .plane
14884                    .remove_overlays_for_source_span(plan.span_ref);
14885                authority.plane.remove_span(plan.span_ref);
14886            }
14887            authority.rebuild_indexes();
14888        }
14889        if clear_computed_overlays {
14890            // Only clear placement cells whose coordinate intersects the affected
14891            // structural region. The structural-op contract preserves cells
14892            // BEFORE the structural boundary; the legacy `clear_computed_overlay_after_*`
14893            // call honors that. Demoting a span whose footprint straddles the
14894            // boundary still must not clear cells before the boundary, even
14895            // though the span as a whole is demoted.
14896            self.clear_computed_overlay_cells_in_region(&placement_cells, &affected_region);
14897        }
14898        for (formula_sheet_id, planned) in planned_by_sheet {
14899            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
14900            self.graph
14901                .bulk_set_formulas_with_plans(&sheet_name, planned)?;
14902        }
14903        if !clear_computed_overlays {
14904            for (formula_sheet_id, row, col) in &placement_cells {
14905                let row0 = row.saturating_sub(1);
14906                let col0 = col.saturating_sub(1);
14907                if dirty_span_coords.contains(&(*formula_sheet_id, row0, col0)) {
14908                    continue;
14909                }
14910                let cell =
14911                    CellRef::new(*formula_sheet_id, Coord::from_excel(*row, *col, true, true));
14912                if let Some(&vertex_id) = self.graph.get_vertex_id_for_address(&cell) {
14913                    self.graph.set_dirty(vertex_id, false);
14914                }
14915            }
14916        }
14917        Ok(())
14918    }
14919
14920    /// Prepare an exact-ref, additions-only refinement of FormulaPlane spans
14921    /// into legacy graph formulas. All fallible relocation, dependency analysis,
14922    /// identifier reservation, and conflict checks happen before commit.
14923    pub(crate) fn prepare_formula_span_demotion(
14924        &mut self,
14925        span_refs: &[FormulaSpanRef],
14926    ) -> Result<PreparedFormulaSpanDemotion, FormulaSpanDemotionError> {
14927        #[cfg(test)]
14928        let fault = self
14929            .formula_span_demotion_fault_for_test
14930            .take()
14931            .unwrap_or_default();
14932        #[cfg(not(test))]
14933        let fault = FormulaSpanDemotionFault::None;
14934
14935        if fault == FormulaSpanDemotionFault::AstPreparation {
14936            return Err(FormulaSpanDemotionError::Injected(fault));
14937        }
14938
14939        fn substitute_literal_slots(ast: &ASTNode, binding: &[LiteralValue]) -> ASTNode {
14940            fn visit(
14941                ast: &ASTNode,
14942                binding: &[LiteralValue],
14943                next: &mut usize,
14944                in_array: bool,
14945            ) -> ASTNode {
14946                let node_type = match &ast.node_type {
14947                    ASTNodeType::Literal(_) if !in_array => {
14948                        let value = binding.get(*next).cloned().unwrap_or(LiteralValue::Empty);
14949                        *next = next.saturating_add(1);
14950                        ASTNodeType::Literal(value)
14951                    }
14952                    ASTNodeType::Literal(value) => ASTNodeType::Literal(value.clone()),
14953                    ASTNodeType::Omitted => ASTNodeType::Omitted,
14954                    ASTNodeType::Reference {
14955                        original,
14956                        reference,
14957                    } => ASTNodeType::Reference {
14958                        original: original.clone(),
14959                        reference: reference.clone(),
14960                    },
14961                    ASTNodeType::UnaryOp { op, expr } => ASTNodeType::UnaryOp {
14962                        op: op.clone(),
14963                        expr: Box::new(visit(expr, binding, next, in_array)),
14964                    },
14965                    ASTNodeType::BinaryOp { op, left, right } => ASTNodeType::BinaryOp {
14966                        op: op.clone(),
14967                        left: Box::new(visit(left, binding, next, in_array)),
14968                        right: Box::new(visit(right, binding, next, in_array)),
14969                    },
14970                    ASTNodeType::Function { name, args } => ASTNodeType::Function {
14971                        name: name.clone(),
14972                        args: args
14973                            .iter()
14974                            .map(|arg| visit(arg, binding, next, in_array))
14975                            .collect(),
14976                    },
14977                    ASTNodeType::Call { callee, args } => ASTNodeType::Call {
14978                        callee: Box::new(visit(callee, binding, next, in_array)),
14979                        args: args
14980                            .iter()
14981                            .map(|arg| visit(arg, binding, next, in_array))
14982                            .collect(),
14983                    },
14984                    ASTNodeType::Array(rows) => ASTNodeType::Array(
14985                        rows.iter()
14986                            .map(|row| {
14987                                row.iter()
14988                                    .map(|cell| visit(cell, binding, next, true))
14989                                    .collect()
14990                            })
14991                            .collect(),
14992                    ),
14993                };
14994                ASTNode::new(node_type, ast.source_token.clone())
14995            }
14996            let mut next = 0;
14997            visit(ast, binding, &mut next, false)
14998        }
14999
15000        struct SpanMaterialization {
15001            span_ref: FormulaSpanRef,
15002            sheet_id: SheetId,
15003            ast: ASTNode,
15004            origin_row: u32,
15005            origin_col: u32,
15006            binding_set_id: Option<crate::formula_plane::runtime::SpanBindingSetId>,
15007            domain: PlacementDomain,
15008        }
15009
15010        let request_id = self
15011            .active_evaluation_resource_request
15012            .as_ref()
15013            .map(|request| request.request_id);
15014        let resource_error =
15015            |reason: formualizer_common::ResourceExhaustionReason, limit: u64, observed: u64| {
15016                FormulaSpanDemotionError::Resource(
15017                    crate::engine::ResourceLedgerError::Exhausted(
15018                        formualizer_common::ResourceExhaustionDetail {
15019                            reason,
15020                            limit,
15021                            observed,
15022                            request_id,
15023                        },
15024                    )
15025                    .into_excel_error(),
15026                )
15027            };
15028        let authority = self.graph.formula_authority();
15029        let expected_plane_epoch = authority.plane.epoch().0;
15030        let expected_indexes_epoch = authority.indexes_epoch();
15031        let expected_indexed_plane_epoch = authority.indexed_plane_epoch();
15032        let mut unique = Vec::new();
15033        unique.try_reserve_exact(span_refs.len()).map_err(|_| {
15034            resource_error(
15035                formualizer_common::ResourceExhaustionReason::ScratchMemory,
15036                u64::MAX,
15037                span_refs.len() as u64,
15038            )
15039        })?;
15040        let mut plans = Vec::new();
15041        plans.try_reserve_exact(span_refs.len()).map_err(|_| {
15042            resource_error(
15043                formualizer_common::ResourceExhaustionReason::ScratchMemory,
15044                u64::MAX,
15045                span_refs.len() as u64,
15046            )
15047        })?;
15048        let mut placement_count = 0usize;
15049        let mut placements_by_sheet: BTreeMap<SheetId, u64> = BTreeMap::new();
15050        for &span_ref in span_refs {
15051            if unique.contains(&span_ref) {
15052                continue;
15053            }
15054            unique.push(span_ref);
15055            let span = authority
15056                .plane
15057                .spans
15058                .get(span_ref)
15059                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15060            let relocation = span.ast_relocation;
15061            let ast = self
15062                .graph
15063                .data_store()
15064                .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())
15065                .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15066            let count = usize::try_from(span.domain.cell_count())
15067                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?;
15068            placement_count = placement_count
15069                .checked_add(count)
15070                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
15071            if u64::try_from(placement_count)
15072                .map_err(|_| FormulaSpanDemotionError::CountOverflow)?
15073                > self.workbook_load_limits.max_formula_plane_fallback_cells
15074            {
15075                return Err(resource_error(
15076                    formualizer_common::ResourceExhaustionReason::MaterializationCells,
15077                    self.workbook_load_limits.max_formula_plane_fallback_cells,
15078                    placement_count as u64,
15079                ));
15080            }
15081            let sheet_count = placements_by_sheet.entry(span.sheet_id).or_default();
15082            *sheet_count = sheet_count
15083                .checked_add(span.domain.cell_count())
15084                .ok_or(FormulaSpanDemotionError::CountOverflow)?;
15085            if *sheet_count > self.workbook_load_limits.max_sheet_logical_cells {
15086                return Err(resource_error(
15087                    formualizer_common::ResourceExhaustionReason::Admission,
15088                    self.workbook_load_limits.max_sheet_logical_cells,
15089                    *sheet_count,
15090                ));
15091            }
15092            let domain_in_bounds = match &span.domain {
15093                PlacementDomain::RowRun { row_end, col, .. } => {
15094                    *row_end < self.workbook_load_limits.max_sheet_rows
15095                        && *col < self.workbook_load_limits.max_sheet_cols
15096                }
15097                PlacementDomain::ColRun { row, col_end, .. } => {
15098                    *row < self.workbook_load_limits.max_sheet_rows
15099                        && *col_end < self.workbook_load_limits.max_sheet_cols
15100                }
15101                PlacementDomain::Rect {
15102                    row_end, col_end, ..
15103                } => {
15104                    *row_end < self.workbook_load_limits.max_sheet_rows
15105                        && *col_end < self.workbook_load_limits.max_sheet_cols
15106                }
15107            };
15108            if !domain_in_bounds {
15109                return Err(resource_error(
15110                    formualizer_common::ResourceExhaustionReason::Admission,
15111                    u64::from(self.workbook_load_limits.max_sheet_rows)
15112                        .saturating_mul(u64::from(self.workbook_load_limits.max_sheet_cols)),
15113                    span.domain.cell_count(),
15114                ));
15115            }
15116            plans.push(SpanMaterialization {
15117                span_ref,
15118                sheet_id: span.sheet_id,
15119                ast,
15120                origin_row: relocation.anchor_row,
15121                origin_col: relocation.anchor_col,
15122                binding_set_id: span.binding_set_id,
15123                domain: span.domain.clone(),
15124            });
15125        }
15126        let mut relocated = Vec::new();
15127        relocated.try_reserve_exact(placement_count).map_err(|_| {
15128            resource_error(
15129                formualizer_common::ResourceExhaustionReason::ScratchMemory,
15130                u64::MAX,
15131                placement_count as u64,
15132            )
15133        })?;
15134        for plan in &plans {
15135            for coord in plan.domain.iter() {
15136                let row = coord.row.saturating_add(1);
15137                let col = coord.col.saturating_add(1);
15138                let bound_ast = if let Some(binding_set_id) = plan.binding_set_id {
15139                    let authority = self.graph.formula_authority();
15140                    let binding_set = authority
15141                        .plane
15142                        .binding_sets
15143                        .get(binding_set_id)
15144                        .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15145                    if binding_set.is_single_literal_binding() {
15146                        plan.ast.clone()
15147                    } else {
15148                        let placement = PlacementCoord::new(
15149                            plan.sheet_id,
15150                            row.saturating_sub(1),
15151                            col.saturating_sub(1),
15152                        );
15153                        let span = authority
15154                            .plane
15155                            .spans
15156                            .get(plan.span_ref)
15157                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15158                        let binding = binding_set
15159                            .literal_bindings_for_placement(&span.domain, placement)
15160                            .ok_or(FormulaSpanDemotionError::InvalidSpan)?;
15161                        substitute_literal_slots(&plan.ast, binding.as_ref())
15162                    }
15163                } else {
15164                    plan.ast.clone()
15165                };
15166                let row_delta = i64::from(row) - i64::from(plan.origin_row);
15167                let col_delta = i64::from(col) - i64::from(plan.origin_col);
15168                let ast = relocate_ast_for_template_placement(&bound_ast, row_delta, col_delta)
15169                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
15170                relocated.push((plan.sheet_id, row, col, ast));
15171            }
15172        }
15173
15174        let mut analyzed = Vec::new();
15175        analyzed.try_reserve_exact(relocated.len()).map_err(|_| {
15176            resource_error(
15177                formualizer_common::ResourceExhaustionReason::ScratchMemory,
15178                u64::MAX,
15179                relocated.len() as u64,
15180            )
15181        })?;
15182        {
15183            let mut pipeline = self.ingest_pipeline();
15184            for (sheet_id, row, col, ast) in relocated {
15185                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
15186                let ingested = pipeline
15187                    .ingest_formula(FormulaAstInput::Tree(ast), placement, None)
15188                    .map_err(FormulaSpanDemotionError::AstPreparation)?;
15189                analyzed.push((sheet_id, row, col, ingested.ast_id, ingested.dep_plan));
15190            }
15191        }
15192
15193        if fault == FormulaSpanDemotionFault::LegacyGraphPreparation {
15194            return Err(FormulaSpanDemotionError::Injected(fault));
15195        }
15196        let legacy_graph = self
15197            .graph
15198            .prepare_legacy_graph_plan_multi_sheet(analyzed)
15199            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
15200        self.prepared_legacy_admission(&legacy_graph, placement_count as u64)
15201            .map_err(FormulaSpanDemotionError::Resource)?;
15202
15203        Ok(PreparedFormulaSpanDemotion {
15204            span_refs: unique,
15205            expected_plane_epoch,
15206            expected_indexes_epoch,
15207            expected_indexed_plane_epoch,
15208            placement_count,
15209            legacy_graph,
15210            fault,
15211        })
15212    }
15213
15214    pub(crate) fn validate_prepared_formula_span_demotion(
15215        &self,
15216        prepared: &PreparedFormulaSpanDemotion,
15217    ) -> Result<(), FormulaSpanDemotionError> {
15218        let authority = self.graph.formula_authority();
15219        if authority.plane.epoch().0 != prepared.expected_plane_epoch
15220            || authority.indexes_epoch() != prepared.expected_indexes_epoch
15221            || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
15222            || prepared
15223                .span_refs
15224                .iter()
15225                .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
15226        {
15227            return Err(FormulaSpanDemotionError::StaleAuthority);
15228        }
15229        self.graph
15230            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
15231            .map_err(FormulaSpanDemotionError::LegacyGraph)
15232    }
15233
15234    /// Validate every recoverable assumption and fault before the first
15235    /// mutation, then compose the existing prevalidated graph append with exact
15236    /// authority removal. Allocator OOM/panic remains process-fatal, matching
15237    /// the E2/E3 prepared-transaction contract; there is no recoverable branch
15238    /// after graph application begins.
15239    pub(crate) fn commit_prepared_formula_span_demotion(
15240        &mut self,
15241        prepared: PreparedFormulaSpanDemotion,
15242    ) -> Result<FormulaSpanDemotionReport, FormulaSpanDemotionError> {
15243        self.prepared_legacy_admission(&prepared.legacy_graph, prepared.placement_count as u64)
15244            .map_err(FormulaSpanDemotionError::Resource)?;
15245        if prepared.fault == FormulaSpanDemotionFault::FinalLegacyGraphValidation {
15246            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15247        }
15248        self.graph
15249            .validate_prepared_legacy_graph_plan(&prepared.legacy_graph)
15250            .map_err(FormulaSpanDemotionError::LegacyGraph)?;
15251        if prepared.fault == FormulaSpanDemotionFault::FinalAuthorityValidation {
15252            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15253        }
15254        {
15255            let authority = self.graph.formula_authority();
15256            if authority.plane.epoch().0 != prepared.expected_plane_epoch
15257                || authority.indexes_epoch() != prepared.expected_indexes_epoch
15258                || authority.indexed_plane_epoch() != prepared.expected_indexed_plane_epoch
15259                || prepared
15260                    .span_refs
15261                    .iter()
15262                    .any(|span_ref| authority.plane.spans.get(*span_ref).is_none())
15263            {
15264                return Err(FormulaSpanDemotionError::StaleAuthority);
15265            }
15266        }
15267        if matches!(
15268            prepared.fault,
15269            FormulaSpanDemotionFault::AllocationReservation
15270                | FormulaSpanDemotionFault::BeforeFirstMutation
15271        ) {
15272            return Err(FormulaSpanDemotionError::Injected(prepared.fault));
15273        }
15274
15275        let PreparedFormulaSpanDemotion {
15276            span_refs,
15277            placement_count,
15278            legacy_graph,
15279            ..
15280        } = prepared;
15281        let _ = self
15282            .graph
15283            .apply_prevalidated_legacy_graph_plan(legacy_graph);
15284        let authority = self.graph.formula_authority_mut();
15285        for span_ref in &span_refs {
15286            authority.plane.remove_overlays_for_source_span(*span_ref);
15287            let _ = authority.plane.remove_span(*span_ref);
15288        }
15289        let _ = authority.rebuild_indexes();
15290        self.mark_topology_edited();
15291        Ok(FormulaSpanDemotionReport {
15292            spans_demoted: span_refs.len(),
15293            placements_materialized: placement_count,
15294        })
15295    }
15296
15297    fn transition_off_mode_spans_to_legacy(&mut self) -> Result<bool, ExcelError> {
15298        if self.config.formula_plane_mode != FormulaPlaneMode::Off {
15299            return Ok(false);
15300        }
15301        let span_refs = self.graph.formula_authority().active_span_refs();
15302        if span_refs.is_empty() {
15303            return Ok(false);
15304        }
15305
15306        let formula_dirty = self.graph.lease_formula_dirty();
15307        let materialization_started = crate::instant::FzInstant::now();
15308        let map_error = |phase: &str, error: FormulaSpanDemotionError| match error {
15309            FormulaSpanDemotionError::Resource(error)
15310            | FormulaSpanDemotionError::AstPreparation(error) => error,
15311            error => ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15312                "FormulaPlane Off-mode span demotion {phase} failed: {error}"
15313            )),
15314        };
15315        let prepared = self
15316            .prepare_formula_span_demotion(&span_refs)
15317            .map_err(|error| map_error("preparation", error))?;
15318        let commit_started = self.preflight_evaluation_commit_window(prepared.placement_count)?;
15319        let report = self
15320            .commit_prepared_formula_span_demotion(prepared)
15321            .map_err(|error| map_error("commit", error))?;
15322        self.ack_formula_dirty_observed(formula_dirty);
15323        self.observe_evaluation_commit_window(commit_started);
15324        self.observe_materialization(
15325            report.placements_materialized,
15326            false,
15327            materialization_started.elapsed(),
15328        );
15329        Ok(true)
15330    }
15331
15332    #[cfg(test)]
15333    pub(crate) fn set_formula_span_demotion_fault_for_test(
15334        &mut self,
15335        fault: FormulaSpanDemotionFault,
15336    ) {
15337        self.formula_span_demotion_fault_for_test = Some(fault);
15338    }
15339
15340    #[cfg(test)]
15341    pub(crate) fn force_non_cycle_schedule_fallback_for_test(&mut self) {
15342        self.force_non_cycle_schedule_fallback_for_test = true;
15343    }
15344
15345    #[cfg(test)]
15346    pub(crate) fn mixed_topology_index_builds_for_test(&self) -> u64 {
15347        self.mixed_topology_index_builds_for_test
15348    }
15349
15350    /// Collect the [`FormulaSpanRef`]s for span producers the mixed scheduler
15351    /// reported as cycle members (gotcha G8, refs #112). These spans must be
15352    /// demoted to legacy so the cycle members are resolved on the legacy SCC
15353    /// path; see [`Self::demote_cyclic_spans`].
15354    fn collect_cyclic_span_refs(
15355        &self,
15356        schedule: &MixedSchedule,
15357        span_refs_by_id: &BTreeMap<FormulaSpanId, FormulaSpanRef>,
15358    ) -> Vec<FormulaSpanRef> {
15359        let mut refs = Vec::new();
15360        for fallback in &schedule.fallbacks {
15361            if fallback.reason != MixedScheduleFallbackReason::CycleDetected {
15362                continue;
15363            }
15364            if let FormulaProducerId::Span(span_id) = fallback.producer
15365                && let Some(span_ref) = span_refs_by_id.get(&span_id)
15366                && !refs.contains(span_ref)
15367            {
15368                refs.push(*span_ref);
15369            }
15370        }
15371        refs
15372    }
15373
15374    /// Demote the given cyclic spans to legacy graph vertices so their member
15375    /// cells participate in the legacy Tarjan SCC pass (gotcha G8, refs #112).
15376    ///
15377    /// Uses one exact-ref prepared transaction for every cyclic span reported
15378    /// by this schedule. All recoverable validation and injected faults precede
15379    /// the first graph/authority mutation, so a multi-span or multi-sheet cycle
15380    /// cannot publish only an earlier subset.
15381    fn demote_cyclic_spans(&mut self, span_refs: &[FormulaSpanRef]) -> Result<(), ExcelError> {
15382        let materialization_started = crate::instant::FzInstant::now();
15383        let prepared = self
15384            .prepare_formula_span_demotion(span_refs)
15385            .map_err(|error| {
15386                if let FormulaSpanDemotionError::Resource(error) = error {
15387                    error
15388                } else {
15389                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15390                        "FormulaPlane cycle-member span demotion preparation failed: {error}"
15391                    ))
15392                }
15393            })?;
15394        let report = self
15395            .commit_prepared_formula_span_demotion(prepared)
15396            .map_err(|error| {
15397                ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15398                    "FormulaPlane cycle-member span demotion commit failed: {error}"
15399                ))
15400            })?;
15401        self.observe_materialization(
15402            report.placements_materialized,
15403            true,
15404            materialization_started.elapsed(),
15405        );
15406        self.formula_plane_cycle_member_span_demotions = self
15407            .formula_plane_cycle_member_span_demotions
15408            .saturating_add(report.spans_demoted as u64);
15409        Self::record_shadow_fallback_reason(
15410            &mut self.formula_ingest_report_total,
15411            PlacementFallbackReason::CycleMember,
15412            report.spans_demoted as u64,
15413        );
15414        Ok(())
15415    }
15416
15417    /// Fail closed when span evaluation produces an array result (refs #388).
15418    ///
15419    /// A span owns exactly one published value per placement, but the legacy
15420    /// evaluator routes `LiteralValue::Array` results into the spill planner.
15421    /// Collapsing the array to its top-left element would broadcast a single
15422    /// value across the whole span while legacy spills a rectangle, so
15423    /// [`SpanEvaluator::evaluate_task`] refuses the conversion and the
15424    /// coordinator lands here with nothing published: the offending layer's
15425    /// `ComputedWriteBuffer` is dropped unflushed. Demote the span through the
15426    /// prepared-transaction demotion path so its placements become legacy
15427    /// vertices, then replan; the legacy pass recomputes them with correct
15428    /// spill semantics.
15429    fn demote_array_result_span(&mut self, span_ref: FormulaSpanRef) -> Result<(), ExcelError> {
15430        let materialization_started = crate::instant::FzInstant::now();
15431        let prepared = self
15432            .prepare_formula_span_demotion(&[span_ref])
15433            .map_err(|error| {
15434                if let FormulaSpanDemotionError::Resource(error) = error {
15435                    error
15436                } else {
15437                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15438                        "FormulaPlane array-result span demotion preparation failed: {error}"
15439                    ))
15440                }
15441            })?;
15442        let report = self
15443            .commit_prepared_formula_span_demotion(prepared)
15444            .map_err(|error| {
15445                ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
15446                    "FormulaPlane array-result span demotion commit failed: {error}"
15447                ))
15448            })?;
15449        self.observe_materialization(
15450            report.placements_materialized,
15451            true,
15452            materialization_started.elapsed(),
15453        );
15454        self.formula_plane_array_result_span_demotions = self
15455            .formula_plane_array_result_span_demotions
15456            .saturating_add(report.spans_demoted as u64);
15457        Self::record_shadow_fallback_reason(
15458            &mut self.formula_ingest_report_total,
15459            PlacementFallbackReason::ArrayResult,
15460            report.spans_demoted as u64,
15461        );
15462        Ok(())
15463    }
15464
15465    /// Evaluate residual *legacy-only* cyclic SCCs before the FormulaPlane
15466    /// mixed schedule runs (gotcha G8, refs #112).
15467    ///
15468    /// After cyclic spans are demoted to legacy ([`Self::demote_cyclic_spans`]),
15469    /// every cycle member is a graph vertex, so the cycle is now visible to the
15470    /// legacy Tarjan pass and lives entirely among legacy producers. The mixed
15471    /// schedule treats any cycle as not authoritative-safe; rather than abandon
15472    /// the surviving spans by falling through to a pure-legacy `evaluate_all`,
15473    /// stamp/evaluate just the cyclic SCC units here (`handle_cycle_unit` honors
15474    /// `CycleDetection::Static` vs `Runtime`), clear their dirty flags, and let
15475    /// the mixed schedule proceed cycle-free over the surviving spans plus the
15476    /// acyclic legacy work.
15477    ///
15478    /// Returns the number of cyclic SCC units that stamped at least one cell.
15479    fn evaluate_legacy_cycle_prepass(&mut self) -> Result<usize, ExcelError> {
15480        let dirty = self.graph.get_evaluation_vertices();
15481        if dirty.is_empty() {
15482            return Ok(0);
15483        }
15484        let (schedule, _vdeps, _meta) = self.create_evaluation_schedule(&dirty)?;
15485        let dirty_set: FxHashSet<VertexId> = dirty.iter().copied().collect();
15486        let mut cycle_errors = 0usize;
15487        let mut stamped_vertices: Vec<VertexId> = Vec::new();
15488        for &unit in &schedule.units {
15489            let ScheduleUnit::Cycle(i) = unit else {
15490                continue;
15491            };
15492            let members = schedule.unit_cycle(i);
15493            let stamped = self.handle_cycle_unit(members, None, Some(&dirty_set), None)?;
15494            if stamped > 0 {
15495                cycle_errors += 1;
15496            }
15497            stamped_vertices.extend(members.iter().copied());
15498        }
15499        // Clear dirty only on the cyclic members so the subsequent mixed
15500        // schedule no longer sees them as dirty legacy producers (which is what
15501        // surfaced the cycle). Acyclic legacy work stays dirty and is scheduled
15502        // normally alongside the surviving spans.
15503        if !stamped_vertices.is_empty() {
15504            self.resource_checkpoint(0)?;
15505            self.graph.clear_dirty_flags(&stamped_vertices);
15506        }
15507        Ok(cycle_errors)
15508    }
15509
15510    fn materialize_deferred_sheet_before_structural_edit(
15511        &mut self,
15512        sheet: &str,
15513    ) -> Result<(), crate::engine::EditorError> {
15514        if self.staged_formulas.contains_key(sheet) {
15515            self.build_graph_for_sheets([sheet])?;
15516        }
15517        Ok(())
15518    }
15519
15520    fn structural_row_occupancy(
15521        &self,
15522        sheet: &str,
15523        sheet_id: SheetId,
15524    ) -> crate::engine::graph::StructuralOccupancy {
15525        if !self.graph.has_compressed_range_dependencies() {
15526            return crate::engine::graph::StructuralOccupancy::default();
15527        }
15528        let mut occupancy = self.graph.structural_occupancy(sheet_id);
15529        if let Some(arrow_sheet) = self.arrow_sheets.sheet(sheet) {
15530            occupancy.include_arrow_sheet(arrow_sheet);
15531            occupancy
15532        } else {
15533            // Missing Arrow state cannot prove an apparently empty column empty.
15534            crate::engine::graph::StructuralOccupancy::conservative()
15535        }
15536    }
15537
15538    fn structural_column_occupancy(&self) -> crate::engine::graph::StructuralOccupancy {
15539        // Arrow exposes occupied columns through chunk metadata and overlay maps,
15540        // but has no cheap occupied-row index. Column edits therefore deliberately
15541        // retain conservative cross-axis invalidation instead of scanning cells.
15542        crate::engine::graph::StructuralOccupancy::conservative()
15543    }
15544
15545    /// Insert rows (1-based) and mirror into Arrow store when enabled
15546    pub fn insert_rows(
15547        &mut self,
15548        sheet: &str,
15549        before: u32,
15550        count: u32,
15551    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15552    {
15553        if count == 0 {
15554            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15555        }
15556        self.observe_function_semantic_epoch()
15557            .map_err(crate::engine::EditorError::Excel)?;
15558        use crate::engine::graph::editor::vertex_editor::VertexEditor;
15559        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15560        let sheet_id = self.ensure_known_sheet_id(sheet)?;
15561        let before0 = before.saturating_sub(1);
15562        let affected_region = Self::structural_row_region(sheet_id, before0);
15563        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
15564        let op = StructuralOp::InsertRows {
15565            sheet_id,
15566            before: before0,
15567            count,
15568        };
15569        self.demote_spans_for_structural_op(op, affected_region)?;
15570        let summary = {
15571            let mut editor =
15572                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15573            editor.insert_rows(sheet_id, before0, count)?
15574        };
15575        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15576            let before0 = before0 as usize;
15577            asheet.insert_rows(before0, count as usize);
15578        }
15579        self.purge_derived_formats_after_row(sheet_id, before0);
15580        self.mark_moved_formula_vertices_dirty(&summary);
15581        self.clear_computed_overlay_after_row(sheet, before0 as usize);
15582        self.shift_row_visibility_insert(sheet_id, before0, count);
15583        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15584        self.mark_topology_edited();
15585        Ok(summary)
15586    }
15587
15588    /// Delete rows (1-based) and mirror into Arrow store when enabled
15589    pub fn delete_rows(
15590        &mut self,
15591        sheet: &str,
15592        start: u32,
15593        count: u32,
15594    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15595    {
15596        if count == 0 {
15597            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15598        }
15599        self.observe_function_semantic_epoch()
15600            .map_err(crate::engine::EditorError::Excel)?;
15601        use crate::engine::graph::editor::vertex_editor::VertexEditor;
15602        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15603        let sheet_id = self.ensure_known_sheet_id(sheet)?;
15604        let start0 = start.saturating_sub(1);
15605        let affected_region = Self::structural_row_region(sheet_id, start0);
15606        let occupancy = self.structural_row_occupancy(sheet, sheet_id);
15607        let op = StructuralOp::DeleteRows {
15608            sheet_id,
15609            start: start0,
15610            count,
15611        };
15612        self.demote_spans_for_structural_op(op, affected_region)?;
15613        let summary = {
15614            let mut editor =
15615                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15616            editor.delete_rows(sheet_id, start0, count)?
15617        };
15618        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15619            let start0 = start0 as usize;
15620            asheet.delete_rows(start0, count as usize);
15621        }
15622        self.purge_derived_formats_after_row(sheet_id, start0);
15623        self.mark_moved_formula_vertices_dirty(&summary);
15624        self.clear_computed_overlay_after_row(sheet, start0 as usize);
15625        self.shift_row_visibility_delete(sheet_id, start0, count);
15626        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15627        self.mark_topology_edited();
15628        Ok(summary)
15629    }
15630
15631    /// Insert columns (1-based) and mirror into Arrow store when enabled
15632    pub fn insert_columns(
15633        &mut self,
15634        sheet: &str,
15635        before: u32,
15636        count: u32,
15637    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15638    {
15639        if count == 0 {
15640            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15641        }
15642        self.observe_function_semantic_epoch()
15643            .map_err(crate::engine::EditorError::Excel)?;
15644        use crate::engine::graph::editor::vertex_editor::VertexEditor;
15645        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15646        let sheet_id = self.graph.sheet_id(sheet).ok_or(
15647            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
15648                name: sheet.to_string(),
15649                reason: "Unknown sheet".to_string(),
15650            },
15651        )?;
15652        let before0 = before.saturating_sub(1);
15653        let affected_region = Self::structural_col_region(sheet_id, before0);
15654        let occupancy = self.structural_column_occupancy();
15655        let op = StructuralOp::InsertColumns {
15656            sheet_id,
15657            before: before0,
15658            count,
15659        };
15660        self.demote_spans_for_structural_op(op, affected_region)?;
15661        let summary = {
15662            let mut editor =
15663                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15664            editor.insert_columns(sheet_id, before0, count)?
15665        };
15666        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15667            let before0 = before0 as usize;
15668            asheet.insert_columns(before0, count as usize);
15669        }
15670        self.purge_derived_formats_after_col(sheet_id, before0);
15671        self.mark_moved_formula_vertices_dirty(&summary);
15672        self.clear_computed_overlay_after_col(sheet, before0 as usize);
15673        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15674        self.mark_topology_edited();
15675        Ok(summary)
15676    }
15677
15678    /// Delete columns (1-based) and mirror into Arrow store when enabled
15679    pub fn delete_columns(
15680        &mut self,
15681        sheet: &str,
15682        start: u32,
15683        count: u32,
15684    ) -> Result<crate::engine::graph::editor::vertex_editor::ShiftSummary, crate::engine::EditorError>
15685    {
15686        if count == 0 {
15687            return Ok(crate::engine::graph::editor::vertex_editor::ShiftSummary::default());
15688        }
15689        self.observe_function_semantic_epoch()
15690            .map_err(crate::engine::EditorError::Excel)?;
15691        use crate::engine::graph::editor::vertex_editor::VertexEditor;
15692        self.materialize_deferred_sheet_before_structural_edit(sheet)?;
15693        let sheet_id = self.graph.sheet_id(sheet).ok_or(
15694            crate::engine::graph::editor::vertex_editor::EditorError::InvalidName {
15695                name: sheet.to_string(),
15696                reason: "Unknown sheet".to_string(),
15697            },
15698        )?;
15699        let start0 = start.saturating_sub(1);
15700        let affected_region = Self::structural_col_region(sheet_id, start0);
15701        let occupancy = self.structural_column_occupancy();
15702        let op = StructuralOp::DeleteColumns {
15703            sheet_id,
15704            start: start0,
15705            count,
15706        };
15707        self.demote_spans_for_structural_op(op, affected_region)?;
15708        let summary = {
15709            let mut editor =
15710                VertexEditor::new(&mut self.graph).with_structural_occupancy(occupancy);
15711            editor.delete_columns(sheet_id, start0, count)?
15712        };
15713        if let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) {
15714            let start0 = start0 as usize;
15715            asheet.delete_columns(start0, count as usize);
15716        }
15717        self.purge_derived_formats_after_col(sheet_id, start0);
15718        self.mark_moved_formula_vertices_dirty(&summary);
15719        self.clear_computed_overlay_after_col(sheet, start0 as usize);
15720        self.record_formula_plane_structural_change(StructuralScope::Region(affected_region));
15721        self.mark_topology_edited();
15722        Ok(summary)
15723    }
15724    /// Arrow-backed used row bounds across a column span (1-based inclusive cols).
15725    fn arrow_used_row_bounds(
15726        &self,
15727        sheet: &str,
15728        start_col: u32,
15729        end_col: u32,
15730    ) -> Option<(u32, u32)> {
15731        let a = self.sheet_store().sheet(sheet)?;
15732        if a.columns.is_empty() {
15733            return None;
15734        }
15735        let sc0 = start_col.saturating_sub(1) as usize;
15736        let ec0 = end_col.saturating_sub(1) as usize;
15737        let col_hi = a.columns.len().saturating_sub(1);
15738        if sc0 > col_hi {
15739            return None;
15740        }
15741        let ec0 = ec0.min(col_hi);
15742        // Pass-scoped cache with snapshot guard
15743        let snap = self.data_snapshot_id();
15744        let mut min_r0: Option<usize> = None;
15745        for ci in sc0..=ec0 {
15746            let sheet_id = self.graph.sheet_id(sheet)?;
15747            if let Some((Some(mv), _)) = self.row_bounds_cache.read().ok().and_then(|g| {
15748                g.as_ref()
15749                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
15750            }) {
15751                let mv = mv as usize;
15752                min_r0 = Some(min_r0.map(|m| m.min(mv)).unwrap_or(mv));
15753                continue;
15754            }
15755            // Compute and store
15756            let (min_c, max_c) = Self::scan_column_used_bounds(a, ci);
15757            if let Ok(mut g) = self.row_bounds_cache.write() {
15758                g.get_or_insert_with(|| RowBoundsCache::new(snap))
15759                    .put_row_bounds(sheet_id, ci, snap, (min_c, max_c));
15760            }
15761            if let Some(m) = min_c {
15762                min_r0 = Some(min_r0.map(|mm| mm.min(m as usize)).unwrap_or(m as usize));
15763            }
15764        }
15765        min_r0?;
15766        let mut max_r0: Option<usize> = None;
15767        for ci in sc0..=ec0 {
15768            let sheet_id = self.graph.sheet_id(sheet)?;
15769            if let Some((_, Some(mv))) = self.row_bounds_cache.read().ok().and_then(|g| {
15770                g.as_ref()
15771                    .and_then(|c| c.get_row_bounds(sheet_id, ci, snap))
15772            }) {
15773                let mv = mv as usize;
15774                max_r0 = Some(max_r0.map(|m| m.max(mv)).unwrap_or(mv));
15775                continue;
15776            }
15777            let (_min_c, max_c) = Self::scan_column_used_bounds(a, ci);
15778            if let Ok(mut g) = self.row_bounds_cache.write() {
15779                g.get_or_insert_with(|| RowBoundsCache::new(snap))
15780                    .put_row_bounds(sheet_id, ci, snap, (_min_c, max_c));
15781            }
15782            if let Some(m) = max_c {
15783                max_r0 = Some(max_r0.map(|mm| mm.max(m as usize)).unwrap_or(m as usize));
15784            }
15785        }
15786        match (min_r0, max_r0) {
15787            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
15788            _ => None,
15789        }
15790    }
15791
15792    fn scan_column_used_bounds(
15793        a: &crate::arrow_store::ArrowSheet,
15794        ci: usize,
15795    ) -> (Option<u32>, Option<u32>) {
15796        let col = &a.columns[ci];
15797
15798        // Min: scan dense chunks first, then sparse chunks in ascending index order.
15799        let mut min_r0: Option<u32> = None;
15800        for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15801            let tags = chunk.type_tag.values();
15802            for (off, &t) in tags.iter().enumerate() {
15803                let overlay_non_empty = chunk
15804                    .overlay
15805                    .get(off)
15806                    .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15807                    .unwrap_or(false)
15808                    || chunk
15809                        .computed_overlay
15810                        .get(off)
15811                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15812                        .unwrap_or(false);
15813                if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15814                    let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15815                        break;
15816                    };
15817                    let row0 = chunk_start + off;
15818                    min_r0 = Some(row0 as u32);
15819                    break;
15820                }
15821            }
15822            if min_r0.is_some() {
15823                break;
15824            }
15825        }
15826        if min_r0.is_none() && !col.sparse_chunks.is_empty() {
15827            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
15828            sparse_idxs.sort_unstable();
15829            for chunk_idx in sparse_idxs {
15830                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
15831                    continue;
15832                };
15833                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15834                    continue;
15835                };
15836                let tags = chunk.type_tag.values();
15837                for (off, &t) in tags.iter().enumerate() {
15838                    let overlay_non_empty = chunk
15839                        .overlay
15840                        .get(off)
15841                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15842                        .unwrap_or(false)
15843                        || chunk
15844                            .computed_overlay
15845                            .get(off)
15846                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15847                            .unwrap_or(false);
15848                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15849                        let row0 = chunk_start + off;
15850                        min_r0 = Some(row0 as u32);
15851                        break;
15852                    }
15853                }
15854                if min_r0.is_some() {
15855                    break;
15856                }
15857            }
15858        }
15859
15860        // Max: scan sparse chunks in descending index order, then dense chunks in reverse.
15861        let mut max_r0: Option<u32> = None;
15862        if !col.sparse_chunks.is_empty() {
15863            let mut sparse_idxs: Vec<usize> = col.sparse_chunks.keys().copied().collect();
15864            sparse_idxs.sort_unstable_by(|a, b| b.cmp(a));
15865            for chunk_idx in sparse_idxs {
15866                let Some(chunk) = col.sparse_chunks.get(&chunk_idx) else {
15867                    continue;
15868                };
15869                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15870                    continue;
15871                };
15872                let tags = chunk.type_tag.values();
15873                for (rev_idx, &t) in tags.iter().enumerate().rev() {
15874                    let overlay_non_empty = chunk
15875                        .overlay
15876                        .get(rev_idx)
15877                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15878                        .unwrap_or(false)
15879                        || chunk
15880                            .computed_overlay
15881                            .get(rev_idx)
15882                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15883                            .unwrap_or(false);
15884                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15885                        let row0 = chunk_start + rev_idx;
15886                        max_r0 = Some(row0 as u32);
15887                        break;
15888                    }
15889                }
15890                if max_r0.is_some() {
15891                    break;
15892                }
15893            }
15894        }
15895        if max_r0.is_none() {
15896            for (chunk_idx, chunk) in col.chunks.iter().enumerate().rev() {
15897                let tags = chunk.type_tag.values();
15898                for (rev_idx, &t) in tags.iter().enumerate().rev() {
15899                    let overlay_non_empty = chunk
15900                        .overlay
15901                        .get(rev_idx)
15902                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15903                        .unwrap_or(false)
15904                        || chunk
15905                            .computed_overlay
15906                            .get(rev_idx)
15907                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15908                            .unwrap_or(false);
15909                    if overlay_non_empty || t != crate::arrow_store::TypeTag::Empty as u8 {
15910                        let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15911                            break;
15912                        };
15913                        let row0 = chunk_start + rev_idx;
15914                        max_r0 = Some(row0 as u32);
15915                        break;
15916                    }
15917                }
15918                if max_r0.is_some() {
15919                    break;
15920                }
15921            }
15922        }
15923
15924        (min_r0, max_r0)
15925    }
15926
15927    /// Arrow-backed used column bounds across a row span (1-based inclusive rows).
15928    fn arrow_used_col_bounds(
15929        &self,
15930        sheet: &str,
15931        start_row: u32,
15932        end_row: u32,
15933    ) -> Option<(u32, u32)> {
15934        let a = self.sheet_store().sheet(sheet)?;
15935        if a.columns.is_empty() {
15936            return None;
15937        }
15938        let sr0 = start_row.saturating_sub(1) as usize;
15939        let er0 = end_row.saturating_sub(1) as usize;
15940        if sr0 > er0 {
15941            return None;
15942        }
15943        // Map start/end rows into chunk ranges
15944        // We will scan each column for any non-empty within [sr0..=er0]
15945        let mut min_c0: Option<usize> = None;
15946        let mut max_c0: Option<usize> = None;
15947        // Precompute chunk bounds for row range
15948        for (ci, col) in a.columns.iter().enumerate() {
15949            let mut any_in_range = false;
15950
15951            let scan_chunk = |chunk_idx: usize, chunk: &crate::arrow_store::ColumnChunk| -> bool {
15952                let Some(&chunk_start) = a.chunk_starts.get(chunk_idx) else {
15953                    return false;
15954                };
15955                let chunk_len = chunk.type_tag.len();
15956                if chunk_len == 0 {
15957                    return false;
15958                }
15959                let chunk_end = chunk_start + chunk_len.saturating_sub(1);
15960                // check intersection
15961                if sr0 > chunk_end || er0 < chunk_start {
15962                    return false;
15963                }
15964                let start_off = sr0.max(chunk_start) - chunk_start;
15965                let end_off = er0.min(chunk_end) - chunk_start;
15966                let tags = chunk.type_tag.values();
15967                for off in start_off..=end_off {
15968                    let overlay_non_empty = chunk
15969                        .overlay
15970                        .get(off)
15971                        .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15972                        .unwrap_or(false)
15973                        || chunk
15974                            .computed_overlay
15975                            .get(off)
15976                            .map(|ov| !matches!(ov, crate::arrow_store::OverlayValue::Empty))
15977                            .unwrap_or(false);
15978                    if overlay_non_empty || tags[off] != crate::arrow_store::TypeTag::Empty as u8 {
15979                        return true;
15980                    }
15981                }
15982                false
15983            };
15984
15985            for (chunk_idx, chunk) in col.chunks.iter().enumerate() {
15986                if scan_chunk(chunk_idx, chunk) {
15987                    any_in_range = true;
15988                    break;
15989                }
15990            }
15991
15992            if !any_in_range && !col.sparse_chunks.is_empty() {
15993                for (&chunk_idx, chunk) in col.sparse_chunks.iter() {
15994                    if scan_chunk(chunk_idx, chunk) {
15995                        any_in_range = true;
15996                        break;
15997                    }
15998                }
15999            }
16000
16001            if any_in_range {
16002                min_c0 = Some(min_c0.map(|m| m.min(ci)).unwrap_or(ci));
16003                max_c0 = Some(max_c0.map(|m| m.max(ci)).unwrap_or(ci));
16004            }
16005        }
16006        match (min_c0, max_c0) {
16007            (Some(a0), Some(b0)) => Some(((a0 as u32) + 1, (b0 as u32) + 1)),
16008            _ => None,
16009        }
16010    }
16011
16012    fn formula_row_bounds_for_columns(
16013        &self,
16014        sheet: &str,
16015        start_col: u32,
16016        end_col: u32,
16017    ) -> Option<(u32, u32)> {
16018        let sheet_id = self.graph.sheet_id(sheet)?;
16019        let sc0 = start_col.saturating_sub(1);
16020        let ec0 = end_col.saturating_sub(1);
16021        let mut min_r0: Option<u32> = None;
16022        let mut max_r0: Option<u32> = None;
16023
16024        if let Some(index) = self.graph.sheet_index(sheet_id) {
16025            for vid in index.vertices_in_col_range(sc0, ec0) {
16026                if !matches!(
16027                    self.graph.get_vertex_kind(vid),
16028                    VertexKind::FormulaScalar | VertexKind::FormulaArray
16029                ) {
16030                    continue;
16031                }
16032                let Some(row0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.row()) else {
16033                    continue;
16034                };
16035                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
16036                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
16037            }
16038        } else {
16039            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
16040                if !matches!(
16041                    self.graph.get_vertex_kind(vid),
16042                    VertexKind::FormulaScalar | VertexKind::FormulaArray
16043                ) {
16044                    continue;
16045                }
16046                let col0 = coord.col();
16047                if col0 < sc0 || col0 > ec0 {
16048                    continue;
16049                }
16050                let row0 = coord.row();
16051                min_r0 = Some(min_r0.map(|m| m.min(row0)).unwrap_or(row0));
16052                max_r0 = Some(max_r0.map(|m| m.max(row0)).unwrap_or(row0));
16053            }
16054        }
16055
16056        match (min_r0, max_r0) {
16057            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
16058            _ => None,
16059        }
16060    }
16061
16062    fn formula_col_bounds_for_rows(
16063        &self,
16064        sheet: &str,
16065        start_row: u32,
16066        end_row: u32,
16067    ) -> Option<(u32, u32)> {
16068        let sheet_id = self.graph.sheet_id(sheet)?;
16069        let sr0 = start_row.saturating_sub(1);
16070        let er0 = end_row.saturating_sub(1);
16071        let mut min_c0: Option<u32> = None;
16072        let mut max_c0: Option<u32> = None;
16073
16074        if let Some(index) = self.graph.sheet_index(sheet_id) {
16075            for vid in index.vertices_in_row_range(sr0, er0) {
16076                if !matches!(
16077                    self.graph.get_vertex_kind(vid),
16078                    VertexKind::FormulaScalar | VertexKind::FormulaArray
16079                ) {
16080                    continue;
16081                }
16082                let Some(col0) = self.graph.vertex_grid_addr(vid).map(|addr| addr.col()) else {
16083                    continue;
16084                };
16085                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
16086                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
16087            }
16088        } else {
16089            for (vid, coord) in self.graph.grid_vertices_in_sheet(sheet_id) {
16090                if !matches!(
16091                    self.graph.get_vertex_kind(vid),
16092                    VertexKind::FormulaScalar | VertexKind::FormulaArray
16093                ) {
16094                    continue;
16095                }
16096                let row0 = coord.row();
16097                if row0 < sr0 || row0 > er0 {
16098                    continue;
16099                }
16100                let col0 = coord.col();
16101                min_c0 = Some(min_c0.map(|m| m.min(col0)).unwrap_or(col0));
16102                max_c0 = Some(max_c0.map(|m| m.max(col0)).unwrap_or(col0));
16103            }
16104        }
16105
16106        match (min_c0, max_c0) {
16107            (Some(a0), Some(b0)) => Some((a0 + 1, b0 + 1)),
16108            _ => None,
16109        }
16110    }
16111
16112    fn union_used_bounds(
16113        first: Option<(u32, u32)>,
16114        second: Option<(u32, u32)>,
16115    ) -> Option<(u32, u32)> {
16116        match (first, second) {
16117            (Some((a0, b0)), Some((a1, b1))) => Some((a0.min(a1), b0.max(b1))),
16118            (Some(bounds), None) | (None, Some(bounds)) => Some(bounds),
16119            (None, None) => None,
16120        }
16121    }
16122
16123    /// Mirror a single cell value into the Arrow overlay if enabled.
16124    /// Handles capacity growth, per-chunk overlay set, and heuristic compaction.
16125    fn mirror_value_to_overlay(&mut self, sheet: &str, row: u32, col: u32, value: &LiteralValue) {
16126        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16127            return;
16128        }
16129        if self.arrow_sheets.sheet(sheet).is_none() {
16130            self.arrow_sheets
16131                .sheets
16132                .push(crate::arrow_store::ArrowSheet {
16133                    name: std::sync::Arc::<str>::from(sheet),
16134                    date_system: self.config.date_system,
16135                    columns: Vec::new(),
16136                    nrows: 0,
16137                    chunk_starts: Vec::new(),
16138                    chunk_rows: 32 * 1024,
16139                });
16140        }
16141
16142        let row0 = row.saturating_sub(1) as usize;
16143        let col0 = col.saturating_sub(1) as usize;
16144
16145        let asheet = self
16146            .arrow_sheets
16147            .sheet_mut(sheet)
16148            .expect("ArrowSheet must exist");
16149
16150        let cur_cols = asheet.columns.len();
16151        if col0 >= cur_cols {
16152            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16153        }
16154
16155        if row0 >= asheet.nrows as usize {
16156            if asheet.columns.is_empty() {
16157                asheet.insert_columns(0, 1);
16158            }
16159            asheet.ensure_row_capacity(row0 + 1);
16160        }
16161        if let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) {
16162            let ov =
16163                crate::arrow_store::OverlayValue::from_literal_value(value, asheet.date_system);
16164            let computed_delta = if let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) {
16165                let _ = ch.overlay.set(in_off, ov);
16166                let format = match value {
16167                    LiteralValue::Date(_) => Some(crate::format::FormatId::DATE),
16168                    LiteralValue::DateTime(_) => Some(crate::format::FormatId::DATETIME),
16169                    LiteralValue::Time(_) => Some(crate::format::FormatId::TIME),
16170                    LiteralValue::Duration(_) => Some(crate::format::FormatId::DURATION),
16171                    _ => None,
16172                };
16173                ch.overlay.set_format(in_off, format);
16174                // A user edit must invalidate any computed (formula/spill) overlay entry at
16175                // this cell. Otherwise, if the delta overlay later compacts into the base lanes
16176                // (clearing `overlay`), a stale `computed_overlay=Empty` could incorrectly mask
16177                // the edited base value under the read cascade.
16178                ch.computed_overlay.remove(in_off)
16179            } else {
16180                return;
16181            };
16182            // Heuristic compaction: > len/50 or > 1024
16183            let abs_threshold = 1024usize;
16184            let frac_den = 50usize;
16185            let freed = asheet.maybe_compact_chunk(col0, ch_idx, abs_threshold, frac_den);
16186            if freed > 0 {
16187                self.overlay_compactions = self.overlay_compactions.saturating_add(1);
16188            }
16189            self.adjust_computed_overlay_bytes(computed_delta);
16190        }
16191    }
16192
16193    /// Remove a delta-overlay entry for a single cell (if present).
16194    ///
16195    /// This is used when transitioning a cell to a formula so that any previous user-edit overlay
16196    /// does not continue to mask computed overlay outputs.
16197    fn clear_delta_overlay_cell(&mut self, sheet: &str, row: u32, col: u32) {
16198        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16199            return;
16200        }
16201        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16202            return;
16203        };
16204        let row0 = row.saturating_sub(1) as usize;
16205        let col0 = col.saturating_sub(1) as usize;
16206        if row0 >= asheet.nrows as usize {
16207            return;
16208        }
16209        if col0 >= asheet.columns.len() {
16210            return;
16211        }
16212        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16213            return;
16214        };
16215        if let Some(ch) = asheet.columns[col0].chunk_mut(ch_idx) {
16216            let _ = ch.overlay.remove(in_off);
16217        }
16218    }
16219
16220    fn clear_computed_overlay_col_row_range(
16221        &mut self,
16222        sheet: &str,
16223        col0: usize,
16224        start_row0: usize,
16225        end_row0_exclusive: usize,
16226    ) {
16227        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16228            return;
16229        }
16230        if start_row0 >= end_row0_exclusive {
16231            return;
16232        }
16233
16234        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16235            return;
16236        };
16237        if col0 >= asheet.columns.len() || start_row0 >= asheet.nrows as usize {
16238            return;
16239        }
16240        let end_row0_exclusive = end_row0_exclusive.min(asheet.nrows as usize);
16241        if start_row0 >= end_row0_exclusive {
16242            return;
16243        }
16244
16245        let starts = asheet.chunk_starts.clone();
16246        let nrows = asheet.nrows as usize;
16247        let mut delta = 0isize;
16248        let Some(col) = asheet.columns.get_mut(col0) else {
16249            return;
16250        };
16251        for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
16252            let Some(&chunk_start) = starts.get(chunk_idx) else {
16253                continue;
16254            };
16255            let chunk_end = starts
16256                .get(chunk_idx + 1)
16257                .copied()
16258                .unwrap_or(nrows)
16259                .min(chunk_start.saturating_add(ch.len()));
16260            let clear_start = start_row0.max(chunk_start);
16261            let clear_end = end_row0_exclusive.min(chunk_end);
16262            if clear_start >= clear_end {
16263                continue;
16264            }
16265            if clear_start == chunk_start && clear_end == chunk_end {
16266                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16267            } else {
16268                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
16269                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
16270                delta = delta.saturating_add(
16271                    ch.computed_overlay
16272                        .remove_range(start_in_chunk..end_in_chunk),
16273                );
16274            }
16275        }
16276        for (chunk_idx, ch) in &mut col.sparse_chunks {
16277            let Some(&chunk_start) = starts.get(*chunk_idx) else {
16278                continue;
16279            };
16280            let chunk_end = starts
16281                .get(*chunk_idx + 1)
16282                .copied()
16283                .unwrap_or(nrows)
16284                .min(chunk_start.saturating_add(ch.len()));
16285            let clear_start = start_row0.max(chunk_start);
16286            let clear_end = end_row0_exclusive.min(chunk_end);
16287            if clear_start >= clear_end {
16288                continue;
16289            }
16290            if clear_start == chunk_start && clear_end == chunk_end {
16291                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16292            } else {
16293                let start_in_chunk = clear_start.saturating_sub(chunk_start).min(ch.len());
16294                let end_in_chunk = clear_end.saturating_sub(chunk_start).min(ch.len());
16295                delta = delta.saturating_add(
16296                    ch.computed_overlay
16297                        .remove_range(start_in_chunk..end_in_chunk),
16298                );
16299            }
16300        }
16301        self.adjust_computed_overlay_bytes(delta);
16302    }
16303
16304    fn clear_computed_overlay_cells_in_region(
16305        &mut self,
16306        cells: &[(SheetId, u32, u32)],
16307        affected_region: &Region,
16308    ) {
16309        let mut by_col: BTreeMap<(SheetId, u32), Vec<u32>> = BTreeMap::new();
16310        for (formula_sheet_id, row, col) in cells {
16311            let row0 = row.saturating_sub(1);
16312            let col0 = col.saturating_sub(1);
16313            let placement_region = Region::point(*formula_sheet_id, row0, col0);
16314            if placement_region.intersects(affected_region) {
16315                by_col
16316                    .entry((*formula_sheet_id, col0))
16317                    .or_default()
16318                    .push(row0);
16319            }
16320        }
16321
16322        for ((formula_sheet_id, col0), mut rows) in by_col {
16323            rows.sort_unstable();
16324            rows.dedup();
16325            let sheet_name = self.graph.sheet_name(formula_sheet_id).to_string();
16326            let mut start = rows[0];
16327            let mut prev = rows[0];
16328            for row in rows.into_iter().skip(1) {
16329                if row == prev.saturating_add(1) {
16330                    prev = row;
16331                    continue;
16332                }
16333                self.clear_computed_overlay_col_row_range(
16334                    &sheet_name,
16335                    col0 as usize,
16336                    start as usize,
16337                    prev.saturating_add(1) as usize,
16338                );
16339                start = row;
16340                prev = row;
16341            }
16342            self.clear_computed_overlay_col_row_range(
16343                &sheet_name,
16344                col0 as usize,
16345                start as usize,
16346                prev.saturating_add(1) as usize,
16347            );
16348        }
16349    }
16350
16351    fn clear_computed_overlay_after_row(&mut self, sheet: &str, start_row0: usize) {
16352        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16353            return;
16354        }
16355
16356        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16357            return;
16358        };
16359        if start_row0 >= asheet.nrows as usize {
16360            return;
16361        }
16362
16363        let starts = asheet.chunk_starts.clone();
16364        let nrows = asheet.nrows as usize;
16365        let mut delta = 0isize;
16366        for col in &mut asheet.columns {
16367            for (chunk_idx, ch) in col.chunks.iter_mut().enumerate() {
16368                let Some(&chunk_start) = starts.get(chunk_idx) else {
16369                    continue;
16370                };
16371                let chunk_end = starts
16372                    .get(chunk_idx + 1)
16373                    .copied()
16374                    .unwrap_or(nrows)
16375                    .min(chunk_start.saturating_add(ch.len()));
16376                if chunk_end <= start_row0 {
16377                    continue;
16378                }
16379                if chunk_start >= start_row0 {
16380                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16381                } else {
16382                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
16383                    delta = delta
16384                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
16385                }
16386            }
16387
16388            for (chunk_idx, ch) in &mut col.sparse_chunks {
16389                let Some(&chunk_start) = starts.get(*chunk_idx) else {
16390                    continue;
16391                };
16392                let chunk_end = starts
16393                    .get(*chunk_idx + 1)
16394                    .copied()
16395                    .unwrap_or(nrows)
16396                    .min(chunk_start.saturating_add(ch.len()));
16397                if chunk_end <= start_row0 {
16398                    continue;
16399                }
16400                if chunk_start >= start_row0 {
16401                    delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16402                } else {
16403                    let start_in_chunk = start_row0.saturating_sub(chunk_start).min(ch.len());
16404                    delta = delta
16405                        .saturating_add(ch.computed_overlay.remove_range(start_in_chunk..ch.len()));
16406                }
16407            }
16408        }
16409        self.adjust_computed_overlay_bytes(delta);
16410    }
16411
16412    fn clear_computed_overlay_after_col(&mut self, sheet: &str, start_col0: usize) {
16413        if !(self.config.arrow_storage_enabled && self.config.write_formula_overlay_enabled) {
16414            return;
16415        }
16416
16417        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
16418            return;
16419        };
16420        if start_col0 >= asheet.columns.len() {
16421            return;
16422        }
16423
16424        let mut delta = 0isize;
16425        for col in asheet.columns.iter_mut().skip(start_col0) {
16426            for ch in &mut col.chunks {
16427                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16428            }
16429            for ch in col.sparse_chunks.values_mut() {
16430                delta = delta.saturating_sub(ch.computed_overlay.clear() as isize);
16431            }
16432        }
16433        self.adjust_computed_overlay_bytes(delta);
16434    }
16435
16436    #[inline]
16437    fn literal_to_overlay_value(
16438        value: &LiteralValue,
16439        date_system: crate::engine::DateSystem,
16440    ) -> crate::arrow_store::OverlayValue {
16441        crate::arrow_store::OverlayValue::from_literal_value(value, date_system)
16442    }
16443
16444    fn arrow_sheet_date_system(&self, sheet: &str) -> crate::engine::DateSystem {
16445        self.arrow_sheets
16446            .sheet(sheet)
16447            .map(|sheet| sheet.date_system)
16448            .unwrap_or(self.config.date_system)
16449    }
16450
16451    /// Read a single cell's delta overlay entry (if present), preserving the distinction between
16452    /// absent and explicit `Empty`.
16453    fn read_delta_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16454        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16455            return None;
16456        }
16457        let asheet = self.arrow_sheets.sheet(sheet)?;
16458        let row0 = row.saturating_sub(1) as usize;
16459        let col0 = col.saturating_sub(1) as usize;
16460        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
16461            return None;
16462        }
16463        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
16464        let ch = asheet.columns[col0].chunk(ch_idx)?;
16465        ch.overlay
16466            .get_scalar(in_off)
16467            .map(|ov| ov.to_literal_for(asheet.date_system))
16468    }
16469
16470    /// Read a single cell's computed overlay entry (if present), preserving the distinction
16471    /// between absent and explicit `Empty`.
16472    fn read_computed_overlay_cell(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
16473        if !(self.config.arrow_storage_enabled
16474            && self.config.delta_overlay_enabled
16475            && self.config.write_formula_overlay_enabled)
16476        {
16477            return None;
16478        }
16479        let asheet = self.arrow_sheets.sheet(sheet)?;
16480        let row0 = row.saturating_sub(1) as usize;
16481        let col0 = col.saturating_sub(1) as usize;
16482        if row0 >= asheet.nrows as usize || col0 >= asheet.columns.len() {
16483            return None;
16484        }
16485        let (ch_idx, in_off) = asheet.chunk_of_row(row0)?;
16486        let ch = asheet.columns[col0].chunk(ch_idx)?;
16487        ch.computed_overlay
16488            .get_scalar(in_off)
16489            .map(|ov| ov.to_literal_for(asheet.date_system))
16490    }
16491
16492    fn set_delta_overlay_cell_raw(
16493        &mut self,
16494        sheet: &str,
16495        row: u32,
16496        col: u32,
16497        value: Option<LiteralValue>,
16498    ) {
16499        if !(self.config.arrow_storage_enabled && self.config.delta_overlay_enabled) {
16500            return;
16501        }
16502
16503        self.ensure_arrow_sheet(sheet);
16504        let date_system = self.arrow_sheet_date_system(sheet);
16505        let ov_opt = value
16506            .as_ref()
16507            .map(|value| Self::literal_to_overlay_value(value, date_system));
16508        let row0 = row.saturating_sub(1) as usize;
16509        let col0 = col.saturating_sub(1) as usize;
16510        let asheet = self
16511            .arrow_sheets
16512            .sheet_mut(sheet)
16513            .expect("ArrowSheet must exist");
16514
16515        let cur_cols = asheet.columns.len();
16516        if col0 >= cur_cols {
16517            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16518        }
16519        if row0 >= asheet.nrows as usize {
16520            if asheet.columns.is_empty() {
16521                asheet.insert_columns(0, 1);
16522            }
16523            asheet.ensure_row_capacity(row0 + 1);
16524        }
16525
16526        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16527            return;
16528        };
16529        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
16530            return;
16531        };
16532
16533        if let Some(ov) = ov_opt {
16534            let _ = ch.overlay.set(in_off, ov);
16535        } else {
16536            let _ = ch.overlay.remove(in_off);
16537        }
16538    }
16539
16540    fn set_computed_overlay_cell_raw(
16541        &mut self,
16542        sheet: &str,
16543        row: u32,
16544        col: u32,
16545        value: Option<LiteralValue>,
16546    ) {
16547        if !(self.config.arrow_storage_enabled
16548            && self.config.delta_overlay_enabled
16549            && self.config.write_formula_overlay_enabled)
16550        {
16551            return;
16552        }
16553
16554        self.ensure_arrow_sheet(sheet);
16555        let date_system = self.arrow_sheet_date_system(sheet);
16556        let ov_opt = value
16557            .as_ref()
16558            .map(|value| Self::literal_to_overlay_value(value, date_system));
16559        let row0 = row.saturating_sub(1) as usize;
16560        let col0 = col.saturating_sub(1) as usize;
16561        let asheet = self
16562            .arrow_sheets
16563            .sheet_mut(sheet)
16564            .expect("ArrowSheet must exist");
16565
16566        let cur_cols = asheet.columns.len();
16567        if col0 >= cur_cols {
16568            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
16569        }
16570        if row0 >= asheet.nrows as usize {
16571            if asheet.columns.is_empty() {
16572                asheet.insert_columns(0, 1);
16573            }
16574            asheet.ensure_row_capacity(row0 + 1);
16575        }
16576
16577        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
16578            return;
16579        };
16580        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
16581            return;
16582        };
16583
16584        let delta = if let Some(ov) = ov_opt {
16585            ch.computed_overlay.set(in_off, ov)
16586        } else {
16587            ch.computed_overlay.remove(in_off)
16588        };
16589        self.adjust_computed_overlay_bytes(delta);
16590    }
16591
16592    fn apply_arrow_undo_batch(&mut self, batch: &crate::engine::ArrowUndoBatch, undo: bool) {
16593        use crate::engine::ArrowOp;
16594
16595        let iter: Box<dyn Iterator<Item = &ArrowOp>> = if undo {
16596            Box::new(batch.ops.iter().rev())
16597        } else {
16598            Box::new(batch.ops.iter())
16599        };
16600
16601        for op in iter {
16602            match op {
16603                ArrowOp::SetDeltaCell {
16604                    sheet_id,
16605                    row0,
16606                    col0,
16607                    old,
16608                    new,
16609                } => {
16610                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
16611                    let v = if undo { old.clone() } else { new.clone() };
16612                    self.set_delta_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
16613                }
16614                ArrowOp::SetComputedCell {
16615                    sheet_id,
16616                    row0,
16617                    col0,
16618                    old,
16619                    new,
16620                } => {
16621                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
16622                    let v = if undo { old.clone() } else { new.clone() };
16623                    self.set_computed_overlay_cell_raw(&sheet, row0 + 1, col0 + 1, v);
16624                }
16625                ArrowOp::RestoreComputedRect {
16626                    sheet_id,
16627                    sr0,
16628                    sc0,
16629                    er0,
16630                    ec0,
16631                    old,
16632                    new,
16633                } => {
16634                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
16635                    let vals = if undo { old } else { new };
16636                    let height = (*er0).saturating_sub(*sr0) as usize + 1;
16637                    let width = (*ec0).saturating_sub(*sc0) as usize + 1;
16638                    for r in 0..height {
16639                        for c in 0..width {
16640                            let v = vals
16641                                .get(r)
16642                                .and_then(|row| row.get(c))
16643                                .cloned()
16644                                .unwrap_or(LiteralValue::Empty);
16645                            self.set_computed_overlay_cell_raw(
16646                                &sheet,
16647                                *sr0 + 1 + r as u32,
16648                                *sc0 + 1 + c as u32,
16649                                Some(v),
16650                            );
16651                        }
16652                    }
16653                }
16654                ArrowOp::InsertRows {
16655                    sheet_id,
16656                    before0,
16657                    count,
16658                } => {
16659                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
16660                    self.ensure_arrow_sheet(&sheet);
16661                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
16662                        if undo {
16663                            asheet.delete_rows(*before0 as usize, *count as usize);
16664                        } else {
16665                            asheet.insert_rows(*before0 as usize, *count as usize);
16666                        }
16667                    }
16668                    self.purge_derived_formats_after_row(*sheet_id, *before0);
16669                }
16670                ArrowOp::InsertCols {
16671                    sheet_id,
16672                    before0,
16673                    count,
16674                } => {
16675                    let sheet = self.graph.sheet_name(*sheet_id).to_string();
16676                    self.ensure_arrow_sheet(&sheet);
16677                    if let Some(asheet) = self.arrow_sheets.sheet_mut(&sheet) {
16678                        if undo {
16679                            asheet.delete_columns(*before0 as usize, *count as usize);
16680                        } else {
16681                            asheet.insert_columns(*before0 as usize, *count as usize);
16682                        }
16683                    }
16684                    self.purge_derived_formats_after_col(*sheet_id, *before0);
16685                }
16686            }
16687        }
16688    }
16689
16690    fn record_spill_ops_into_arrow_undo(
16691        &mut self,
16692        undo: &mut crate::engine::ArrowUndoBatch,
16693        events: &[crate::engine::ChangeEvent],
16694    ) {
16695        use crate::engine::ChangeEvent;
16696        use formualizer_common::LiteralValue;
16697
16698        #[allow(clippy::type_complexity)]
16699        let rect_from_snapshot =
16700            |snap: &crate::engine::graph::editor::change_log::SpillSnapshot|
16701             -> Option<(SheetId, u32, u32, u32, u32, Vec<Vec<LiteralValue>>)> {
16702                if snap.target_cells.is_empty() {
16703                    return None;
16704                }
16705                let sheet_id = snap.target_cells[0].sheet_id;
16706                let sr0 = snap.target_cells[0].coord.row();
16707                let sc0 = snap.target_cells[0].coord.col();
16708                if snap.values.is_empty() || snap.values[0].is_empty() {
16709                    return None;
16710                }
16711                let h = snap.values.len() as u32;
16712                let w = snap.values[0].len() as u32;
16713                let er0 = sr0.saturating_add(h.saturating_sub(1));
16714                let ec0 = sc0.saturating_add(w.saturating_sub(1));
16715                Some((sheet_id, sr0, sc0, er0, ec0, snap.values.clone()))
16716            };
16717
16718        for ev in events {
16719            match ev {
16720                ChangeEvent::SpillCommitted { old, new, .. } => {
16721                    if let Some((sid, sr0, sc0, er0, ec0, new_vals)) = rect_from_snapshot(new) {
16722                        let old_vals = if let Some(old_snap) = old {
16723                            rect_from_snapshot(old_snap)
16724                                .map(|(_, _, _, _, _, v)| v)
16725                                .unwrap_or_else(|| {
16726                                    vec![
16727                                        vec![LiteralValue::Empty; new_vals[0].len()];
16728                                        new_vals.len()
16729                                    ]
16730                                })
16731                        } else {
16732                            vec![vec![LiteralValue::Empty; new_vals[0].len()]; new_vals.len()]
16733                        };
16734                        undo.record_restore_computed_rect(
16735                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
16736                        );
16737                    }
16738                }
16739                ChangeEvent::SpillCleared { old, .. } => {
16740                    if let Some((sid, sr0, sc0, er0, ec0, old_vals)) = rect_from_snapshot(old) {
16741                        let new_vals =
16742                            vec![vec![LiteralValue::Empty; old_vals[0].len()]; old_vals.len()];
16743                        undo.record_restore_computed_rect(
16744                            sid, sr0, sc0, er0, ec0, old_vals, new_vals,
16745                        );
16746                    }
16747                }
16748                _ => {}
16749            }
16750        }
16751    }
16752
16753    /// Mirror a value into the computed overlay (formula/spill outputs).
16754    ///
16755    /// This path is subject to `EvalConfig.max_overlay_memory_bytes`.
16756    /// If the cap is exceeded, computed overlays are compacted into base lanes.
16757    fn mirror_value_to_computed_overlay(
16758        &mut self,
16759        sheet: &str,
16760        row: u32,
16761        col: u32,
16762        value: &LiteralValue,
16763    ) {
16764        if !(self.config.arrow_storage_enabled
16765            && self.config.delta_overlay_enabled
16766            && self.config.write_formula_overlay_enabled)
16767        {
16768            return;
16769        }
16770        if self.computed_overlay_mirroring_disabled {
16771            return;
16772        }
16773
16774        let date_system = self.arrow_sheet_date_system(sheet);
16775        let ov = Self::literal_to_overlay_value(value, date_system);
16776        self.write_computed_overlay_value_0based(
16777            sheet,
16778            row.saturating_sub(1),
16779            col.saturating_sub(1),
16780            ov,
16781        );
16782    }
16783
16784    fn record_derived_format(&self, vertex_id: VertexId, format: Option<crate::format::FormatId>) {
16785        if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
16786            self.record_derived_format_at(cell, format);
16787        }
16788    }
16789
16790    fn record_derived_format_at(&self, cell: CellRef, format: Option<crate::format::FormatId>) {
16791        #[cfg(test)]
16792        self.derived_format_operations_for_test
16793            .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
16794        let mut formats = self.derived_formats.write().unwrap();
16795        match format.filter(|id| *id != crate::format::FormatId::GENERAL) {
16796            Some(format) => {
16797                formats.insert(cell, format);
16798            }
16799            None => {
16800                formats.remove(&cell);
16801            }
16802        }
16803    }
16804
16805    fn prepare_formula_plane_derived_format_sync(&self) -> Option<DerivedFormatPlaneSyncPlan> {
16806        let plane_epoch = self.graph.formula_authority().plane.epoch().0;
16807        if self.derived_formats_plane_epoch == plane_epoch {
16808            return None;
16809        }
16810
16811        if self.derived_formats.read().unwrap().is_empty() {
16812            return Some(DerivedFormatPlaneSyncPlan {
16813                plane_epoch,
16814                active_regions: Vec::new(),
16815                overlay_regions: Vec::new(),
16816            });
16817        }
16818
16819        let authority = self.graph.formula_authority();
16820        let active_regions = authority
16821            .plane
16822            .spans
16823            .active_spans()
16824            .map(|span| {
16825                crate::formula_plane::region_index::Region::from_domain(span.result_region.domain())
16826            })
16827            .collect();
16828        let overlay_regions = authority
16829            .plane
16830            .formula_overlay
16831            .active_entries()
16832            .map(|(entry, _)| {
16833                crate::formula_plane::region_index::Region::from_domain(&entry.domain)
16834            })
16835            .collect();
16836        Some(DerivedFormatPlaneSyncPlan {
16837            plane_epoch,
16838            active_regions,
16839            overlay_regions,
16840        })
16841    }
16842
16843    fn commit_formula_plane_derived_format_sync(&mut self, plan: DerivedFormatPlaneSyncPlan) {
16844        let DerivedFormatPlaneSyncPlan {
16845            plane_epoch,
16846            active_regions,
16847            overlay_regions,
16848        } = plan;
16849        if !active_regions.is_empty() {
16850            self.derived_formats.write().unwrap().retain(|cell, _| {
16851                let key = crate::formula_plane::region_index::RegionKey::new(
16852                    cell.sheet_id,
16853                    cell.coord.row(),
16854                    cell.coord.col(),
16855                );
16856                let plane_owned = active_regions.iter().any(|region| region.contains_key(key))
16857                    && overlay_regions
16858                        .iter()
16859                        .all(|region| !region.contains_key(key));
16860                !plane_owned
16861            });
16862        }
16863        self.derived_formats_plane_epoch = plane_epoch;
16864
16865        #[cfg(debug_assertions)]
16866        {
16867            let formats = self.derived_formats.read().unwrap();
16868            debug_assert!(formats.keys().all(|cell| {
16869                let key = crate::formula_plane::region_index::RegionKey::new(
16870                    cell.sheet_id,
16871                    cell.coord.row(),
16872                    cell.coord.col(),
16873                );
16874                active_regions
16875                    .iter()
16876                    .all(|region| !region.contains_key(key))
16877                    || overlay_regions
16878                        .iter()
16879                        .any(|region| region.contains_key(key))
16880            }));
16881        }
16882    }
16883
16884    fn clear_cell_format_state(&mut self, sheet: &str, cell: CellRef) {
16885        self.derived_formats.write().unwrap().remove(&cell);
16886        self.derived_format_results
16887            .write()
16888            .unwrap()
16889            .retain(|vertex, _| self.graph.get_cell_ref(*vertex) != Some(cell));
16890        if let Some(arrow) = self.arrow_sheets.sheet_mut(sheet) {
16891            arrow.clear_format(cell.coord.row() as usize, cell.coord.col() as usize);
16892        }
16893    }
16894
16895    fn clear_logged_cell_format_states(&mut self, events: &[ChangeEvent]) {
16896        let cells = events
16897            .iter()
16898            .filter_map(|event| match event {
16899                ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
16900                    Some(*addr)
16901                }
16902                _ => None,
16903            })
16904            .collect::<FxHashSet<_>>();
16905        for cell in cells {
16906            let sheet = self.graph.sheet_name(cell.sheet_id).to_string();
16907            self.clear_cell_format_state(&sheet, cell);
16908        }
16909    }
16910
16911    fn purge_derived_formats_after_row(&mut self, sheet_id: SheetId, start0: u32) {
16912        self.derived_formats
16913            .write()
16914            .unwrap()
16915            .retain(|cell, _| cell.sheet_id != sheet_id || cell.coord.row() < start0);
16916    }
16917
16918    fn purge_derived_formats_after_col(&mut self, sheet_id: SheetId, start0: u32) {
16919        self.derived_formats
16920            .write()
16921            .unwrap()
16922            .retain(|cell, _| cell.sheet_id != sheet_id || cell.coord.col() < start0);
16923    }
16924
16925    fn purge_derived_formats_for_sheet(&mut self, sheet_id: SheetId) {
16926        self.derived_formats
16927            .write()
16928            .unwrap()
16929            .retain(|cell, _| cell.sheet_id != sheet_id);
16930    }
16931
16932    #[cfg(test)]
16933    pub(crate) fn debug_computed_overlay_format_0based(
16934        &self,
16935        sheet: &str,
16936        row0: u32,
16937        col0: u32,
16938    ) -> Option<crate::format::FormatId> {
16939        let sheet = self.arrow_sheets.sheet(sheet)?;
16940        let (chunk_idx, row_in_chunk) = sheet.chunk_of_row(row0 as usize)?;
16941        sheet
16942            .columns
16943            .get(col0 as usize)?
16944            .chunk(chunk_idx)?
16945            .computed_overlay
16946            .get_format(row_in_chunk)
16947    }
16948
16949    #[cfg(test)]
16950    pub(crate) fn debug_computed_overlay_chunk_has_formats_0based(
16951        &self,
16952        sheet: &str,
16953        row0: u32,
16954        col0: u32,
16955    ) -> bool {
16956        let Some(sheet) = self.arrow_sheets.sheet(sheet) else {
16957            return false;
16958        };
16959        let Some((chunk_idx, _)) = sheet.chunk_of_row(row0 as usize) else {
16960            return false;
16961        };
16962        sheet
16963            .columns
16964            .get(col0 as usize)
16965            .and_then(|column| column.chunk(chunk_idx))
16966            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
16967    }
16968
16969    #[cfg(test)]
16970    pub(crate) fn debug_clear_derived_format_0based(&mut self, sheet: &str, row0: u32, col0: u32) {
16971        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
16972            self.derived_formats
16973                .write()
16974                .unwrap()
16975                .remove(&CellRef::new_absolute(sheet_id, row0, col0));
16976        }
16977    }
16978
16979    #[cfg(test)]
16980    pub(crate) fn debug_record_derived_format_0based(
16981        &self,
16982        sheet: &str,
16983        row0: u32,
16984        col0: u32,
16985        format: Option<crate::format::FormatId>,
16986    ) {
16987        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
16988            self.record_derived_format_at(CellRef::new_absolute(sheet_id, row0, col0), format);
16989        }
16990    }
16991
16992    #[cfg(test)]
16993    pub(crate) fn debug_derived_format_0based(
16994        &self,
16995        sheet: &str,
16996        row0: u32,
16997        col0: u32,
16998    ) -> Option<crate::format::FormatId> {
16999        let sheet_id = self.graph.sheet_id(sheet)?;
17000        self.derived_formats
17001            .read()
17002            .unwrap()
17003            .get(&CellRef::new_absolute(sheet_id, row0, col0))
17004            .copied()
17005    }
17006
17007    #[cfg(test)]
17008    pub(crate) fn debug_reset_format_write_operation_counts(&mut self) {
17009        self.derived_format_operations_for_test
17010            .store(0, std::sync::atomic::Ordering::Relaxed);
17011        self.computed_overlay_set_explicit_entry_operations_for_test = 0;
17012        self.computed_overlay_stale_clear_range_effects_for_test = 0;
17013        self.computed_overlay_stale_clear_offset_attempts_for_test = 0;
17014        self.computed_format_vector_allocations_for_test
17015            .store(0, std::sync::atomic::Ordering::Relaxed);
17016    }
17017
17018    #[cfg(test)]
17019    pub(crate) fn debug_format_write_operation_counts(&self) -> (u64, u64, u64, u64, u64) {
17020        (
17021            self.derived_format_operations_for_test
17022                .load(std::sync::atomic::Ordering::Relaxed),
17023            self.computed_overlay_set_explicit_entry_operations_for_test,
17024            self.computed_format_vector_allocations_for_test
17025                .load(std::sync::atomic::Ordering::Relaxed),
17026            self.computed_overlay_stale_clear_range_effects_for_test,
17027            self.computed_overlay_stale_clear_offset_attempts_for_test,
17028        )
17029    }
17030
17031    fn write_computed_overlay_format_0based(
17032        &mut self,
17033        sheet: &str,
17034        row0: u32,
17035        col0: u32,
17036        format: Option<crate::format::FormatId>,
17037    ) {
17038        self.ensure_arrow_sheet(sheet);
17039        let (row0, col0) = (row0 as usize, col0 as usize);
17040        let Some(asheet) = self.arrow_sheets.sheet_mut(sheet) else {
17041            return;
17042        };
17043        if col0 >= asheet.columns.len() {
17044            asheet.insert_columns(asheet.columns.len(), col0 + 1 - asheet.columns.len());
17045        }
17046        if row0 >= asheet.nrows as usize {
17047            asheet.ensure_row_capacity(row0 + 1);
17048        }
17049        let Some((chunk, offset)) = asheet.chunk_of_row(row0) else {
17050            return;
17051        };
17052        if let Some(chunk) = asheet.ensure_column_chunk_mut(col0, chunk) {
17053            chunk.computed_overlay.set_format(offset, format);
17054        }
17055    }
17056
17057    fn write_computed_overlay_value_0based(
17058        &mut self,
17059        sheet: &str,
17060        row0: u32,
17061        col0: u32,
17062        value: OverlayValue,
17063    ) {
17064        if !(self.config.arrow_storage_enabled
17065            && self.config.delta_overlay_enabled
17066            && self.config.write_formula_overlay_enabled)
17067        {
17068            return;
17069        }
17070        if self.computed_overlay_mirroring_disabled {
17071            return;
17072        }
17073
17074        self.ensure_arrow_sheet(sheet);
17075
17076        let row0 = row0 as usize;
17077        let col0 = col0 as usize;
17078        let asheet = self
17079            .arrow_sheets
17080            .sheet_mut(sheet)
17081            .expect("ArrowSheet must exist");
17082
17083        let cur_cols = asheet.columns.len();
17084        if col0 >= cur_cols {
17085            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
17086        }
17087
17088        if row0 >= asheet.nrows as usize {
17089            if asheet.columns.is_empty() {
17090                asheet.insert_columns(0, 1);
17091            }
17092            asheet.ensure_row_capacity(row0 + 1);
17093        }
17094
17095        let Some((ch_idx, in_off)) = asheet.chunk_of_row(row0) else {
17096            return;
17097        };
17098        let Some(ch) = asheet.ensure_column_chunk_mut(col0, ch_idx) else {
17099            return;
17100        };
17101
17102        let delta = ch.computed_overlay.set_scalar(in_off, value);
17103        self.adjust_computed_overlay_bytes(delta);
17104
17105        if let Some(cap) = self.config.max_overlay_memory_bytes
17106            && self.computed_overlay_bytes_estimate > cap
17107        {
17108            self.disable_computed_overlay_mirroring_due_to_budget(cap);
17109        }
17110    }
17111
17112    pub(crate) fn plan_computed_write_coalescing(
17113        &self,
17114        buffer: &ComputedWriteBuffer,
17115    ) -> ComputedWriteCoalescingPlan {
17116        self.plan_computed_write_coalescing_from_writes(
17117            buffer.writes().iter().cloned(),
17118            buffer.formats_present,
17119        )
17120    }
17121
17122    fn plan_owned_computed_write_coalescing(
17123        &self,
17124        writes: Vec<ComputedWrite>,
17125        formats_present: bool,
17126    ) -> ComputedWriteCoalescingPlan {
17127        self.plan_computed_write_coalescing_from_writes(writes, formats_present)
17128    }
17129
17130    fn plan_computed_write_coalescing_from_writes(
17131        &self,
17132        writes: impl IntoIterator<Item = ComputedWrite>,
17133        formats_present: bool,
17134    ) -> ComputedWriteCoalescingPlan {
17135        let mut groups: BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>> =
17136            BTreeMap::new();
17137        let mut input_cells = 0usize;
17138
17139        for write in writes {
17140            match write {
17141                ComputedWrite::Cell {
17142                    seq,
17143                    sheet_id,
17144                    row0,
17145                    col0,
17146                    value,
17147                    format_id,
17148                } => {
17149                    input_cells = input_cells.saturating_add(1);
17150                    self.push_computed_write_plan_entry(
17151                        &mut groups,
17152                        seq,
17153                        sheet_id,
17154                        row0,
17155                        col0,
17156                        value,
17157                        format_id,
17158                    );
17159                }
17160                ComputedWrite::Rect {
17161                    seq,
17162                    sheet_id,
17163                    sr0,
17164                    sc0,
17165                    values,
17166                } => {
17167                    for (r_off, row) in values.into_iter().enumerate() {
17168                        for (c_off, value) in row.into_iter().enumerate() {
17169                            input_cells = input_cells.saturating_add(1);
17170                            self.push_computed_write_plan_entry(
17171                                &mut groups,
17172                                seq,
17173                                sheet_id,
17174                                sr0.saturating_add(r_off as u32),
17175                                sc0.saturating_add(c_off as u32),
17176                                value,
17177                                None,
17178                            );
17179                        }
17180                    }
17181                }
17182            }
17183        }
17184
17185        let mut plan = ComputedWriteCoalescingPlan {
17186            chunks: Vec::with_capacity(groups.len()),
17187            input_cells,
17188            coalesced_cells: 0,
17189            overwritten_cells: 0,
17190        };
17191        for (key, entries) in groups {
17192            let computed_lane_has_formats =
17193                self.computed_overlay_chunk_has_formats(key.sheet_id, key.col0, key.chunk_idx);
17194            let (chunk_plan, overwritten) = ComputedWriteChunkPlan::from_group(
17195                key,
17196                entries,
17197                formats_present,
17198                computed_lane_has_formats,
17199            );
17200            #[cfg(test)]
17201            if matches!(
17202                &chunk_plan.format_effect,
17203                ComputedWriteChunkFormatEffect::SetExplicit(_)
17204            ) {
17205                self.computed_format_vector_allocations_for_test
17206                    .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
17207            }
17208            plan.coalesced_cells = plan
17209                .coalesced_cells
17210                .saturating_add(chunk_plan.entries.len());
17211            plan.overwritten_cells = plan.overwritten_cells.saturating_add(overwritten);
17212            plan.chunks.push(chunk_plan);
17213        }
17214        debug_assert_eq!(
17215            plan.input_cells,
17216            plan.coalesced_cells.saturating_add(plan.overwritten_cells)
17217        );
17218        plan
17219    }
17220
17221    fn push_computed_write_plan_entry(
17222        &self,
17223        groups: &mut BTreeMap<ComputedWriteChunkKey, Vec<ComputedWriteChunkEntryPlan>>,
17224        seq: u64,
17225        sheet_id: SheetId,
17226        row0: u32,
17227        col0: u32,
17228        value: OverlayValue,
17229        format_id: Option<crate::format::FormatId>,
17230    ) {
17231        let (chunk_idx, chunk_start_row0, row_in_chunk) =
17232            self.locate_computed_write_chunk(sheet_id, row0);
17233        let key = ComputedWriteChunkKey {
17234            sheet_id,
17235            col0,
17236            chunk_idx,
17237            chunk_start_row0,
17238        };
17239        groups
17240            .entry(key)
17241            .or_default()
17242            .push(ComputedWriteChunkEntryPlan {
17243                row_in_chunk,
17244                seq,
17245                value,
17246                format_id,
17247            });
17248    }
17249
17250    fn computed_overlay_chunk_has_formats(
17251        &self,
17252        sheet_id: SheetId,
17253        col0: u32,
17254        chunk_idx: usize,
17255    ) -> bool {
17256        let sheet_name = self.graph.sheet_name(sheet_id);
17257        self.arrow_sheets
17258            .sheet(sheet_name)
17259            .and_then(|sheet| sheet.columns.get(col0 as usize))
17260            .and_then(|column| column.chunk(chunk_idx))
17261            .is_some_and(|chunk| chunk.computed_overlay.has_formats())
17262    }
17263
17264    fn locate_computed_write_chunk(&self, sheet_id: SheetId, row0: u32) -> (usize, u32, usize) {
17265        let sheet_name = self.graph.sheet_name(sheet_id);
17266        if let Some(sheet) = self.arrow_sheets.sheet(sheet_name) {
17267            return Self::locate_row_in_sheet_for_computed_write_plan(sheet, row0 as usize);
17268        }
17269        Self::locate_row_in_empty_sheet_for_computed_write_plan(row0 as usize, 32 * 1024)
17270    }
17271
17272    fn locate_row_in_sheet_for_computed_write_plan(
17273        sheet: &crate::arrow_store::ArrowSheet,
17274        row0: usize,
17275    ) -> (usize, u32, usize) {
17276        if row0 < sheet.nrows as usize
17277            && let Some((chunk_idx, row_in_chunk)) = sheet.chunk_of_row(row0)
17278        {
17279            let chunk_start = sheet.chunk_starts.get(chunk_idx).copied().unwrap_or(0);
17280            return (chunk_idx, chunk_start as u32, row_in_chunk);
17281        }
17282
17283        let chunk_rows = sheet.chunk_rows.max(1);
17284        if sheet.chunk_starts.is_empty() {
17285            return Self::locate_row_in_empty_sheet_for_computed_write_plan(row0, chunk_rows);
17286        }
17287
17288        let mut chunk_idx = sheet.chunk_starts.len().saturating_sub(1);
17289        let mut chunk_start = sheet.chunk_starts[chunk_idx];
17290        while chunk_start.saturating_add(chunk_rows) <= row0 {
17291            chunk_idx = chunk_idx.saturating_add(1);
17292            chunk_start = chunk_start.saturating_add(chunk_rows);
17293        }
17294        (
17295            chunk_idx,
17296            chunk_start as u32,
17297            row0.saturating_sub(chunk_start),
17298        )
17299    }
17300
17301    fn locate_row_in_empty_sheet_for_computed_write_plan(
17302        row0: usize,
17303        chunk_rows: usize,
17304    ) -> (usize, u32, usize) {
17305        let chunk_rows = chunk_rows.max(1);
17306        let chunk_idx = row0 / chunk_rows;
17307        let chunk_start = chunk_idx.saturating_mul(chunk_rows);
17308        (
17309            chunk_idx,
17310            chunk_start as u32,
17311            row0.saturating_sub(chunk_start),
17312        )
17313    }
17314
17315    #[cfg(test)]
17316    pub(crate) fn debug_plan_computed_write_coalescing(
17317        &self,
17318        buffer: &ComputedWriteBuffer,
17319    ) -> ComputedWriteCoalescingPlan {
17320        self.plan_computed_write_coalescing(buffer)
17321    }
17322
17323    fn record_computed_write_buffer_delta(
17324        &self,
17325        buffer: &ComputedWriteBuffer,
17326        delta: &mut DeltaCollector,
17327    ) {
17328        if delta.mode == DeltaMode::Off {
17329            return;
17330        }
17331        let mut final_values = BTreeMap::new();
17332        for write in buffer.writes() {
17333            match write {
17334                ComputedWrite::Cell {
17335                    sheet_id,
17336                    row0,
17337                    col0,
17338                    value,
17339                    ..
17340                } => {
17341                    final_values.insert((*sheet_id, *row0, *col0), value.clone());
17342                }
17343                ComputedWrite::Rect {
17344                    sheet_id,
17345                    sr0,
17346                    sc0,
17347                    values,
17348                    ..
17349                } => {
17350                    for (row_offset, row) in values.iter().enumerate() {
17351                        for (col_offset, value) in row.iter().enumerate() {
17352                            final_values.insert(
17353                                (
17354                                    *sheet_id,
17355                                    sr0.saturating_add(row_offset as u32),
17356                                    sc0.saturating_add(col_offset as u32),
17357                                ),
17358                                value.clone(),
17359                            );
17360                        }
17361                    }
17362                }
17363            }
17364        }
17365        for ((sheet_id, row0, col0), value) in final_values {
17366            let old = self
17367                .read_cell_value(self.graph.sheet_name(sheet_id), row0 + 1, col0 + 1)
17368                .unwrap_or(LiteralValue::Empty);
17369            if old != value.to_literal_for(self.config.date_system) {
17370                delta.record_cell(sheet_id, row0, col0);
17371            }
17372        }
17373    }
17374
17375    pub(crate) fn flush_computed_write_buffer(
17376        &mut self,
17377        buffer: &mut ComputedWriteBuffer,
17378    ) -> Result<(), ExcelError> {
17379        self.flush_computed_write_buffer_inner(buffer, false)
17380    }
17381
17382    fn flush_authoritative_computed_write_buffer(
17383        &mut self,
17384        buffer: &mut ComputedWriteBuffer,
17385    ) -> Result<(), ExcelError> {
17386        self.flush_computed_write_buffer_inner(buffer, true)
17387    }
17388
17389    fn flush_computed_write_buffer_inner(
17390        &mut self,
17391        buffer: &mut ComputedWriteBuffer,
17392        synchronize_formula_plane_formats: bool,
17393    ) -> Result<(), ExcelError> {
17394        if buffer.is_empty() {
17395            // No plane-owned placement is published by this layer. It is
17396            // either legacy-only or its selected span placements are all
17397            // overlay punchouts. A newly admitted dirty span with any
17398            // plane-owned placement necessarily contributes a buffered write.
17399            return Ok(());
17400        }
17401
17402        #[cfg(test)]
17403        if synchronize_formula_plane_formats
17404            && std::mem::take(&mut self.cancel_before_formula_plane_layer_commit_once_for_test)
17405        {
17406            self.active_cancel_flag
17407                .as_ref()
17408                .expect("test cancellation hook requires a cancellable evaluation")
17409                .cancel();
17410            self.cancellation_checkpoint("Evaluation cancelled before FormulaPlane layer commit")?;
17411        }
17412
17413        // Keep ownership of all pending writes until the final request
17414        // checkpoint and bounded commit-window preflight succeed so failures
17415        // remain retry safe. The synchronization and flush that follow are
17416        // infallible mutations with no cancellation point.
17417        self.resource_checkpoint(0)?;
17418        let format_sync = synchronize_formula_plane_formats
17419            .then(|| self.prepare_formula_plane_derived_format_sync())
17420            .flatten();
17421        let commit_started = self.preflight_evaluation_commit_window(buffer.len())?;
17422        if let Some(format_sync) = format_sync {
17423            self.commit_formula_plane_derived_format_sync(format_sync);
17424        }
17425        let (writes, formats_present) = buffer.take_writes();
17426        let plan = self.plan_owned_computed_write_coalescing(writes, formats_present);
17427        self.flush_computed_write_plan(plan);
17428        self.observe_evaluation_commit_window(commit_started);
17429
17430        Ok(())
17431    }
17432
17433    fn flush_computed_write_plan(&mut self, plan: ComputedWriteCoalescingPlan) {
17434        for chunk in plan.chunks {
17435            self.flush_computed_write_chunk_plan(chunk);
17436        }
17437    }
17438
17439    fn flush_computed_write_chunk_plan(&mut self, chunk: ComputedWriteChunkPlan) {
17440        match &chunk.shape {
17441            ComputedWriteChunkPlanShape::Point => {
17442                self.flush_computed_write_chunk_plan_as_points(chunk);
17443            }
17444            ComputedWriteChunkPlanShape::SparseOffsets { .. } => {
17445                self.flush_computed_write_chunk_plan_as_sparse_fragment_or_points(chunk);
17446            }
17447            ComputedWriteChunkPlanShape::DenseRange { .. } => {
17448                self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
17449            }
17450            ComputedWriteChunkPlanShape::RunRange { len, runs, .. } => {
17451                if Self::should_emit_computed_run_fragment(*len, *runs) {
17452                    self.flush_computed_write_chunk_plan_as_run_fragment(chunk);
17453                } else {
17454                    self.flush_computed_write_chunk_plan_as_dense_fragment(chunk);
17455                }
17456            }
17457        }
17458    }
17459
17460    #[inline]
17461    fn should_emit_computed_run_fragment(len: usize, runs: usize) -> bool {
17462        runs <= len / 2
17463    }
17464
17465    fn flush_computed_write_chunk_plan_as_points(&mut self, chunk: ComputedWriteChunkPlan) {
17466        let sheet_name = self.graph.sheet_name(chunk.sheet_id).to_string();
17467        for entry in chunk.entries {
17468            let row0 = chunk
17469                .chunk_start_row0
17470                .saturating_add(entry.row_in_chunk as u32);
17471            self.write_computed_overlay_value_0based(&sheet_name, row0, chunk.col0, entry.value);
17472        }
17473        self.apply_computed_overlay_format_effect(
17474            chunk.sheet_id,
17475            chunk.col0,
17476            chunk.chunk_idx,
17477            chunk.format_effect,
17478        );
17479    }
17480
17481    fn flush_computed_write_chunk_plan_as_sparse_fragment_or_points(
17482        &mut self,
17483        chunk: ComputedWriteChunkPlan,
17484    ) {
17485        let point_estimate = Self::computed_write_chunk_plan_point_estimate(&chunk);
17486        let sheet_id = chunk.sheet_id;
17487        let col0 = chunk.col0;
17488        let chunk_idx = chunk.chunk_idx;
17489        let chunk_start_row0 = chunk.chunk_start_row0;
17490        let format_effect = chunk.format_effect;
17491        let items: Vec<(usize, OverlayValue)> = chunk
17492            .entries
17493            .into_iter()
17494            .map(|entry| (entry.row_in_chunk, entry.value))
17495            .collect();
17496        match OverlayFragment::sparse_offsets_if_estimated_smaller_than_points(
17497            items,
17498            point_estimate,
17499        ) {
17500            Some(Ok(fragment)) => {
17501                self.apply_computed_overlay_fragment(sheet_id, col0, chunk_idx, fragment);
17502            }
17503            Some(Err(cells)) => {
17504                self.flush_computed_overlay_cells_as_points(
17505                    sheet_id,
17506                    col0,
17507                    chunk_start_row0,
17508                    cells,
17509                );
17510            }
17511            None => {}
17512        }
17513        self.apply_computed_overlay_format_effect(sheet_id, col0, chunk_idx, format_effect);
17514    }
17515
17516    #[inline]
17517    fn computed_write_chunk_plan_point_estimate(chunk: &ComputedWriteChunkPlan) -> usize {
17518        chunk
17519            .entries
17520            .iter()
17521            .map(|entry| ComputedWriteBuffer::estimate_value_bytes(&entry.value))
17522            .fold(0usize, usize::saturating_add)
17523    }
17524
17525    fn flush_computed_overlay_cells_as_points(
17526        &mut self,
17527        sheet_id: SheetId,
17528        col0: u32,
17529        chunk_start_row0: u32,
17530        cells: Vec<(usize, OverlayValue)>,
17531    ) {
17532        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
17533        for (row_in_chunk, value) in cells {
17534            let row0 = chunk_start_row0.saturating_add(row_in_chunk as u32);
17535            self.write_computed_overlay_value_0based(&sheet_name, row0, col0, value);
17536        }
17537    }
17538
17539    fn flush_computed_write_chunk_plan_as_dense_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
17540        if chunk.entries.is_empty() {
17541            return;
17542        }
17543        let start = chunk.entries[0].row_in_chunk;
17544        let values: Vec<OverlayValue> =
17545            chunk.entries.into_iter().map(|entry| entry.value).collect();
17546        if let Some(fragment) = OverlayFragment::dense_range(start, values) {
17547            self.apply_computed_overlay_fragment(
17548                chunk.sheet_id,
17549                chunk.col0,
17550                chunk.chunk_idx,
17551                fragment,
17552            );
17553        }
17554        self.apply_computed_overlay_format_effect(
17555            chunk.sheet_id,
17556            chunk.col0,
17557            chunk.chunk_idx,
17558            chunk.format_effect,
17559        );
17560    }
17561
17562    fn flush_computed_write_chunk_plan_as_run_fragment(&mut self, chunk: ComputedWriteChunkPlan) {
17563        if chunk.entries.is_empty() {
17564            return;
17565        }
17566        let start = chunk.entries[0].row_in_chunk;
17567        let values: Vec<OverlayValue> =
17568            chunk.entries.into_iter().map(|entry| entry.value).collect();
17569        if let Some(fragment) = OverlayFragment::run_range(start, values) {
17570            self.apply_computed_overlay_fragment(
17571                chunk.sheet_id,
17572                chunk.col0,
17573                chunk.chunk_idx,
17574                fragment,
17575            );
17576        }
17577        self.apply_computed_overlay_format_effect(
17578            chunk.sheet_id,
17579            chunk.col0,
17580            chunk.chunk_idx,
17581            chunk.format_effect,
17582        );
17583    }
17584
17585    fn apply_computed_overlay_format_effect(
17586        &mut self,
17587        sheet_id: SheetId,
17588        col0: u32,
17589        chunk_idx: usize,
17590        effect: ComputedWriteChunkFormatEffect,
17591    ) {
17592        if !(self.config.arrow_storage_enabled
17593            && self.config.delta_overlay_enabled
17594            && self.config.write_formula_overlay_enabled)
17595            || self.computed_overlay_mirroring_disabled
17596        {
17597            return;
17598        }
17599
17600        let sheet_name = self.graph.sheet_name(sheet_id);
17601        let Some(sheet) = self.arrow_sheets.sheet_mut(sheet_name) else {
17602            return;
17603        };
17604        let Some(chunk) = sheet
17605            .columns
17606            .get_mut(col0 as usize)
17607            .and_then(|column| column.chunk_mut(chunk_idx))
17608        else {
17609            return;
17610        };
17611        match effect {
17612            ComputedWriteChunkFormatEffect::NoFormatWork => {}
17613            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Range {
17614                start,
17615                end,
17616            }) => {
17617                #[cfg(test)]
17618                {
17619                    self.computed_overlay_stale_clear_range_effects_for_test = self
17620                        .computed_overlay_stale_clear_range_effects_for_test
17621                        .saturating_add(1);
17622                }
17623                chunk.computed_overlay.clear_format_range(start, end);
17624            }
17625            ComputedWriteChunkFormatEffect::ClearStale(ComputedWriteFormatClear::Offsets(
17626                offsets,
17627            )) => {
17628                #[cfg(test)]
17629                {
17630                    self.computed_overlay_stale_clear_offset_attempts_for_test = self
17631                        .computed_overlay_stale_clear_offset_attempts_for_test
17632                        .saturating_add(offsets.len() as u64);
17633                }
17634                chunk.computed_overlay.clear_format_offsets(&offsets);
17635            }
17636            ComputedWriteChunkFormatEffect::SetExplicit(formats) => {
17637                #[cfg(test)]
17638                {
17639                    self.computed_overlay_set_explicit_entry_operations_for_test = self
17640                        .computed_overlay_set_explicit_entry_operations_for_test
17641                        .saturating_add(formats.len() as u64);
17642                }
17643                for (row_in_chunk, format_id) in formats {
17644                    chunk.computed_overlay.set_format(row_in_chunk, format_id);
17645                }
17646            }
17647        }
17648    }
17649
17650    fn apply_computed_overlay_fragment(
17651        &mut self,
17652        sheet_id: SheetId,
17653        col0: u32,
17654        chunk_idx: usize,
17655        fragment: OverlayFragment,
17656    ) {
17657        if !(self.config.arrow_storage_enabled
17658            && self.config.delta_overlay_enabled
17659            && self.config.write_formula_overlay_enabled)
17660        {
17661            return;
17662        }
17663        if self.computed_overlay_mirroring_disabled {
17664            return;
17665        }
17666
17667        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
17668        self.ensure_arrow_sheet(&sheet_name);
17669
17670        let col0 = col0 as usize;
17671        let asheet = self
17672            .arrow_sheets
17673            .sheet_mut(&sheet_name)
17674            .expect("ArrowSheet must exist");
17675
17676        let cur_cols = asheet.columns.len();
17677        if col0 >= cur_cols {
17678            asheet.insert_columns(cur_cols, (col0 + 1) - cur_cols);
17679        }
17680
17681        let start_row0 = asheet
17682            .chunk_starts
17683            .get(chunk_idx)
17684            .copied()
17685            .unwrap_or_else(|| chunk_idx.saturating_mul(asheet.chunk_rows.max(1)));
17686        let required_rows =
17687            start_row0.saturating_add(fragment.max_covered_offset().saturating_add(1));
17688        if required_rows > asheet.nrows as usize {
17689            if asheet.columns.is_empty() {
17690                asheet.insert_columns(0, 1);
17691            }
17692            asheet.ensure_row_capacity(required_rows);
17693        }
17694
17695        let Some(ch) = asheet.ensure_column_chunk_mut(col0, chunk_idx) else {
17696            return;
17697        };
17698        let delta = ch.computed_overlay.apply_fragment(fragment);
17699        self.adjust_computed_overlay_bytes(delta);
17700
17701        if let Some(cap) = self.config.max_overlay_memory_bytes
17702            && self.computed_overlay_bytes_estimate > cap
17703        {
17704            self.disable_computed_overlay_mirroring_due_to_budget(cap);
17705        }
17706    }
17707
17708    #[inline]
17709    fn adjust_computed_overlay_bytes(&mut self, delta: isize) {
17710        if delta >= 0 {
17711            self.computed_overlay_bytes_estimate = self
17712                .computed_overlay_bytes_estimate
17713                .saturating_add(delta as usize);
17714        } else {
17715            self.computed_overlay_bytes_estimate = self
17716                .computed_overlay_bytes_estimate
17717                .saturating_sub((-delta) as usize);
17718        }
17719    }
17720
17721    fn clear_all_computed_overlays(&mut self) {
17722        let mut freed_total = 0usize;
17723        for sh in self.arrow_sheets.sheets.iter_mut() {
17724            for col in sh.columns.iter_mut() {
17725                for ch in col.chunks.iter_mut() {
17726                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
17727                }
17728                for ch in col.sparse_chunks.values_mut() {
17729                    freed_total = freed_total.saturating_add(ch.computed_overlay.clear());
17730                }
17731            }
17732        }
17733        self.computed_overlay_bytes_estimate = self
17734            .computed_overlay_bytes_estimate
17735            .saturating_sub(freed_total);
17736    }
17737
17738    fn disable_computed_overlay_mirroring_due_to_budget(&mut self, _cap: usize) {
17739        // Phase 1 (ticket 610): Arrow-truth is the only supported mode.
17740        // Handle budget pressure by compacting computed overlays into base lanes.
17741        self.compact_all_computed_overlays();
17742    }
17743
17744    /// Fold all computed overlay entries across all sheets into their base arrays.
17745    /// This preserves data while freeing overlay memory, allowing mirroring to continue.
17746    fn compact_all_computed_overlays(&mut self) {
17747        let mut freed_total = 0usize;
17748        for sheet in self.arrow_sheets.sheets.iter_mut() {
17749            for col_idx in 0..sheet.columns.len() {
17750                // Dense chunks
17751                let num_dense = sheet.columns[col_idx].chunks.len();
17752                for ch_idx in 0..num_dense {
17753                    freed_total += sheet.compact_computed_overlay_chunk(col_idx, ch_idx);
17754                }
17755                // Sparse chunks
17756                let sparse_keys: Vec<usize> = sheet.columns[col_idx]
17757                    .sparse_chunks
17758                    .keys()
17759                    .copied()
17760                    .collect();
17761                for ch_idx in sparse_keys {
17762                    freed_total += sheet.compact_computed_overlay_sparse_chunk(col_idx, ch_idx);
17763                }
17764            }
17765        }
17766        self.computed_overlay_bytes_estimate = self
17767            .computed_overlay_bytes_estimate
17768            .saturating_sub(freed_total);
17769        self.overlay_compactions = self.overlay_compactions.saturating_add(1);
17770    }
17771
17772    fn mirror_vertex_value_to_overlay(&mut self, vertex_id: VertexId, value: &LiteralValue) {
17773        let _ = self.record_vertex_value_to_overlay(vertex_id, value, None);
17774    }
17775
17776    fn record_vertex_value_to_overlay(
17777        &mut self,
17778        vertex_id: VertexId,
17779        value: &LiteralValue,
17780        computed_writes: Option<&mut ComputedWriteBuffer>,
17781    ) -> Result<(), ExcelError> {
17782        if !(self.config.arrow_storage_enabled
17783            && self.config.delta_overlay_enabled
17784            && self.config.write_formula_overlay_enabled)
17785        {
17786            return Ok(());
17787        }
17788        if self.computed_overlay_mirroring_disabled {
17789            return Ok(());
17790        }
17791        if !matches!(
17792            self.graph.get_vertex_kind(vertex_id),
17793            VertexKind::FormulaScalar | VertexKind::FormulaArray
17794        ) {
17795            return Ok(());
17796        }
17797        let Some(cell) = self.graph.get_cell_ref(vertex_id) else {
17798            return Ok(());
17799        };
17800        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
17801        let date_system = self.arrow_sheet_date_system(&sheet_name);
17802        let ov = Self::literal_to_overlay_value(value, date_system);
17803        if let Some(buffer) = computed_writes {
17804            let format_id = self.derived_formats.read().unwrap().get(&cell).copied();
17805            buffer.push_cell_with_format(
17806                cell.sheet_id,
17807                cell.coord.row(),
17808                cell.coord.col(),
17809                ov,
17810                format_id,
17811            );
17812            if self.should_flush_computed_write_buffer(buffer) {
17813                self.flush_computed_write_buffer(buffer)?;
17814            }
17815        } else {
17816            self.write_computed_overlay_value_0based(
17817                &sheet_name,
17818                cell.coord.row(),
17819                cell.coord.col(),
17820                ov,
17821            );
17822        }
17823        Ok(())
17824    }
17825
17826    #[inline]
17827    fn should_flush_computed_write_buffer(&self, buffer: &ComputedWriteBuffer) -> bool {
17828        self.config.max_overlay_memory_bytes.is_some_and(|cap| {
17829            if cap == 0 {
17830                return false;
17831            }
17832            self.computed_overlay_bytes_estimate
17833                .saturating_add(buffer.estimated_bytes())
17834                > cap
17835        })
17836    }
17837
17838    /// Estimated memory usage for computed overlays (formula/spill mirroring).
17839    pub fn overlay_memory_usage(&self) -> usize {
17840        self.computed_overlay_bytes_estimate
17841    }
17842
17843    #[cfg(test)]
17844    pub(crate) fn debug_overlay_compactions(&self) -> u64 {
17845        self.overlay_compactions
17846    }
17847
17848    #[cfg(test)]
17849    pub(crate) fn debug_recompute_computed_overlay_bytes(&mut self) -> usize {
17850        let mut total = 0usize;
17851        for sheet in &self.arrow_sheets.sheets {
17852            for column in &sheet.columns {
17853                for chunk in &column.chunks {
17854                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
17855                }
17856                for chunk in column.sparse_chunks.values() {
17857                    total = total.saturating_add(chunk.computed_overlay.estimated_bytes());
17858                }
17859            }
17860        }
17861        self.computed_overlay_bytes_estimate = total;
17862        total
17863    }
17864
17865    fn resolve_sheet_locator_for_write(
17866        &mut self,
17867        loc: formualizer_common::SheetLocator<'_>,
17868        current_sheet: &str,
17869    ) -> Result<SheetId, ExcelError> {
17870        Ok(match loc {
17871            formualizer_common::SheetLocator::Id(id) => id,
17872            formualizer_common::SheetLocator::Name(name) => self.graph.sheet_id_mut(name.as_ref()),
17873            formualizer_common::SheetLocator::Current => self.graph.sheet_id_mut(current_sheet),
17874        })
17875    }
17876
17877    fn resolve_sheet_locator_for_read(
17878        &self,
17879        loc: formualizer_common::SheetLocator<'_>,
17880        current_sheet: &str,
17881    ) -> Result<SheetId, ExcelError> {
17882        match loc {
17883            formualizer_common::SheetLocator::Id(id) => Ok(id),
17884            formualizer_common::SheetLocator::Name(name) => self
17885                .graph
17886                .sheet_id(name.as_ref())
17887                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
17888            formualizer_common::SheetLocator::Current => self
17889                .graph
17890                .sheet_id(current_sheet)
17891                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref)),
17892        }
17893    }
17894
17895    /// Set a cell value
17896    pub fn set_cell_value(
17897        &mut self,
17898        sheet: &str,
17899        row: u32,
17900        col: u32,
17901        value: LiteralValue,
17902    ) -> Result<(), ExcelError> {
17903        self.observe_function_semantic_epoch()?;
17904        let sheet_existed = self.graph.sheet_id(sheet).is_some();
17905        let sheet_id = self.graph.sheet_id_mut(sheet);
17906        let cell_ref = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
17907        let replaced_formula =
17908            self.graph
17909                .get_vertex_id_for_address(&cell_ref)
17910                .is_some_and(|vertex| {
17911                    matches!(
17912                        self.graph.get_vertex_kind(*vertex),
17913                        VertexKind::FormulaScalar | VertexKind::FormulaArray
17914                    )
17915                });
17916        self.demote_span_containing_cell_for_write(
17917            sheet_id,
17918            row.saturating_sub(1),
17919            col.saturating_sub(1),
17920        )
17921        .map_err(Self::editor_error_to_excel)?;
17922        self.graph.set_cell_value(sheet, row, col, value.clone())?;
17923        self.clear_cell_format_state(sheet, cell_ref);
17924        self.record_formula_plane_changed_cell(sheet, row, col);
17925        if !sheet_existed || replaced_formula {
17926            self.mark_topology_edited();
17927        }
17928        // Mirror into Arrow overlay when enabled
17929        self.mirror_value_to_overlay(sheet, row, col, &value);
17930        // Advance snapshot to reflect external mutation.
17931        self.mark_data_edited();
17932        Ok(())
17933    }
17934
17935    /// Record a single-cell change in FormulaPlane authority so the next
17936    /// `evaluate_all` under `AuthoritativeExperimental` can derive bounded
17937    /// span work from `FormulaConsumerReadIndex` instead of recomputing every
17938    /// active span.
17939    fn record_formula_plane_changed_cell(&mut self, sheet: &str, row: u32, col: u32) {
17940        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
17941            return;
17942        }
17943        let sheet_id = self.graph.sheet_id_mut(sheet);
17944        self.record_formula_plane_structural_change(StructuralScope::Cell {
17945            sheet: sheet_id,
17946            row: row.saturating_sub(1),
17947            col: col.saturating_sub(1),
17948        });
17949    }
17950
17951    fn record_formula_plane_change_for_event(&mut self, event: &ChangeEvent) {
17952        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
17953            return;
17954        }
17955
17956        match event {
17957            ChangeEvent::SetValue { addr, .. } | ChangeEvent::SetFormula { addr, .. } => {
17958                self.record_formula_plane_structural_change(StructuralScope::Cell {
17959                    sheet: addr.sheet_id,
17960                    row: addr.coord.row(),
17961                    col: addr.coord.col(),
17962                });
17963            }
17964            ChangeEvent::SpillCommitted { new, .. } => {
17965                if let Some(scope) = Self::formula_plane_region_from_cells(&new.target_cells) {
17966                    self.record_formula_plane_structural_change(scope);
17967                }
17968            }
17969            ChangeEvent::SpillCleared { old, .. } => {
17970                if let Some(scope) = Self::formula_plane_region_from_cells(&old.target_cells) {
17971                    self.record_formula_plane_structural_change(scope);
17972                }
17973            }
17974            ChangeEvent::DefineName { .. }
17975            | ChangeEvent::UpdateName { .. }
17976            | ChangeEvent::DeleteName { .. }
17977            | ChangeEvent::NamedRangeAdjusted { .. } => {
17978                // Direct name events are preflighted by the logged-name APIs.
17979                // Structural entry points preflight spans before emitting a
17980                // NamedRangeAdjusted event. Epoch changes only rebuild caches.
17981                self.record_formula_plane_structural_change(StructuralScope::AllSheets);
17982            }
17983            ChangeEvent::VertexMoved { .. } | ChangeEvent::FormulaAdjusted { .. } => {
17984                // Structural entry points publish their axis delta once after
17985                // graph, Arrow, and span geometry commits.
17986            }
17987            ChangeEvent::SetRowVisibility { sheet_id, row0, .. } => {
17988                self.record_formula_plane_structural_change(StructuralScope::Region(
17989                    Region::whole_row(*sheet_id, *row0),
17990                ));
17991            }
17992            ChangeEvent::AddVertex { .. }
17993            | ChangeEvent::RemoveVertex { .. }
17994            | ChangeEvent::EdgeAdded { .. }
17995            | ChangeEvent::EdgeRemoved { .. }
17996            | ChangeEvent::CompoundStart { .. }
17997            | ChangeEvent::CompoundEnd { .. }
17998            | ChangeEvent::StagedFormulaCellChanged { .. } => {}
17999        }
18000    }
18001
18002    fn record_formula_plane_structural_change(&mut self, scope: StructuralScope) {
18003        if self.config.formula_plane_mode == FormulaPlaneMode::Off {
18004            return;
18005        }
18006
18007        match scope {
18008            StructuralScope::Cell { sheet, row, col } => {
18009                self.graph
18010                    .mark_formula_region_dirty(Region::point(sheet, row, col));
18011            }
18012            StructuralScope::Region(region) => {
18013                self.graph.mark_formula_region_dirty(region);
18014            }
18015            StructuralScope::Sheet(sheet_id) => {
18016                self.graph
18017                    .mark_formula_region_dirty(Region::whole_sheet(sheet_id));
18018            }
18019            StructuralScope::RemovedSheet(sheet_id) => {
18020                let removed_refs = {
18021                    let authority = self.graph.formula_authority();
18022                    authority
18023                        .active_span_refs()
18024                        .into_iter()
18025                        .filter(|span_ref| {
18026                            authority
18027                                .plane
18028                                .spans
18029                                .get(*span_ref)
18030                                .map(|span| span.sheet_id == sheet_id)
18031                                .unwrap_or(false)
18032                        })
18033                        .collect::<Vec<_>>()
18034                };
18035
18036                {
18037                    let authority = self.graph.formula_authority_mut();
18038                    for span_ref in removed_refs {
18039                        authority.plane.remove_span(span_ref);
18040                    }
18041                    let _ = authority.rebuild_indexes();
18042                }
18043                self.graph
18044                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
18045            }
18046            StructuralScope::OpaqueGlobal => {
18047                let _ = self.graph.formula_authority_mut().rebuild_indexes();
18048                self.graph
18049                    .mark_all_formula_spans_dirty(WholeSpanDirtyReason::GlobalInvalidation);
18050            }
18051            StructuralScope::AllSheets => {
18052                // Name/table mutations dirty their exact graph dependents,
18053                // and name-dependent spans are demoted by the name invalidation
18054                // registry. A topology rebuild is cache invalidation only.
18055                let _ = self.graph.formula_authority_mut().rebuild_indexes();
18056            }
18057        }
18058    }
18059
18060    fn formula_plane_region_from_cells(cells: &[CellRef]) -> Option<StructuralScope> {
18061        let first = cells.first()?;
18062        let sheet_id = first.sheet_id;
18063        if cells.iter().any(|cell| cell.sheet_id != sheet_id) {
18064            return Some(StructuralScope::OpaqueGlobal);
18065        }
18066        let mut row_start = first.coord.row();
18067        let mut row_end = row_start;
18068        let mut col_start = first.coord.col();
18069        let mut col_end = col_start;
18070        for cell in cells.iter().skip(1) {
18071            row_start = row_start.min(cell.coord.row());
18072            row_end = row_end.max(cell.coord.row());
18073            col_start = col_start.min(cell.coord.col());
18074            col_end = col_end.max(cell.coord.col());
18075        }
18076        Some(StructuralScope::Region(Region::rect(
18077            sheet_id, row_start, row_end, col_start, col_end,
18078        )))
18079    }
18080
18081    pub fn set_cell_value_ref(
18082        &mut self,
18083        cell: formualizer_common::SheetCellRef<'_>,
18084        current_sheet: &str,
18085        value: LiteralValue,
18086    ) -> Result<(), ExcelError> {
18087        let owned = cell.into_owned();
18088        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
18089        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
18090        self.set_cell_value(
18091            &sheet_name,
18092            owned.coord.row() + 1,
18093            owned.coord.col() + 1,
18094            value,
18095        )
18096    }
18097
18098    pub fn set_cell_formula_ref(
18099        &mut self,
18100        cell: formualizer_common::SheetCellRef<'_>,
18101        current_sheet: &str,
18102        ast: ASTNode,
18103    ) -> Result<(), ExcelError> {
18104        let owned = cell.into_owned();
18105        let sheet_id = self.resolve_sheet_locator_for_write(owned.sheet, current_sheet)?;
18106        let sheet_name = self.graph.sheet_name(sheet_id).to_string();
18107        self.set_cell_formula(
18108            &sheet_name,
18109            owned.coord.row() + 1,
18110            owned.coord.col() + 1,
18111            ast,
18112        )
18113    }
18114
18115    pub fn get_cell_value_ref(
18116        &self,
18117        cell: formualizer_common::SheetCellRef<'_>,
18118        current_sheet: &str,
18119    ) -> Result<Option<LiteralValue>, ExcelError> {
18120        let owned = cell.into_owned();
18121        let sheet_id = self.resolve_sheet_locator_for_read(owned.sheet, current_sheet)?;
18122        let sheet_name = self.graph.sheet_name(sheet_id);
18123        Ok(self.get_cell_value(sheet_name, owned.coord.row() + 1, owned.coord.col() + 1))
18124    }
18125
18126    pub fn resolve_range_view_sheet_ref<'c>(
18127        &'c self,
18128        r: &formualizer_common::SheetRef<'_>,
18129        current_sheet: &str,
18130    ) -> Result<RangeView<'c>, ExcelError> {
18131        use formualizer_common::SheetLocator;
18132
18133        let sheet_to_opt_name = |loc: SheetLocator<'_>| -> Result<Option<String>, ExcelError> {
18134            match loc {
18135                SheetLocator::Current => Ok(None),
18136                SheetLocator::Name(name) => Ok(Some(name.as_ref().to_string())),
18137                SheetLocator::Id(id) => Ok(Some(self.graph.sheet_name(id).to_string())),
18138            }
18139        };
18140
18141        let rt = match r {
18142            formualizer_common::SheetRef::Cell(cell) => ReferenceType::Cell {
18143                sheet: sheet_to_opt_name(cell.sheet.clone())?,
18144                row: cell.coord.row() + 1,
18145                col: cell.coord.col() + 1,
18146                row_abs: cell.coord.row_abs(),
18147                col_abs: cell.coord.col_abs(),
18148            },
18149            formualizer_common::SheetRef::Range(range) => ReferenceType::Range {
18150                sheet: sheet_to_opt_name(range.sheet.clone())?,
18151                start_row: range.start_row.map(|b| b.index + 1),
18152                start_col: range.start_col.map(|b| b.index + 1),
18153                end_row: range.end_row.map(|b| b.index + 1),
18154                end_col: range.end_col.map(|b| b.index + 1),
18155                start_row_abs: range.start_row.map(|b| b.abs).unwrap_or(false),
18156                start_col_abs: range.start_col.map(|b| b.abs).unwrap_or(false),
18157                end_row_abs: range.end_row.map(|b| b.abs).unwrap_or(false),
18158                end_col_abs: range.end_col.map(|b| b.abs).unwrap_or(false),
18159            },
18160        };
18161
18162        crate::traits::EvaluationContext::resolve_range_view(self, &rt, current_sheet)
18163    }
18164
18165    /// Set a cell formula
18166    pub fn set_cell_formula(
18167        &mut self,
18168        sheet: &str,
18169        row: u32,
18170        col: u32,
18171        ast: ASTNode,
18172    ) -> Result<(), ExcelError> {
18173        self.observe_function_semantic_epoch()?;
18174        let sheet_id = self.graph.sheet_id_mut(sheet);
18175        self.demote_span_containing_cell_for_write(
18176            sheet_id,
18177            row.saturating_sub(1),
18178            col.saturating_sub(1),
18179        )
18180        .map_err(Self::editor_error_to_excel)?;
18181        let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18182        let ingested = {
18183            let mut pipeline = self.ingest_pipeline();
18184            pipeline.ingest_formula(FormulaAstInput::Tree(ast), placement, None)?
18185        };
18186        self.graph.set_cell_formula_with_plan(
18187            sheet,
18188            row,
18189            col,
18190            ingested.ast_id,
18191            &ingested.dep_plan,
18192            ingested.dep_plan.volatile,
18193            ingested.dep_plan.dynamic,
18194        )?;
18195        self.clear_cell_format_state(sheet, placement);
18196        self.record_formula_plane_changed_cell(sheet, row, col);
18197
18198        // If the cell previously held a user value in the delta overlay, it must not continue
18199        // to mask the formula result under Arrow-canonical reads (overlay precedence is
18200        // delta -> computed -> base). Remove the overlay entry instead of writing `Empty`,
18201        // because an explicit `Empty` overlay would still take precedence over computed values.
18202        self.clear_delta_overlay_cell(sheet, row, col);
18203
18204        // Advance snapshot to reflect external mutation
18205        self.mark_topology_edited();
18206        Ok(())
18207    }
18208
18209    /// Bulk set many formulas on a sheet. Skips per-cell snapshot bumping and minimizes edge rebuilds.
18210    pub fn bulk_set_formulas<I>(&mut self, sheet: &str, items: I) -> Result<usize, ExcelError>
18211    where
18212        I: IntoIterator<Item = (u32, u32, ASTNode)>,
18213    {
18214        let collected: Vec<(u32, u32, ASTNode)> = items.into_iter().collect();
18215        let edited_cells: Vec<(u32, u32)> = collected.iter().map(|(r, c, _)| (*r, *c)).collect();
18216        let sheet_id = self.graph.sheet_id_mut(sheet);
18217        let writes_inside_active_span = edited_cells.iter().any(|(row, col)| {
18218            let placement =
18219                PlacementCoord::new(sheet_id, row.saturating_sub(1), col.saturating_sub(1));
18220            self.graph
18221                .formula_authority()
18222                .plane
18223                .spans
18224                .find_at(placement)
18225                .is_some()
18226        });
18227        if writes_inside_active_span {
18228            self.demote_spans_preserving_computed_overlays(sheet_id, Region::whole_sheet(sheet_id))
18229                .map_err(Self::editor_error_to_excel)?;
18230        }
18231        let ingested = {
18232            let mut pipeline = self.ingest_pipeline();
18233            let inputs = collected.into_iter().map(|(row, col, ast)| {
18234                let placement = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18235                (FormulaAstInput::Tree(ast), placement, None)
18236            });
18237            pipeline.ingest_batch(inputs)?
18238        };
18239        let planned: Vec<(u32, u32, AstNodeId, DependencyPlanRow)> = ingested
18240            .into_iter()
18241            .map(|formula| {
18242                (
18243                    formula.placement.coord.row() + 1,
18244                    formula.placement.coord.col() + 1,
18245                    formula.ast_id,
18246                    formula.dep_plan,
18247                )
18248            })
18249            .collect();
18250        let n = self.graph.bulk_set_formulas_with_plans(sheet, planned)?;
18251        for (row, col) in edited_cells {
18252            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18253            self.clear_cell_format_state(sheet, cell);
18254            self.record_formula_plane_changed_cell(sheet, row, col);
18255        }
18256        // Single topology bump after batch
18257        if n > 0 {
18258            self.mark_topology_edited();
18259        }
18260        Ok(n)
18261    }
18262
18263    #[inline]
18264    fn normalize_public_cell_read(v: LiteralValue) -> Option<LiteralValue> {
18265        match v {
18266            LiteralValue::Empty => None,
18267            LiteralValue::Int(i) => Some(LiteralValue::Number(i as f64)),
18268            other => Some(other),
18269        }
18270    }
18271
18272    fn materialize_temporal_egress(
18273        value: LiteralValue,
18274        class: Option<&formualizer_common::numfmt::FormatClass>,
18275        policy: crate::engine::TemporalEgress,
18276        date_system: crate::engine::DateSystem,
18277    ) -> LiteralValue {
18278        use formualizer_common::numfmt::FormatClass;
18279        if policy == crate::engine::TemporalEgress::Serial {
18280            return value;
18281        }
18282        let LiteralValue::Number(serial) = value else {
18283            return value;
18284        };
18285        match class {
18286            Some(FormatClass::Date) => {
18287                formualizer_common::try_serial_to_date_for(date_system, serial)
18288                    .map(LiteralValue::Date)
18289                    .unwrap_or(LiteralValue::Number(serial))
18290            }
18291            Some(FormatClass::DateTime) => {
18292                formualizer_common::try_serial_to_datetime_for(date_system, serial)
18293                    .map(LiteralValue::DateTime)
18294                    .unwrap_or(LiteralValue::Number(serial))
18295            }
18296            Some(FormatClass::Time) => {
18297                let seconds = (serial.rem_euclid(1.0) * 86_400.0).round() as u32 % 86_400;
18298                chrono::NaiveTime::from_num_seconds_from_midnight_opt(seconds, 0)
18299                    .map(LiteralValue::Time)
18300                    .unwrap_or(LiteralValue::Number(serial))
18301            }
18302            Some(FormatClass::Duration) => {
18303                let nanos = (serial * 86_400.0 * 1_000_000_000.0).round();
18304                if nanos.is_finite() && nanos >= i64::MIN as f64 && nanos <= i64::MAX as f64 {
18305                    LiteralValue::Duration(chrono::Duration::nanoseconds(nanos as i64))
18306                } else {
18307                    LiteralValue::Number(serial)
18308                }
18309            }
18310            _ => LiteralValue::Number(serial),
18311        }
18312    }
18313
18314    pub(crate) fn effective_format_id(
18315        &self,
18316        sheet: &str,
18317        row: u32,
18318        col: u32,
18319    ) -> Option<crate::format::FormatId> {
18320        let arrow = self.arrow_sheets.sheet(sheet).and_then(|arrow| {
18321            arrow.format_id(
18322                row.saturating_sub(1) as usize,
18323                col.saturating_sub(1) as usize,
18324            )
18325        });
18326        arrow.or_else(|| {
18327            let sheet_id = self.graph.sheet_id(sheet)?;
18328            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
18329            self.derived_formats.read().unwrap().get(&cell).copied()
18330        })
18331    }
18332
18333    /// Get a cell value through the single temporal egress boundary.
18334    pub fn get_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
18335        let raw = self.read_cell_value(sheet, row, col)?;
18336        let format = self.effective_format_id(sheet, row, col);
18337        let class = format.and_then(|id| self.format_registry.class(id));
18338        Self::normalize_public_cell_read(Self::materialize_temporal_egress(
18339            raw,
18340            class,
18341            self.config.temporal_egress,
18342            self.config.date_system,
18343        ))
18344    }
18345
18346    /// Read a rectangular range through the temporal egress boundary.
18347    pub fn get_range_values(
18348        &self,
18349        sheet: &str,
18350        sr: u32,
18351        sc: u32,
18352        er: u32,
18353        ec: u32,
18354    ) -> Vec<Vec<LiteralValue>> {
18355        let height = er.saturating_sub(sr).saturating_add(1) as usize;
18356        let width = ec.saturating_sub(sc).saturating_add(1) as usize;
18357        let Some(asheet) = self.sheet_store().sheet(sheet) else {
18358            return vec![vec![LiteralValue::Empty; width]; height];
18359        };
18360        let view = asheet.range_view(
18361            sr.saturating_sub(1) as usize,
18362            sc.saturating_sub(1) as usize,
18363            er.saturating_sub(1) as usize,
18364            ec.saturating_sub(1) as usize,
18365        );
18366        let sheet_id = self.graph.sheet_id(sheet);
18367        let derived_formats = self.derived_formats.read().unwrap();
18368        let has_derived_formats = sheet_id
18369            .is_some_and(|sheet_id| derived_formats.keys().any(|cell| cell.sheet_id == sheet_id));
18370        let mut out = Vec::with_capacity(height);
18371        if !asheet.has_formats() && !has_derived_formats {
18372            for rr in 0..height {
18373                let mut row = Vec::with_capacity(width);
18374                for cc in 0..width {
18375                    row.push(view.get_cell(rr, cc));
18376                }
18377                out.push(row);
18378            }
18379            return out;
18380        }
18381        let format_registry = &self.format_registry;
18382        for rr in 0..height {
18383            let mut row = Vec::with_capacity(width);
18384            for cc in 0..width {
18385                let raw = view.get_cell(rr, cc);
18386                let row0 = sr.saturating_sub(1).saturating_add(rr as u32);
18387                let col0 = sc.saturating_sub(1).saturating_add(cc as u32);
18388                let format = asheet.format_id(row0 as usize, col0 as usize).or_else(|| {
18389                    let cell = CellRef::new(sheet_id?, Coord::new(row0, col0, true, true));
18390                    derived_formats.get(&cell).copied()
18391                });
18392                let class = format.and_then(|id| format_registry.class(id));
18393                row.push(Self::materialize_temporal_egress(
18394                    raw,
18395                    class,
18396                    self.config.temporal_egress,
18397                    self.config.date_system,
18398                ));
18399            }
18400            out.push(row);
18401        }
18402        out
18403    }
18404
18405    /// Unified internal read API for a single cell value (Arrow-truth).
18406    pub(crate) fn read_cell_value(&self, sheet: &str, row: u32, col: u32) -> Option<LiteralValue> {
18407        let asheet = self.sheet_store().sheet(sheet)?;
18408        let r0 = row.saturating_sub(1) as usize;
18409        let c0 = col.saturating_sub(1) as usize;
18410        let v = asheet.get_cell_value(r0, c0);
18411        if matches!(v, LiteralValue::Empty) {
18412            None
18413        } else {
18414            Some(v)
18415        }
18416    }
18417
18418    /// Unified internal read API for a range of cell values (Arrow-truth).
18419    pub(crate) fn read_range_values(
18420        &self,
18421        sheet: &str,
18422        sr: u32,
18423        sc: u32,
18424        er: u32,
18425        ec: u32,
18426    ) -> RangeView<'_> {
18427        let Some(asheet) = self.sheet_store().sheet(sheet) else {
18428            return RangeView::from_owned_rows(Vec::new(), self.config.date_system);
18429        };
18430        if er < sr || ec < sc {
18431            return asheet.range_view(1, 1, 0, 0);
18432        }
18433        let sr0 = sr.saturating_sub(1) as usize;
18434        let sc0 = sc.saturating_sub(1) as usize;
18435        let er0 = er.saturating_sub(1) as usize;
18436        let ec0 = ec.saturating_sub(1) as usize;
18437        asheet.range_view(sr0, sc0, er0, ec0)
18438    }
18439
18440    pub(crate) fn formula_plane_span_ast_at(
18441        &self,
18442        span_ref: FormulaSpanRef,
18443        placement: PlacementCoord,
18444    ) -> Option<ASTNode> {
18445        let authority = self.graph.formula_authority();
18446        let span = authority.plane.spans.get(span_ref)?;
18447        let relocation = span.ast_relocation;
18448        let mut ast = self
18449            .graph
18450            .data_store()
18451            .retrieve_ast(relocation.ast_id, self.graph.sheet_reg())?;
18452        if let Some(binding_set_id) = span.binding_set_id {
18453            let binding_set = authority.plane.binding_sets.get(binding_set_id)?;
18454            if !binding_set.is_single_literal_binding() {
18455                let binding =
18456                    binding_set.literal_bindings_for_placement(&span.domain, placement)?;
18457                ast = substitute_formula_plane_literal_slots(&ast, binding.as_ref());
18458            }
18459        }
18460        let row = placement.row.checked_add(1)?;
18461        let col = placement.col.checked_add(1)?;
18462        let row_delta = i64::from(row) - i64::from(relocation.anchor_row);
18463        let col_delta = i64::from(col) - i64::from(relocation.anchor_col);
18464        relocate_ast_for_template_placement(&ast, row_delta, col_delta).ok()
18465    }
18466
18467    /// Get formula AST (if any) and current stored value for a cell
18468    pub fn get_cell(
18469        &self,
18470        sheet: &str,
18471        row: u32,
18472        col: u32,
18473    ) -> Option<(Option<formualizer_parse::ASTNode>, Option<LiteralValue>)> {
18474        let v = self.get_cell_value(sheet, row, col);
18475        let sheet_id = self.graph.sheet_id(sheet)?;
18476        let coord = Coord::from_excel(row, col, true, true);
18477        let cell = CellRef::new(sheet_id, coord);
18478        let placement =
18479            crate::formula_plane::runtime::PlacementCoord::new(sheet_id, coord.row(), coord.col());
18480        let handle = self
18481            .graph
18482            .formula_authority()
18483            .plane
18484            .resolve_formula_at(placement, None);
18485        match handle.resolution {
18486            crate::formula_plane::runtime::FormulaResolution::SpanPlacement { span, .. } => {
18487                // Span authority wins before graph lookup. Missing or invalid
18488                // relocation state is fail-closed: never expose a stale legacy
18489                // vertex for a coordinate owned by a span.
18490                let ast = self.formula_plane_span_ast_at(span, placement);
18491                return Some((ast, v));
18492            }
18493            crate::formula_plane::runtime::FormulaResolution::Overlay(overlay_ref) => {
18494                let ast = self
18495                    .graph
18496                    .formula_authority()
18497                    .plane
18498                    .formula_overlay
18499                    .get(overlay_ref)
18500                    .and_then(|overlay| match overlay.kind {
18501                        crate::formula_plane::runtime::FormulaOverlayEntryKind::FormulaOverride(
18502                            template_id,
18503                        ) => self
18504                            .graph
18505                            .formula_authority()
18506                            .plane
18507                            .templates
18508                            .get(template_id)
18509                            .and_then(|template| {
18510                                self.graph
18511                                    .data_store()
18512                                    .retrieve_ast(template.ast_id, self.graph.sheet_reg())
18513                            }),
18514                        _ => None,
18515                    });
18516                return Some((ast, v));
18517            }
18518            _ => {}
18519        }
18520
18521        if let Some(vid) = self.graph.get_vertex_for_cell(&cell) {
18522            let ast = self.graph.get_formula_id(vid).and_then(|ast_id| {
18523                self.graph
18524                    .data_store()
18525                    .retrieve_ast(ast_id, self.graph.sheet_reg())
18526            });
18527            Some((ast, v))
18528        } else if v.is_some() {
18529            Some((None, v))
18530        } else {
18531            None
18532        }
18533    }
18534
18535    /// Begin batch operations - defer CSR rebuilds for better performance
18536    pub fn begin_batch(&mut self) {
18537        self.graph.begin_batch();
18538    }
18539
18540    /// End batch operations and trigger CSR rebuild
18541    pub fn end_batch(&mut self) {
18542        self.graph.end_batch();
18543    }
18544
18545    /// Begin a deferred-dirty scope for a multi-edit batch: while active,
18546    /// every edit's dirty propagation queues its sources instead of running
18547    /// a full BFS per edit, and the outermost `end_deferred_dirty` flushes
18548    /// the union with ONE multi-source propagation (O(component) instead of
18549    /// O(edits × component)). See `DependencyGraph::begin_deferred_dirty`.
18550    ///
18551    /// Callers MUST run `end_deferred_dirty` on every exit path, including
18552    /// error returns; evaluation entry points `debug_assert` no scope leaked.
18553    pub fn begin_deferred_dirty(&mut self) {
18554        self.graph.begin_deferred_dirty();
18555    }
18556
18557    /// End a deferred-dirty scope, flushing the queued propagation when the
18558    /// outermost scope closes. See `Engine::begin_deferred_dirty`.
18559    pub fn end_deferred_dirty(&mut self) {
18560        let _ = self.graph.end_deferred_dirty();
18561    }
18562
18563    /// Total vertices processed by dirty-propagation BFS loops since graph
18564    /// creation. Perf-shape observability only (cross-crate tests assert
18565    /// batched edits propagate O(component), not O(edits × component)).
18566    pub fn dirty_propagation_visits(&self) -> u64 {
18567        self.graph.dirty_propagation_visits()
18568    }
18569
18570    /// Evaluate a single vertex.
18571    /// This is the core of the sequential evaluation logic for Milestone 3.1.
18572    #[inline]
18573    fn record_cell_if_changed(
18574        delta: &mut DeltaCollector,
18575        cell: &CellRef,
18576        old: &LiteralValue,
18577        new: &LiteralValue,
18578    ) {
18579        if old != new {
18580            delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
18581        }
18582    }
18583
18584    pub fn evaluate_vertex(&mut self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
18585        self.observe_evaluation_resource_request(EvaluationRequestKind::Vertex, |engine| {
18586            engine.observe_function_semantic_epoch()?;
18587            // A direct request selects exactly one vertex, regardless of its formula kind.
18588            engine.resource_checkpoint(1)?;
18589            if !engine.graph.vertex_exists(vertex_id) {
18590                return engine.evaluate_vertex_impl(vertex_id, None);
18591            }
18592            engine.transition_off_mode_spans_to_legacy()?;
18593            let is_formula = matches!(
18594                engine.graph.get_vertex_kind(vertex_id),
18595                VertexKind::FormulaScalar | VertexKind::FormulaArray
18596            );
18597            if is_formula {
18598                engine.begin_evaluation_request();
18599                engine.graph.flush_pending_edge_deltas();
18600                let roots = [crate::engine::target_preparation::TargetProducer::Legacy(
18601                    vertex_id,
18602                )];
18603                if engine.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
18604                    && engine.graph.formula_authority().active_span_count() > 0
18605                {
18606                    engine.evaluate_authoritative_formula_plane_targets(&roots, None)?;
18607                } else {
18608                    engine.evaluate_legacy_target_roots(&roots, None)?;
18609                }
18610            } else if engine.config.formula_plane_mode
18611                == FormulaPlaneMode::AuthoritativeExperimental
18612                && engine.graph.formula_authority().active_span_count() > 0
18613            {
18614                engine.begin_evaluation_request();
18615                engine.graph.flush_pending_edge_deltas();
18616                engine.evaluate_authoritative_formula_plane(None, None)?;
18617            }
18618            engine.evaluate_vertex_impl(vertex_id, None)
18619        })
18620    }
18621
18622    fn evaluate_vertex_impl(
18623        &mut self,
18624        vertex_id: VertexId,
18625        delta: Option<&mut DeltaCollector>,
18626    ) -> Result<LiteralValue, ExcelError> {
18627        // Preserve the direct evaluator's compatibility behavior for invalid IDs, literal cells,
18628        // names, and other non-formula vertices. Only formula publication needs the C1a final
18629        // deadline checkpoint and effects pipeline.
18630        if !self.graph.vertex_exists(vertex_id) {
18631            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
18632                .with_message(format!("Vertex not found: {vertex_id:?}")));
18633        }
18634        if self.active_resource_ledger.is_some()
18635            && matches!(
18636                self.graph.get_vertex_kind(vertex_id),
18637                VertexKind::FormulaScalar | VertexKind::FormulaArray
18638            )
18639        {
18640            let value = self
18641                .evaluate_vertex_immutable(vertex_id)
18642                .unwrap_or_else(LiteralValue::Error);
18643            let effects = self.plan_vertex_effects(vertex_id, value.clone(), None)?;
18644            // Do not publish the selected result until the outer request's deadline succeeds.
18645            self.resource_checkpoint(0)?;
18646            let mut delta = delta;
18647            for effect in &effects {
18648                self.apply_effect_with_computed_writes(effect, delta.as_deref_mut(), None, None)?;
18649            }
18650            return Ok(value);
18651        }
18652
18653        let mut delta = delta;
18654
18655        // Get vertex kind and check if it needs evaluation
18656        let kind = self.graph.get_vertex_kind(vertex_id);
18657        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
18658
18659        let ast_id = match kind {
18660            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
18661                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
18662                    ast_id
18663                } else {
18664                    return Ok(LiteralValue::Number(0.0));
18665                }
18666            }
18667            VertexKind::Empty | VertexKind::Cell => {
18668                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
18669                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
18670                    let row = cell_ref.coord.row() + 1;
18671                    let col = cell_ref.coord.col() + 1;
18672                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
18673                        return Ok(v);
18674                    }
18675                }
18676                return Ok(LiteralValue::Number(0.0));
18677            }
18678            VertexKind::NamedScalar => {
18679                let value = self.evaluate_named_scalar(vertex_id, sheet_id)?;
18680                return Ok(value);
18681            }
18682            VertexKind::NamedArray => {
18683                let value = self.evaluate_named_array(vertex_id, sheet_id)?;
18684                return Ok(value);
18685            }
18686            VertexKind::InfiniteRange
18687            | VertexKind::Range
18688            | VertexKind::External
18689            | VertexKind::Table => {
18690                // Not directly evaluatable here.
18691                return Ok(LiteralValue::Number(0.0));
18692            }
18693        };
18694
18695        // The interpreter uses a reference to the engine as the context.
18696        let sheet_name = self.graph.sheet_name(sheet_id);
18697        let cell_ref = self
18698            .graph
18699            .get_cell_ref(vertex_id)
18700            .expect("cell ref for vertex");
18701        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
18702
18703        let result =
18704            interpreter.evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg());
18705
18706        // If array result, perform spill from the anchor cell
18707        match result {
18708            Ok(cv) => {
18709                let derived_format = cv.format_id();
18710                self.record_derived_format(vertex_id, derived_format);
18711                let result_literal =
18712                    crate::engine::result_finalization::finalize_formula_result(cv.into_literal());
18713                let output_sheet_name = sheet_name.to_string();
18714                self.write_computed_overlay_format_0based(
18715                    &output_sheet_name,
18716                    cell_ref.coord.row(),
18717                    cell_ref.coord.col(),
18718                    derived_format,
18719                );
18720                match result_literal {
18721                    LiteralValue::Array(rows) => {
18722                        // Update kind to FormulaArray for tracking
18723                        self.graph
18724                            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
18725                        // Build target cells rectangle starting from anchor
18726                        let anchor = self
18727                            .graph
18728                            .get_cell_ref(vertex_id)
18729                            .expect("cell ref for vertex");
18730                        let sheet_id = anchor.sheet_id;
18731                        let h = rows.len() as u32;
18732                        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
18733
18734                        // Hard cap to avoid vertex explosion from huge dynamic arrays.
18735                        let spill_cells = (h as u64).saturating_mul(w as u64);
18736                        if spill_cells > self.config.spill.max_spill_cells as u64 {
18737                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
18738                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18739                                .with_message("SpillTooLarge")
18740                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18741                                    expected_rows: h,
18742                                    expected_cols: w,
18743                                });
18744                            let spill_val = LiteralValue::Error(spill_err.clone());
18745                            if let Some(d) = delta.as_deref_mut() {
18746                                let old = self
18747                                    .read_cell_value(
18748                                        self.graph.sheet_name(anchor.sheet_id),
18749                                        anchor.coord.row() + 1,
18750                                        anchor.coord.col() + 1,
18751                                    )
18752                                    .unwrap_or(LiteralValue::Empty);
18753                                if old != spill_val {
18754                                    d.record_cell(
18755                                        anchor.sheet_id,
18756                                        anchor.coord.row(),
18757                                        anchor.coord.col(),
18758                                    );
18759                                }
18760                            }
18761                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
18762                            if self.config.arrow_storage_enabled
18763                                && self.config.delta_overlay_enabled
18764                                && self.config.write_formula_overlay_enabled
18765                            {
18766                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
18767                                self.mirror_value_to_computed_overlay(
18768                                    &sheet_name,
18769                                    anchor.coord.row() + 1,
18770                                    anchor.coord.col() + 1,
18771                                    &spill_val,
18772                                );
18773                            }
18774                            return Ok(spill_val);
18775                        }
18776                        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
18777                        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
18778                        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
18779                        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
18780                        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
18781                        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
18782                            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
18783                            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18784                                .with_message("Spill exceeds sheet bounds")
18785                                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18786                                    expected_rows: h,
18787                                    expected_cols: w,
18788                                });
18789                            let spill_val = LiteralValue::Error(spill_err.clone());
18790                            if let Some(d) = delta.as_deref_mut() {
18791                                let old = self
18792                                    .read_cell_value(
18793                                        self.graph.sheet_name(anchor.sheet_id),
18794                                        anchor.coord.row() + 1,
18795                                        anchor.coord.col() + 1,
18796                                    )
18797                                    .unwrap_or(LiteralValue::Empty);
18798                                if old != spill_val {
18799                                    d.record_cell(
18800                                        anchor.sheet_id,
18801                                        anchor.coord.row(),
18802                                        anchor.coord.col(),
18803                                    );
18804                                }
18805                            }
18806                            self.graph.update_vertex_value(vertex_id, spill_val.clone());
18807                            if self.config.arrow_storage_enabled
18808                                && self.config.delta_overlay_enabled
18809                                && self.config.write_formula_overlay_enabled
18810                            {
18811                                let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
18812                                self.mirror_value_to_computed_overlay(
18813                                    &sheet_name,
18814                                    anchor.coord.row() + 1,
18815                                    anchor.coord.col() + 1,
18816                                    &spill_val,
18817                                );
18818                            }
18819                            return Ok(spill_val);
18820                        }
18821                        let mut targets = Vec::new();
18822                        for r in 0..h {
18823                            for c in 0..w {
18824                                targets.push(self.graph.make_cell_ref_internal(
18825                                    sheet_id,
18826                                    anchor.coord.row() + r,
18827                                    anchor.coord.col() + c,
18828                                ));
18829                            }
18830                        }
18831
18832                        // Plan spill via spill manager shim
18833                        match self.spill_mgr.reserve(
18834                            vertex_id,
18835                            anchor,
18836                            SpillShape { rows: h, cols: w },
18837                            SpillMeta {
18838                                epoch: self.recalc_epoch,
18839                                config: self.config.spill,
18840                            },
18841                        ) {
18842                            Ok(()) => {
18843                                // Commit: write values to grid
18844                                // Default conflict policy is Error + FirstWins; reserve() enforces in-flight locks
18845                                // and plan_spill_region enforces overlap with committed formulas/spills/values.
18846                                if let Err(e) = self.commit_spill_and_mirror(
18847                                    vertex_id,
18848                                    &targets,
18849                                    rows.clone(),
18850                                    delta.as_deref_mut(),
18851                                    None,
18852                                ) {
18853                                    // If commit fails, mark as error
18854                                    self.clear_spill_projection_and_mirror(
18855                                        vertex_id,
18856                                        delta.as_deref_mut(),
18857                                    );
18858                                    if let Some(d) = delta.as_deref_mut() {
18859                                        let old = self
18860                                            .read_cell_value(
18861                                                self.graph.sheet_name(anchor.sheet_id),
18862                                                anchor.coord.row() + 1,
18863                                                anchor.coord.col() + 1,
18864                                            )
18865                                            .unwrap_or(LiteralValue::Empty);
18866                                        let new = LiteralValue::Error(e.clone());
18867                                        if old != new {
18868                                            d.record_cell(
18869                                                anchor.sheet_id,
18870                                                anchor.coord.row(),
18871                                                anchor.coord.col(),
18872                                            );
18873                                        }
18874                                    }
18875                                    let err_val = LiteralValue::Error(e.clone());
18876                                    self.graph.update_vertex_value(vertex_id, err_val.clone());
18877                                    if self.config.arrow_storage_enabled
18878                                        && self.config.delta_overlay_enabled
18879                                        && self.config.write_formula_overlay_enabled
18880                                    {
18881                                        let sheet_name =
18882                                            self.graph.sheet_name(anchor.sheet_id).to_string();
18883                                        self.mirror_value_to_computed_overlay(
18884                                            &sheet_name,
18885                                            anchor.coord.row() + 1,
18886                                            anchor.coord.col() + 1,
18887                                            &err_val,
18888                                        );
18889                                    }
18890                                    return Ok(err_val);
18891                                }
18892                                // Anchor shows the top-left value, like Excel
18893                                let top_left = rows
18894                                    .first()
18895                                    .and_then(|r| r.first())
18896                                    .cloned()
18897                                    .unwrap_or(LiteralValue::Empty);
18898                                self.graph.update_vertex_value(vertex_id, top_left.clone());
18899                                Ok(top_left)
18900                            }
18901                            Err(e) => {
18902                                self.clear_spill_projection_and_mirror(
18903                                    vertex_id,
18904                                    delta.as_deref_mut(),
18905                                );
18906                                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
18907                                    .with_message(
18908                                        e.message.unwrap_or_else(|| "Spill blocked".to_string()),
18909                                    )
18910                                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
18911                                        expected_rows: h,
18912                                        expected_cols: w,
18913                                    });
18914                                let spill_val = LiteralValue::Error(spill_err.clone());
18915                                if let Some(d) = delta.as_deref_mut() {
18916                                    let old = self
18917                                        .read_cell_value(
18918                                            self.graph.sheet_name(anchor.sheet_id),
18919                                            anchor.coord.row() + 1,
18920                                            anchor.coord.col() + 1,
18921                                        )
18922                                        .unwrap_or(LiteralValue::Empty);
18923                                    if old != spill_val {
18924                                        d.record_cell(
18925                                            anchor.sheet_id,
18926                                            anchor.coord.row(),
18927                                            anchor.coord.col(),
18928                                        );
18929                                    }
18930                                }
18931                                self.graph.update_vertex_value(vertex_id, spill_val.clone());
18932                                if self.config.arrow_storage_enabled
18933                                    && self.config.delta_overlay_enabled
18934                                    && self.config.write_formula_overlay_enabled
18935                                {
18936                                    let sheet_name =
18937                                        self.graph.sheet_name(anchor.sheet_id).to_string();
18938                                    self.mirror_value_to_computed_overlay(
18939                                        &sheet_name,
18940                                        anchor.coord.row() + 1,
18941                                        anchor.coord.col() + 1,
18942                                        &spill_val,
18943                                    );
18944                                }
18945                                Ok(spill_val)
18946                            }
18947                        }
18948                    }
18949                    other => {
18950                        // Scalar result: store value and ensure any previous spill is cleared
18951                        let spill_cells = self
18952                            .graph
18953                            .spill_cells_for_anchor(vertex_id)
18954                            .map(|cells| cells.to_vec())
18955                            .unwrap_or_default();
18956                        if let Some(d) = delta.as_deref_mut()
18957                            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
18958                        {
18959                            if spill_cells.is_empty() {
18960                                let old = self
18961                                    .read_cell_value(
18962                                        self.graph.sheet_name(anchor.sheet_id),
18963                                        anchor.coord.row() + 1,
18964                                        anchor.coord.col() + 1,
18965                                    )
18966                                    .unwrap_or(LiteralValue::Empty);
18967                                if old != other {
18968                                    d.record_cell(
18969                                        anchor.sheet_id,
18970                                        anchor.coord.row(),
18971                                        anchor.coord.col(),
18972                                    );
18973                                }
18974                            } else {
18975                                for cell in spill_cells.iter() {
18976                                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
18977                                    let old = self
18978                                        .get_cell_value(
18979                                            sheet_name,
18980                                            cell.coord.row() + 1,
18981                                            cell.coord.col() + 1,
18982                                        )
18983                                        .unwrap_or(LiteralValue::Empty);
18984                                    let new = if cell.sheet_id == anchor.sheet_id
18985                                        && cell.coord.row() == anchor.coord.row()
18986                                        && cell.coord.col() == anchor.coord.col()
18987                                    {
18988                                        other.clone()
18989                                    } else {
18990                                        LiteralValue::Empty
18991                                    };
18992                                    Self::record_cell_if_changed(d, cell, &old, &new);
18993                                }
18994                            }
18995                        }
18996                        self.graph.clear_spill_region(vertex_id);
18997                        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
18998                            self.record_formula_plane_structural_change(scope);
18999                        }
19000                        if self.config.arrow_storage_enabled
19001                            && self.config.delta_overlay_enabled
19002                            && self.config.write_formula_overlay_enabled
19003                        {
19004                            let empty = LiteralValue::Empty;
19005                            for cell in spill_cells.iter() {
19006                                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19007                                self.mirror_value_to_computed_overlay(
19008                                    &sheet_name,
19009                                    cell.coord.row() + 1,
19010                                    cell.coord.col() + 1,
19011                                    &empty,
19012                                );
19013                            }
19014                        }
19015                        self.graph.update_vertex_value(vertex_id, other.clone());
19016                        // Optionally mirror into Arrow overlay for Arrow-backed reads
19017                        if self.config.arrow_storage_enabled
19018                            && self.config.delta_overlay_enabled
19019                            && self.config.write_formula_overlay_enabled
19020                        {
19021                            let anchor = self
19022                                .graph
19023                                .get_cell_ref(vertex_id)
19024                                .expect("cell ref for vertex");
19025                            let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
19026                            self.mirror_value_to_computed_overlay(
19027                                &sheet_name,
19028                                anchor.coord.row() + 1,
19029                                anchor.coord.col() + 1,
19030                                &other,
19031                            );
19032                        }
19033                        Ok(other)
19034                    }
19035                }
19036            }
19037            Err(e) => {
19038                // Runtime Excel error: store as a cell value instead of propagating
19039                // as an exception so bulk eval paths don't fail the whole pass.
19040                let spill_cells = self
19041                    .graph
19042                    .spill_cells_for_anchor(vertex_id)
19043                    .map(|cells| cells.to_vec())
19044                    .unwrap_or_default();
19045                let err_val = LiteralValue::Error(e.clone());
19046                if let Some(d) = delta
19047                    && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
19048                {
19049                    if spill_cells.is_empty() {
19050                        let old = self
19051                            .read_cell_value(
19052                                self.graph.sheet_name(anchor.sheet_id),
19053                                anchor.coord.row() + 1,
19054                                anchor.coord.col() + 1,
19055                            )
19056                            .unwrap_or(LiteralValue::Empty);
19057                        if old != err_val {
19058                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
19059                        }
19060                    } else {
19061                        for cell in spill_cells.iter() {
19062                            let sheet_name = self.graph.sheet_name(cell.sheet_id);
19063                            let old = self
19064                                .get_cell_value(
19065                                    sheet_name,
19066                                    cell.coord.row() + 1,
19067                                    cell.coord.col() + 1,
19068                                )
19069                                .unwrap_or(LiteralValue::Empty);
19070                            let new = if cell.sheet_id == anchor.sheet_id
19071                                && cell.coord.row() == anchor.coord.row()
19072                                && cell.coord.col() == anchor.coord.col()
19073                            {
19074                                err_val.clone()
19075                            } else {
19076                                LiteralValue::Empty
19077                            };
19078                            Self::record_cell_if_changed(d, cell, &old, &new);
19079                        }
19080                    }
19081                }
19082                self.graph.clear_spill_region(vertex_id);
19083                if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
19084                    self.record_formula_plane_structural_change(scope);
19085                }
19086                if self.config.arrow_storage_enabled
19087                    && self.config.delta_overlay_enabled
19088                    && self.config.write_formula_overlay_enabled
19089                {
19090                    let empty = LiteralValue::Empty;
19091                    for cell in spill_cells.iter() {
19092                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
19093                        self.mirror_value_to_computed_overlay(
19094                            &sheet_name,
19095                            cell.coord.row() + 1,
19096                            cell.coord.col() + 1,
19097                            &empty,
19098                        );
19099                    }
19100                }
19101                self.graph.update_vertex_value(vertex_id, err_val.clone());
19102                if self.config.arrow_storage_enabled
19103                    && self.config.delta_overlay_enabled
19104                    && self.config.write_formula_overlay_enabled
19105                {
19106                    let anchor = self
19107                        .graph
19108                        .get_cell_ref(vertex_id)
19109                        .expect("cell ref for vertex");
19110                    let sheet_name = self.graph.sheet_name(anchor.sheet_id).to_string();
19111                    self.mirror_value_to_computed_overlay(
19112                        &sheet_name,
19113                        anchor.coord.row() + 1,
19114                        anchor.coord.col() + 1,
19115                        &err_val,
19116                    );
19117                }
19118                Ok(err_val)
19119            }
19120        }
19121    }
19122
19123    fn evaluate_named_scalar(
19124        &mut self,
19125        vertex_id: VertexId,
19126        sheet_id: SheetId,
19127    ) -> Result<LiteralValue, ExcelError> {
19128        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19129            ExcelError::new(ExcelErrorKind::Name)
19130                .with_message("Named range metadata missing".to_string())
19131        })?;
19132
19133        match &named_range.definition {
19134            NamedDefinition::Cell(cell_ref) => {
19135                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19136                let row = cell_ref.coord.row() + 1;
19137                let col = cell_ref.coord.col() + 1;
19138
19139                if let Some(dep_vertex) = self.graph.get_vertex_for_cell(cell_ref)
19140                    && matches!(
19141                        self.graph.get_vertex_kind(dep_vertex),
19142                        VertexKind::FormulaScalar | VertexKind::FormulaArray
19143                    )
19144                {
19145                    // Graph does not cache cell/formula values; ensure the precedent is evaluated.
19146                    let value = self.evaluate_vertex(dep_vertex)?;
19147                    self.graph.update_vertex_value(vertex_id, value.clone());
19148                    Ok(value)
19149                } else {
19150                    let value = self
19151                        .get_cell_value(sheet_name, row, col)
19152                        .unwrap_or(LiteralValue::Empty);
19153                    self.graph.update_vertex_value(vertex_id, value.clone());
19154                    Ok(value)
19155                }
19156            }
19157            NamedDefinition::Literal(v) => {
19158                let out = v.clone();
19159                self.graph.update_vertex_value(vertex_id, out.clone());
19160                Ok(out)
19161            }
19162            NamedDefinition::Formula { ast, .. } => {
19163                let context_sheet = match named_range.scope {
19164                    NameScope::Sheet(id) => id,
19165                    NameScope::Workbook => sheet_id,
19166                };
19167                let sheet_name = self.graph.sheet_name(context_sheet);
19168                let cell_ref = self
19169                    .graph
19170                    .get_cell_ref(vertex_id)
19171                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19172                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
19173                match interpreter.evaluate_ast(ast) {
19174                    Ok(cv) => {
19175                        let value = cv.into_literal();
19176                        match value {
19177                            LiteralValue::Array(_) => {
19178                                let err = ExcelError::new(ExcelErrorKind::NImpl)
19179                                    .with_message("Array result in scalar named range".to_string());
19180                                let err_val = LiteralValue::Error(err.clone());
19181                                self.graph.update_vertex_value(vertex_id, err_val.clone());
19182                                Ok(err_val)
19183                            }
19184                            other => {
19185                                self.graph.update_vertex_value(vertex_id, other.clone());
19186                                Ok(other)
19187                            }
19188                        }
19189                    }
19190                    Err(err) => {
19191                        let err_val = LiteralValue::Error(err.clone());
19192                        self.graph.update_vertex_value(vertex_id, err_val.clone());
19193                        Ok(err_val)
19194                    }
19195                }
19196            }
19197            NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
19198                .with_message("Range-valued name evaluated as scalar".to_string())),
19199        }
19200    }
19201
19202    fn evaluate_named_array(
19203        &mut self,
19204        vertex_id: VertexId,
19205        sheet_id: SheetId,
19206    ) -> Result<LiteralValue, ExcelError> {
19207        let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
19208            ExcelError::new(ExcelErrorKind::Name)
19209                .with_message("Named range metadata missing".to_string())
19210        })?;
19211
19212        let out = match &named_range.definition {
19213            NamedDefinition::Range(range_ref) => {
19214                if range_ref.start.sheet_id != range_ref.end.sheet_id {
19215                    return Err(ExcelError::new(ExcelErrorKind::Ref)
19216                        .with_message("Named range cannot span sheets".to_string()));
19217                }
19218
19219                let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
19220                let sr0 = range_ref.start.coord.row();
19221                let sc0 = range_ref.start.coord.col();
19222                let er0 = range_ref.end.coord.row();
19223                let ec0 = range_ref.end.coord.col();
19224                if sr0 > er0 || sc0 > ec0 {
19225                    return Err(ExcelError::new(ExcelErrorKind::Ref)
19226                        .with_message("Invalid named range bounds".to_string()));
19227                }
19228
19229                let h = (er0 - sr0 + 1) as usize;
19230                let w = (ec0 - sc0 + 1) as usize;
19231                let cell_count = (h as u64).saturating_mul(w as u64);
19232                if cell_count > self.config.spill.max_spill_cells as u64 {
19233                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
19234                        "Named range too large to materialize as an array".to_string(),
19235                    ));
19236                }
19237
19238                let mut rows = Vec::with_capacity(h);
19239                for r0 in sr0..=er0 {
19240                    let mut row = Vec::with_capacity(w);
19241                    for c0 in sc0..=ec0 {
19242                        let v = self
19243                            .get_cell_value(sheet_name, r0 + 1, c0 + 1)
19244                            .unwrap_or(LiteralValue::Empty);
19245                        row.push(v);
19246                    }
19247                    rows.push(row);
19248                }
19249                LiteralValue::Array(rows)
19250            }
19251            NamedDefinition::Cell(cell_ref) => {
19252                let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
19253                let row = cell_ref.coord.row() + 1;
19254                let col = cell_ref.coord.col() + 1;
19255                let v = self
19256                    .get_cell_value(sheet_name, row, col)
19257                    .unwrap_or(LiteralValue::Empty);
19258                LiteralValue::Array(vec![vec![v]])
19259            }
19260            NamedDefinition::Literal(v) => LiteralValue::Array(vec![vec![v.clone()]]),
19261            NamedDefinition::Formula { ast, .. } => {
19262                let context_sheet = match named_range.scope {
19263                    NameScope::Sheet(id) => id,
19264                    NameScope::Workbook => sheet_id,
19265                };
19266                let sheet_name = self.graph.sheet_name(context_sheet);
19267                let cell_ref = self
19268                    .graph
19269                    .get_cell_ref(vertex_id)
19270                    .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
19271                let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
19272                match interpreter.evaluate_ast(ast) {
19273                    Ok(cv) => {
19274                        let v = cv.into_literal();
19275                        match v {
19276                            LiteralValue::Array(_) => v,
19277                            other => LiteralValue::Array(vec![vec![other]]),
19278                        }
19279                    }
19280                    Err(err) => LiteralValue::Error(err),
19281                }
19282            }
19283        };
19284
19285        self.graph.update_vertex_value(vertex_id, out.clone());
19286        Ok(out)
19287    }
19288
19289    fn replan_exhausted_error(&self, limit: usize, context: &str) -> ExcelError {
19290        crate::engine::ResourceLedgerError::Exhausted(
19291            formualizer_common::ResourceExhaustionDetail {
19292                reason: formualizer_common::ResourceExhaustionReason::WorkUnits,
19293                limit: limit as u64,
19294                observed: limit.saturating_add(1) as u64,
19295                request_id: self
19296                    .active_evaluation_resource_request
19297                    .as_ref()
19298                    .map(|stats| stats.request_id),
19299            },
19300        )
19301        .into_excel_error()
19302        .with_message(format!("{context} did not converge after {limit} replans"))
19303    }
19304
19305    fn transient_target_preparation_stale(error: &ExcelError) -> bool {
19306        matches!(
19307            &error.extra,
19308            formualizer_common::ExcelErrorExtra::PreparationStale {
19309                reason: formualizer_common::PreparationStaleReason::Semantic
19310                    | formualizer_common::PreparationStaleReason::Provider
19311            }
19312        )
19313    }
19314
19315    fn prepare_graph_for_routed_evaluation(
19316        &mut self,
19317        targets: &[crate::engine::EvaluationTarget],
19318        options: &crate::engine::TargetEvalOptions<'_>,
19319    ) -> Result<crate::engine::PreparedTargetGraphReport, ExcelError> {
19320        const MAX_TRANSIENT_PREPARATION_RETRIES: usize = 2;
19321        let mut retries = 0usize;
19322        loop {
19323            match self.prepare_graph_for_targets_unobserved(targets, options) {
19324                Err(error)
19325                    if Self::transient_target_preparation_stale(&error)
19326                        && retries < MAX_TRANSIENT_PREPARATION_RETRIES =>
19327                {
19328                    retries = retries.saturating_add(1);
19329                }
19330                result => return result,
19331            }
19332        }
19333    }
19334
19335    fn evaluate_mixed_targets(
19336        &mut self,
19337        targets: &[crate::engine::EvaluationTarget],
19338        delta: Option<&mut DeltaCollector>,
19339    ) -> Result<EvalResult, ExcelError> {
19340        let _source_cache = self.source_cache_session();
19341        let cancel = self.active_cancel_flag.clone();
19342        let options = crate::engine::TargetEvalOptions {
19343            request_id: self
19344                .active_evaluation_resource_request
19345                .as_ref()
19346                .map(|stats| stats.request_id),
19347            cancel,
19348            deadline: None,
19349            budgets: None,
19350            opaque_policy: crate::engine::OpaquePreparePolicy::Widen,
19351        };
19352        self.prepare_and_execute_target_recipe(targets, &options, delta)
19353    }
19354
19355    fn prepare_and_execute_target_recipe(
19356        &mut self,
19357        targets: &[crate::engine::EvaluationTarget],
19358        options: &crate::engine::TargetEvalOptions<'_>,
19359        delta: Option<&mut DeltaCollector>,
19360    ) -> Result<EvalResult, ExcelError> {
19361        let preparation = self.prepare_graph_for_routed_evaluation(targets, options)?;
19362        self.execute_prepared_target_recipe(targets, &preparation.widened_scope, delta)
19363    }
19364
19365    fn execute_prepared_target_recipe(
19366        &mut self,
19367        targets: &[crate::engine::EvaluationTarget],
19368        scope: &crate::engine::PrepareScope,
19369        delta: Option<&mut DeltaCollector>,
19370    ) -> Result<EvalResult, ExcelError> {
19371        self.transition_off_mode_spans_to_legacy()?;
19372        if matches!(scope, crate::engine::PrepareScope::Workbook)
19373            && let Some(stats) = self.active_evaluation_resource_request.as_mut()
19374        {
19375            stats.workbook_exact_attempts = stats.workbook_exact_attempts.max(1);
19376        }
19377        let mut roots = self.resolve_target_producers(targets)?;
19378        if let crate::engine::PrepareScope::Sheets(sheets) = scope {
19379            let request_id = self
19380                .active_evaluation_resource_request
19381                .as_ref()
19382                .map(|request| request.request_id);
19383            let root_count = roots.len();
19384            let mut widened_roots =
19385                OrderedTargetProducers::from_ordered(std::mem::take(&mut roots))
19386                    .map_err(|_| target_root_allocation_error(root_count, request_id))?;
19387            let sheet_ids = sheets
19388                .iter()
19389                .filter_map(|sheet| self.graph.sheet_id(sheet))
19390                .collect::<FxHashSet<_>>();
19391            for vertex in self.graph.formula_vertices() {
19392                if sheet_ids.contains(&self.graph.get_vertex_sheet_id(vertex)) {
19393                    widened_roots
19394                        .push(crate::engine::target_preparation::TargetProducer::Legacy(
19395                            vertex,
19396                        ))
19397                        .map_err(|_| {
19398                            target_root_allocation_error(widened_roots.len() + 1, request_id)
19399                        })?;
19400                }
19401            }
19402            let authority = self.graph.formula_authority();
19403            for span_ref in authority.active_span_refs() {
19404                let Some(span) = authority.plane.spans.get(span_ref) else {
19405                    continue;
19406                };
19407                if sheet_ids.contains(&span.sheet_id) {
19408                    widened_roots
19409                        .push(crate::engine::target_preparation::TargetProducer::Span {
19410                            span_ref,
19411                            demanded: Region::from_domain(span.result_region.domain()),
19412                        })
19413                        .map_err(|_| {
19414                            target_root_allocation_error(widened_roots.len() + 1, request_id)
19415                        })?;
19416                }
19417            }
19418            roots = widened_roots.into_vec();
19419        }
19420        self.begin_evaluation_request();
19421        self.graph.flush_pending_edge_deltas();
19422        let workbook_scope = matches!(scope, crate::engine::PrepareScope::Workbook);
19423        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
19424            && self.graph.formula_authority().active_span_count() > 0
19425        {
19426            if workbook_scope {
19427                self.evaluate_authoritative_formula_plane(None, delta)
19428            } else {
19429                self.evaluate_authoritative_formula_plane_targets(&roots, delta)
19430            }
19431        } else {
19432            if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
19433                self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
19434            }
19435            if workbook_scope {
19436                if let Some(delta) = delta {
19437                    self.evaluate_all_with_delta_collector(delta)
19438                } else {
19439                    self.evaluate_all_legacy_impl()
19440                }
19441            } else {
19442                self.evaluate_legacy_target_roots(&roots, delta)
19443            }
19444        }
19445    }
19446
19447    fn legacy_coordinate_targets(
19448        &mut self,
19449        targets: &[(&str, u32, u32)],
19450    ) -> Vec<crate::engine::EvaluationTarget> {
19451        targets
19452            .iter()
19453            .map(|(sheet, row, col)| {
19454                // Compatibility APIs historically interned an unknown target sheet
19455                // and returned an empty value rather than rejecting the target.
19456                self.graph.sheet_id_mut(sheet);
19457                crate::engine::EvaluationTarget::Cell {
19458                    sheet: (*sheet).to_string(),
19459                    row: *row,
19460                    col: *col,
19461                }
19462            })
19463            .collect()
19464    }
19465
19466    /// Evaluate the necessary mixed producer closure for typed cell, range, name, and table targets.
19467    pub fn evaluate_targets(
19468        &mut self,
19469        targets: &[crate::engine::EvaluationTarget],
19470    ) -> Result<EvalResult, ExcelError> {
19471        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19472            engine.observe_function_semantic_epoch()?;
19473            engine.validate_deterministic_mode()?;
19474            engine.evaluate_mixed_targets(targets, None)
19475        })
19476    }
19477
19478    /// Evaluate typed targets with explicit preparation policy and request controls.
19479    pub fn evaluate_targets_with_options(
19480        &mut self,
19481        targets: &[crate::engine::EvaluationTarget],
19482        options: crate::engine::TargetEvalOptions<'_>,
19483    ) -> Result<EvalResult, ExcelError> {
19484        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19485            engine.active_cancel_flag = options.cancel.clone();
19486            engine.active_evaluation_deadline = options.deadline;
19487            let result = (|| {
19488                engine.cancellation_checkpoint("Evaluation cancelled before target preparation")?;
19489                engine.observe_function_semantic_epoch()?;
19490                engine.validate_deterministic_mode()?;
19491                let _source_cache = engine.source_cache_session();
19492                engine.prepare_and_execute_target_recipe(targets, &options, None)
19493            })();
19494            engine.active_cancel_flag = None;
19495            engine.active_evaluation_deadline = None;
19496            result
19497        })
19498    }
19499
19500    /// Evaluate typed targets and return the versioned run/region delta for the request.
19501    pub fn evaluate_targets_with_delta(
19502        &mut self,
19503        targets: &[crate::engine::EvaluationTarget],
19504    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
19505        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
19506            engine.observe_function_semantic_epoch()?;
19507            engine.validate_deterministic_mode()?;
19508            let mut collector = DeltaCollector::new(DeltaMode::Cells);
19509            let result = engine.evaluate_mixed_targets(targets, Some(&mut collector))?;
19510            Ok((result, collector.finish_target()))
19511        })
19512    }
19513
19514    /// Evaluate only the necessary precedents for specific target cells (demand-driven)
19515    pub fn evaluate_until(
19516        &mut self,
19517        targets: &[(&str, u32, u32)],
19518    ) -> Result<EvalResult, ExcelError> {
19519        self.observe_evaluation_resource_request(EvaluationRequestKind::Targeted, |engine| {
19520            engine.evaluate_until_unobserved(targets)
19521        })
19522    }
19523
19524    fn evaluate_until_unobserved(
19525        &mut self,
19526        targets: &[(&str, u32, u32)],
19527    ) -> Result<EvalResult, ExcelError> {
19528        self.observe_function_semantic_epoch()?;
19529        let targets = self.legacy_coordinate_targets(targets);
19530        self.evaluate_mixed_targets(&targets, None)
19531    }
19532
19533    fn evaluate_until_with_delta_collector(
19534        &mut self,
19535        targets: &[(&str, u32, u32)],
19536        delta: &mut DeltaCollector,
19537    ) -> Result<EvalResult, ExcelError> {
19538        let targets = self.legacy_coordinate_targets(targets);
19539        self.evaluate_mixed_targets(&targets, Some(delta))
19540    }
19541
19542    fn evaluate_legacy_target_roots(
19543        &mut self,
19544        roots: &[crate::engine::target_preparation::TargetProducer],
19545        mut delta: Option<&mut DeltaCollector>,
19546    ) -> Result<EvalResult, ExcelError> {
19547        use crate::engine::target_preparation::TargetProducer;
19548        #[cfg(any(test, feature = "benchmark_internal"))]
19549        {
19550            self.recalc_reuse_probe
19551                .get_mut()
19552                .unwrap()
19553                .legacy_target_requests += 1;
19554        }
19555        let start = crate::instant::FzInstant::now();
19556        let root_vertices = roots
19557            .iter()
19558            .filter_map(|root| match root {
19559                TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => Some(*vertex),
19560                TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
19561            })
19562            .collect::<Vec<_>>();
19563        let mut computed_vertices = 0usize;
19564        let mut cycle_errors = 0usize;
19565        let mut replans = 0usize;
19566        const MAX_REPLAN: usize = 5;
19567        loop {
19568            let (precedents_to_eval, old_vdeps) = self.build_demand_subgraph(&root_vertices);
19569            if precedents_to_eval.is_empty() {
19570                break;
19571            }
19572            #[cfg(any(test, feature = "benchmark_internal"))]
19573            {
19574                self.recalc_reuse_probe
19575                    .get_mut()
19576                    .unwrap()
19577                    .target_schedule_builds += 1;
19578            }
19579            let scheduler = Scheduler::new(&self.graph);
19580            let schedule =
19581                scheduler.create_schedule_with_virtual(&precedents_to_eval, &old_vdeps)?;
19582            for &unit in &schedule.units {
19583                self.cancellation_checkpoint("Evaluation cancelled before target schedule unit")?;
19584                match unit {
19585                    ScheduleUnit::Cycle(index) => {
19586                        if self.handle_cycle_unit(
19587                            schedule.unit_cycle(index),
19588                            delta.as_deref_mut(),
19589                            None,
19590                            None,
19591                        )? > 0
19592                        {
19593                            cycle_errors = cycle_errors.saturating_add(1);
19594                        }
19595                    }
19596                    ScheduleUnit::Layer(index) => {
19597                        let layer = schedule.unit_layer(index);
19598                        let evaluated = if let Some(delta) = delta.as_deref_mut() {
19599                            if self.thread_pool.is_some() && layer.vertices.len() > 1 {
19600                                self.evaluate_layer_parallel_with_delta(layer, delta)?
19601                            } else {
19602                                self.evaluate_layer_sequential_with_delta(layer, delta)?
19603                            }
19604                        } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
19605                            self.evaluate_layer_parallel(layer)?
19606                        } else {
19607                            self.evaluate_layer_sequential(layer)?
19608                        };
19609                        computed_vertices = computed_vertices.saturating_add(evaluated);
19610                    }
19611                }
19612            }
19613            let changed = self.changed_virtual_dep_vertices(&precedents_to_eval, &old_vdeps);
19614            self.resource_checkpoint(0)?;
19615            self.graph.clear_dirty_flags(&precedents_to_eval);
19616            for vertex in &changed {
19617                self.graph.set_dirty(*vertex, true);
19618            }
19619            if changed.is_empty() {
19620                break;
19621            }
19622            if replans >= MAX_REPLAN {
19623                return Err(self.replan_exhausted_error(
19624                    MAX_REPLAN,
19625                    "targeted legacy dynamic dependency evaluation",
19626                ));
19627            }
19628            replans = replans.saturating_add(1);
19629        }
19630        self.redirty_for_next_recalc();
19631        Ok(EvalResult {
19632            computed_vertices,
19633            cycle_errors,
19634            elapsed: start.elapsed(),
19635        })
19636    }
19637
19638    /// Build a revision-bound compatibility plan covering every prepared formula vertex.
19639    pub fn build_recalc_plan(&self) -> Result<RecalcPlan, ExcelError> {
19640        if self.has_staged_formulas() || self.staged_formula_index.has_packages() {
19641            return Err(
19642                Self::plan_stale(formualizer_common::PlanStaleReason::Staged).with_message(
19643                    "compatibility recalculation plans require all staged formulas to be prepared",
19644                ),
19645            );
19646        }
19647        let key = self.recalc_plan_key();
19648        let mut vertices: Vec<VertexId> = self.graph.vertices_with_formulas().collect();
19649        vertices.sort_unstable();
19650        let has_dynamic_refs = vertices.iter().copied().any(|v| self.graph.is_dynamic(v));
19651        let schedule = if vertices.is_empty() {
19652            crate::engine::Schedule {
19653                units: Vec::new(),
19654                layers: Vec::new(),
19655                cycles: Vec::new(),
19656            }
19657        } else {
19658            self.create_evaluation_schedule_uncached(&vertices)?.0
19659        };
19660        self.validate_recalc_plan_key(&key)?;
19661        Ok(RecalcPlan {
19662            key,
19663            kind: RecalcPlanKind::CompatibilityFull {
19664                schedule,
19665                has_dynamic_refs,
19666            },
19667        })
19668    }
19669
19670    /// Prepare stable typed targets and retain a revision-bound run-local recipe.
19671    pub fn build_recalc_plan_for_targets(
19672        &mut self,
19673        targets: &[crate::engine::EvaluationTarget],
19674    ) -> Result<RecalcPlan, ExcelError> {
19675        self.build_recalc_plan_for_targets_with_options(
19676            targets,
19677            crate::engine::TargetEvalOptions::default(),
19678        )
19679    }
19680
19681    pub fn build_recalc_plan_for_targets_with_options(
19682        &mut self,
19683        targets: &[crate::engine::EvaluationTarget],
19684        options: crate::engine::TargetEvalOptions<'_>,
19685    ) -> Result<RecalcPlan, ExcelError> {
19686        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19687            engine.observe_function_semantic_epoch()?;
19688            engine.validate_deterministic_mode()?;
19689            let _source_cache = engine.source_cache_session();
19690            let preparation = engine.prepare_graph_for_routed_evaluation(targets, &options)?;
19691            engine.graph.flush_pending_edge_deltas();
19692            let topology = if matches!(
19693                preparation.widened_scope,
19694                crate::engine::PrepareScope::Workbook
19695            ) {
19696                RecalcTopology::Workbook
19697            } else {
19698                RecalcTopology::RunLocalRecipe
19699            };
19700            Ok(RecalcPlan {
19701                key: engine.recalc_plan_key(),
19702                kind: RecalcPlanKind::Target {
19703                    targets: targets.to_vec(),
19704                    scope: preparation.widened_scope,
19705                    topology,
19706                    dynamic_policy: DynamicPlanPolicy::BoundedTargetReplan,
19707                },
19708            })
19709        })
19710    }
19711
19712    /// Evaluate using a previously constructed compatibility or target plan.
19713    pub fn evaluate_recalc_plan(&mut self, plan: &RecalcPlan) -> Result<EvalResult, ExcelError> {
19714        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19715            engine.evaluate_recalc_plan_unobserved(plan)
19716        })
19717    }
19718
19719    pub fn evaluate_recalc_plan_with_controls(
19720        &mut self,
19721        plan: &RecalcPlan,
19722        cancel: Option<crate::engine::CancelToken>,
19723        deadline: Option<Instant>,
19724    ) -> Result<EvalResult, ExcelError> {
19725        self.observe_evaluation_resource_request(EvaluationRequestKind::RecalcPlan, |engine| {
19726            engine.active_cancel_flag = cancel.clone();
19727            engine.active_evaluation_deadline = deadline;
19728            let result = engine.evaluate_recalc_plan_unobserved(plan);
19729            engine.active_cancel_flag = None;
19730            engine.active_evaluation_deadline = None;
19731            result
19732        })
19733    }
19734
19735    fn evaluate_recalc_plan_unobserved(
19736        &mut self,
19737        plan: &RecalcPlan,
19738    ) -> Result<EvalResult, ExcelError> {
19739        self.graph.flush_pending_edge_deltas();
19740        self.validate_recalc_plan_key(&plan.key)?;
19741        self.cancellation_checkpoint("Evaluation cancelled before recalculation plan execution")?;
19742        self.validate_deterministic_mode()?;
19743
19744        match &plan.kind {
19745            RecalcPlanKind::Target {
19746                targets,
19747                scope,
19748                topology,
19749                dynamic_policy,
19750            } => {
19751                debug_assert_eq!(*dynamic_policy, DynamicPlanPolicy::BoundedTargetReplan);
19752                debug_assert_eq!(
19753                    matches!(topology, RecalcTopology::Workbook),
19754                    matches!(scope, crate::engine::PrepareScope::Workbook)
19755                );
19756                let _source_cache = self.source_cache_session();
19757                self.execute_prepared_target_recipe(targets, scope, None)
19758            }
19759            RecalcPlanKind::CompatibilityFull {
19760                schedule,
19761                has_dynamic_refs,
19762            } => {
19763                let _source_cache = self.source_cache_session();
19764                let transitioned = self.transition_off_mode_spans_to_legacy()?;
19765                self.begin_evaluation_request();
19766                if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
19767                    && self.graph.formula_authority().active_span_count() > 0
19768                {
19769                    return self.evaluate_authoritative_formula_plane_all();
19770                }
19771                if *has_dynamic_refs {
19772                    self.virtual_dep_fallback_activations =
19773                        self.virtual_dep_fallback_activations.saturating_add(1);
19774                    if !transitioned {
19775                        return self.evaluate_all_coordinator();
19776                    }
19777                }
19778                if transitioned {
19779                    return self.evaluate_all_legacy_impl();
19780                }
19781
19782                let start = crate::instant::FzInstant::now();
19783                let dirty_vertices = self.graph.get_evaluation_vertices();
19784                if dirty_vertices.is_empty() {
19785                    return Ok(EvalResult {
19786                        computed_vertices: 0,
19787                        cycle_errors: 0,
19788                        elapsed: start.elapsed(),
19789                    });
19790                }
19791
19792                let dirty_set: FxHashSet<VertexId> = dirty_vertices.iter().copied().collect();
19793                let mut computed_vertices = 0;
19794                let mut cycle_errors = 0;
19795                for &unit in &schedule.units {
19796                    self.cancellation_checkpoint(
19797                        "Evaluation cancelled before recalculation plan schedule unit",
19798                    )?;
19799                    match unit {
19800                        ScheduleUnit::Cycle(i) => {
19801                            let stamped = self.handle_cycle_unit(
19802                                schedule.unit_cycle(i),
19803                                None,
19804                                Some(&dirty_set),
19805                                None,
19806                            )?;
19807                            if stamped > 0 {
19808                                cycle_errors += 1;
19809                            }
19810                        }
19811                        ScheduleUnit::Layer(i) => {
19812                            let work: Vec<VertexId> = schedule
19813                                .unit_layer(i)
19814                                .vertices
19815                                .iter()
19816                                .copied()
19817                                .filter(|v| dirty_set.contains(v))
19818                                .collect();
19819                            if work.is_empty() {
19820                                continue;
19821                            }
19822                            let temp_layer = crate::engine::scheduler::Layer { vertices: work };
19823                            if self.thread_pool.is_some() && temp_layer.vertices.len() > 1 {
19824                                computed_vertices += self.evaluate_layer_parallel(&temp_layer)?;
19825                            } else {
19826                                computed_vertices += self.evaluate_layer_sequential(&temp_layer)?;
19827                            }
19828                        }
19829                    }
19830                }
19831
19832                self.resource_checkpoint(0)?;
19833                self.graph.clear_dirty_flags(&dirty_vertices);
19834                self.redirty_for_next_recalc();
19835                Ok(EvalResult {
19836                    computed_vertices,
19837                    cycle_errors,
19838                    elapsed: start.elapsed(),
19839                })
19840            }
19841        }
19842    }
19843    fn evaluate_all_legacy_and_ack_dirty(
19844        &mut self,
19845        formula_dirty: FormulaDirtyLease,
19846    ) -> Result<EvalResult, ExcelError> {
19847        let result = self.evaluate_all_legacy_impl()?;
19848        self.checked_ack_formula_dirty_observed(formula_dirty)?;
19849        Ok(result)
19850    }
19851
19852    fn evaluate_formula_plane_capacity_fallback(
19853        &mut self,
19854        mut formula_dirty: FormulaDirtyLease,
19855        owned_dirty_events: &[usize],
19856        selected_span_refs: &[FormulaSpanRef],
19857        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19858        delta: Option<&mut DeltaCollector>,
19859    ) -> Result<EvalResult, ExcelError> {
19860        let leased_len = formula_dirty.len();
19861        if !selected_span_refs.is_empty() {
19862            let materialization_started = crate::instant::FzInstant::now();
19863            let prepared = self
19864                .prepare_formula_span_demotion(selected_span_refs)
19865                .map_err(|error| {
19866                    if let FormulaSpanDemotionError::Resource(error) = error {
19867                        error
19868                    } else {
19869                        ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
19870                            "FormulaPlane capacity fallback demotion preparation failed: {error}"
19871                        ))
19872                    }
19873                })?;
19874            let report = self
19875                .commit_prepared_formula_span_demotion(prepared)
19876                .map_err(|error| {
19877                    ExcelError::new(ExcelErrorKind::NImpl).with_message(format!(
19878                        "FormulaPlane capacity fallback demotion commit failed: {error}"
19879                    ))
19880                })?;
19881            self.observe_materialization(
19882                report.placements_materialized,
19883                false,
19884                materialization_started.elapsed(),
19885            );
19886        }
19887        #[cfg(test)]
19888        {
19889            self.last_formula_plane_span_eval_report = None;
19890        }
19891        if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone()) {
19892            formula_dirty = extended;
19893        }
19894        let mut acknowledged = owned_dirty_events.to_vec();
19895        acknowledged.extend(leased_len..formula_dirty.len());
19896        let result = if let Some(target_roots) = target_roots {
19897            let refreshed = self.resolve_target_producers_from_existing_roots(target_roots)?;
19898            self.evaluate_legacy_target_roots(&refreshed, delta)?
19899        } else if let Some(delta) = delta {
19900            self.evaluate_all_with_delta_collector(delta)?
19901        } else {
19902            self.evaluate_all_legacy_impl()?
19903        };
19904        self.formula_plane_capacity_bailouts =
19905            self.formula_plane_capacity_bailouts.saturating_add(1);
19906        if target_roots.is_some() {
19907            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &acknowledged)?;
19908        } else {
19909            self.checked_ack_formula_dirty_observed(formula_dirty)?;
19910        }
19911        Ok(result)
19912    }
19913
19914    fn evaluate_authoritative_formula_plane_all(&mut self) -> Result<EvalResult, ExcelError> {
19915        self.evaluate_authoritative_formula_plane(None, None)
19916    }
19917
19918    fn evaluate_authoritative_formula_plane_targets(
19919        &mut self,
19920        roots: &[crate::engine::target_preparation::TargetProducer],
19921        delta: Option<&mut DeltaCollector>,
19922    ) -> Result<EvalResult, ExcelError> {
19923        self.evaluate_authoritative_formula_plane(Some(roots), delta)
19924    }
19925
19926    fn evaluate_authoritative_formula_plane(
19927        &mut self,
19928        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
19929        mut delta: Option<&mut DeltaCollector>,
19930    ) -> Result<EvalResult, ExcelError> {
19931        // The public/request coordinators begin exactly once. Every primitive
19932        // below is deliberately non-beginning and non-finalising so the dirty
19933        // lease, iterative accumulator and §7.11 clock sample have one owner.
19934        let mut formula_dirty = self.graph.lease_formula_dirty();
19935        self.observe_dirty_lease_acquired(formula_dirty.is_empty());
19936
19937        // SingletonUnique formulas intentionally remain legacy graph vertices;
19938        // when no spans are active, execute through the private legacy primitive.
19939        if self.graph.formula_authority().active_span_count() == 0 {
19940            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoActiveSpans);
19941            #[cfg(test)]
19942            {
19943                self.last_formula_plane_span_eval_report = None;
19944            }
19945            if let Some(target_roots) = target_roots {
19946                let result =
19947                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
19948                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
19949                return Ok(result);
19950            }
19951            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
19952        }
19953
19954        // With no graph-owned span/region dirtiness, only sparse legacy work
19955        // (including volatiles) can remain. Preserve the warm no-op fast path.
19956        if formula_dirty.is_empty() {
19957            self.observe_topology_strategy(FormulaPlaneTopologyStrategy::SkippedNoDirtyWork);
19958            #[cfg(test)]
19959            {
19960                self.last_formula_plane_span_eval_report = None;
19961            }
19962            if let Some(target_roots) = target_roots {
19963                let result =
19964                    self.evaluate_legacy_target_roots(target_roots, delta.as_deref_mut())?;
19965                self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
19966                return Ok(result);
19967            }
19968            return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
19969        }
19970
19971        let start = crate::instant::FzInstant::now();
19972        let mut runtime_target_roots = target_roots.map(<[_]>::to_vec);
19973        let mut workbook_exact_attempted = target_roots.is_none();
19974        // #CIRC stamps produced by demoting cyclic spans and resolving the
19975        // residual legacy-only cycle ahead of the mixed schedule (gotcha G8).
19976        let mut prepass_cycle_errors = 0usize;
19977        const MAX_CYCLE_DEMOTE_ITERS: usize = 64;
19978        let mut cycle_demote_iters = 0usize;
19979        let mut include_dirty_regions = true;
19980        let mut retry_whole_spans = Vec::new();
19981        let mut computed_vertices = 0usize;
19982        let mut runtime_replan_iterations = 0usize;
19983        const MAX_RUNTIME_REPLAN: usize = 5;
19984        let mut array_result_demotions = 0usize;
19985        const MAX_ARRAY_RESULT_DEMOTIONS: usize = 64;
19986        let mut virtual_telemetry = self
19987            .config
19988            .enable_virtual_dep_telemetry
19989            .then(|| self.start_virtual_dep_telemetry());
19990        'virtual_replan: loop {
19991            self.cancellation_checkpoint("Evaluation cancelled before mixed topology")?;
19992            let (
19993                schedule,
19994                span_refs_by_id,
19995                plane_epoch,
19996                legacy_vertices,
19997                owned_dirty_events,
19998                island_plan,
19999            ) = loop {
20000                let (
20001                    schedule,
20002                    span_refs_by_id,
20003                    plane_epoch,
20004                    legacy_vertices,
20005                    owned_dirty_events,
20006                    island_plan,
20007                ) = if let Some(target_roots) = runtime_target_roots.as_deref() {
20008                    self.build_formula_plane_target_schedule(
20009                        &formula_dirty,
20010                        &retry_whole_spans,
20011                        include_dirty_regions,
20012                        target_roots,
20013                    )?
20014                } else {
20015                    self.build_formula_plane_mixed_schedule(
20016                        &formula_dirty,
20017                        &retry_whole_spans,
20018                        include_dirty_regions,
20019                    )?
20020                };
20021                #[cfg(test)]
20022                let mut schedule = schedule;
20023                #[cfg(test)]
20024                if std::mem::take(&mut self.force_non_cycle_schedule_fallback_for_test) {
20025                    let producer = schedule
20026                        .layers
20027                        .iter()
20028                        .flat_map(|layer| layer.work.iter())
20029                        .map(|work| work.producer)
20030                        .next()
20031                        .unwrap_or(FormulaProducerId::Legacy(VertexId(0)));
20032                    schedule.fallbacks.push(
20033                        crate::formula_plane::scheduler::MixedScheduleFallback {
20034                            producer,
20035                            reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
20036                        },
20037                    );
20038                }
20039
20040                if schedule.is_authoritative_safe() {
20041                    break (
20042                        schedule,
20043                        span_refs_by_id,
20044                        plane_epoch,
20045                        legacy_vertices,
20046                        owned_dirty_events,
20047                        island_plan,
20048                    );
20049                }
20050
20051                // Non-cycle unsafe schedules cannot safely run legacy-only while a
20052                // scheduled span remains virtual. Refine exactly every span in this
20053                // request's scheduled work, then complete the request in one legacy
20054                // pass. Clean, unscheduled spans keep their authority and overlays.
20055                let has_cycle_fallback = schedule.stats.cycle_count > 0
20056                    || schedule
20057                        .fallbacks
20058                        .iter()
20059                        .any(|fb| fb.reason == MixedScheduleFallbackReason::CycleDetected);
20060                if !has_cycle_fallback {
20061                    let selected_span_refs = schedule
20062                        .layers
20063                        .iter()
20064                        .flat_map(|layer| layer.work.iter())
20065                        .filter_map(|work| match work.producer {
20066                            FormulaProducerId::Span(span_id) => {
20067                                span_refs_by_id.get(&span_id).copied()
20068                            }
20069                            FormulaProducerId::Legacy(_) => None,
20070                        })
20071                        .collect::<Vec<_>>();
20072
20073                    return self.evaluate_formula_plane_capacity_fallback(
20074                        formula_dirty,
20075                        &owned_dirty_events,
20076                        &selected_span_refs,
20077                        runtime_target_roots.as_deref(),
20078                        delta.as_deref_mut(),
20079                    );
20080                }
20081
20082                // Gotcha G8 (refs #112): a span whose member cell participates in a
20083                // statically-cyclic SCC must never be span-evaluated. Cross-cell
20084                // cycles that route through a span producer are invisible to the
20085                // legacy Tarjan pass (the span member has no graph vertex) and only
20086                // surface here, as `CycleDetected` fallbacks in the producer-bounded
20087                // mixed schedule. Demote the cyclic spans to legacy graph vertices
20088                // so the cycle members move onto the legacy SCC path, then resolve
20089                // the now legacy-only cycle ahead of the schedule and rebuild.
20090                // Spans that do not touch the cycle are left untouched.
20091                let cyclic_spans = self.collect_cyclic_span_refs(&schedule, &span_refs_by_id);
20092                if !cyclic_spans.is_empty() {
20093                    self.demote_cyclic_spans(&cyclic_spans)?;
20094                }
20095
20096                if self.graph.formula_authority().active_span_count() == 0 {
20097                    // All spans demoted; nothing left for the FP coordinator. The
20098                    // legacy evaluator resolves the (now fully legacy) cycle.
20099                    if let Some(target_roots) = runtime_target_roots.as_deref() {
20100                        let refreshed =
20101                            self.resolve_target_producers_from_existing_roots(target_roots)?;
20102                        let result =
20103                            self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
20104                        self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
20105                        return Ok(result);
20106                    }
20107                    return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
20108                }
20109
20110                // Resolve the residual legacy-only cycle (`handle_cycle_unit`
20111                // honors Static vs Runtime) before rebuilding so the mixed schedule
20112                // is cycle-free and the surviving spans still get evaluated.
20113                prepass_cycle_errors =
20114                    prepass_cycle_errors.saturating_add(self.evaluate_legacy_cycle_prepass()?);
20115
20116                // Re-seed every surviving span explicitly through the graph-owned
20117                // dirty authority, then renew the lease so successful completion
20118                // acknowledges both the original prefix and retry seeds.
20119                retry_whole_spans = if runtime_target_roots.is_some() {
20120                    let active = self
20121                        .graph
20122                        .formula_authority()
20123                        .active_span_refs()
20124                        .into_iter()
20125                        .map(|span_ref| (span_ref.id, span_ref))
20126                        .collect::<BTreeMap<_, _>>();
20127                    let mut refs = schedule
20128                        .layers
20129                        .iter()
20130                        .flat_map(|layer| layer.work.iter())
20131                        .filter_map(|work| match work.producer {
20132                            FormulaProducerId::Span(id) => active.get(&id).copied(),
20133                            FormulaProducerId::Legacy(_) => None,
20134                        })
20135                        .collect::<Vec<_>>();
20136                    refs.extend(schedule.fallbacks.iter().filter_map(|fallback| {
20137                        match fallback.producer {
20138                            FormulaProducerId::Span(id) => active.get(&id).copied(),
20139                            FormulaProducerId::Legacy(_) => None,
20140                        }
20141                    }));
20142                    refs.sort_by_key(|span_ref| span_ref.id);
20143                    refs.dedup();
20144                    refs
20145                } else {
20146                    self.graph.formula_authority().active_span_refs()
20147                };
20148                self.graph.mark_formula_spans_dirty(
20149                    retry_whole_spans.iter().copied(),
20150                    WholeSpanDirtyReason::CycleRetry,
20151                );
20152                // Only the first renewal can enlarge this generation's owned
20153                // prefix. If another cycle iteration is required, its redundant
20154                // retry seeds stay pending rather than risking acknowledgement of
20155                // work recorded after the first renewal.
20156                if let Some(extended) = self.graph.extend_formula_dirty_lease(formula_dirty.clone())
20157                {
20158                    formula_dirty = extended;
20159                    #[cfg(test)]
20160                    if std::mem::take(
20161                        &mut self.rerecord_cycle_retry_span_after_lease_extension_for_test,
20162                    ) {
20163                        self.graph.mark_formula_spans_dirty(
20164                            retry_whole_spans.iter().copied().take(1),
20165                            WholeSpanDirtyReason::GlobalInvalidation,
20166                        );
20167                    }
20168                }
20169                include_dirty_regions = false;
20170
20171                cycle_demote_iters += 1;
20172                if cycle_demote_iters >= MAX_CYCLE_DEMOTE_ITERS {
20173                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
20174                        "FormulaPlane cyclic demotion did not converge".to_string(),
20175                    ));
20176                }
20177            };
20178            self.observe_formula_plane_route(
20179                &island_plan,
20180                FormulaPlaneRoutePhase::Planned,
20181                FormulaPlaneRouteTransitionReason::ProvenIsolated,
20182                cycle_demote_iters,
20183                runtime_replan_iterations,
20184            );
20185            if !island_plan.dirty_vertices.is_empty() {
20186                self.cancellation_checkpoint("Evaluation cancelled before legacy island")?;
20187                let (island_computed, island_cycles) = self.evaluate_legacy_island_non_finalizing(
20188                    &island_plan.dirty_vertices,
20189                    delta.as_deref_mut(),
20190                )?;
20191                computed_vertices = computed_vertices.saturating_add(island_computed);
20192                prepass_cycle_errors = prepass_cycle_errors.saturating_add(island_cycles);
20193                self.observe_formula_plane_route(
20194                    &island_plan,
20195                    FormulaPlaneRoutePhase::Executed,
20196                    FormulaPlaneRouteTransitionReason::ProvenIsolated,
20197                    cycle_demote_iters,
20198                    runtime_replan_iterations,
20199                );
20200                self.cancellation_checkpoint("Evaluation cancelled after legacy island")?;
20201            }
20202            let old_virtual_dependencies = VirtualDepBuilder::new(self).build(&legacy_vertices).0;
20203            let old_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
20204
20205            #[cfg(test)]
20206            {
20207                self.last_formula_plane_span_eval_report = None;
20208            }
20209            // #388: set when a span placement evaluated to an array. The layer
20210            // loop breaks out immediately, leaving that layer's write buffer
20211            // unflushed, so nothing from the offending span is published.
20212            let mut array_result_span: Option<FormulaSpanRef> = None;
20213            'layers: for layer in schedule.layers {
20214                self.cancellation_checkpoint("Evaluation cancelled before mixed layer")?;
20215                let mut buffer = ComputedWriteBuffer::default();
20216                let mut sink = SpanComputedWriteSink::new(&mut buffer);
20217                let work_items = layer.work;
20218                let mut work_index = 0usize;
20219                while work_index < work_items.len() {
20220                    match work_items[work_index].producer {
20221                        FormulaProducerId::Span(span_id) => {
20222                            let span_ref = *span_refs_by_id.get(&span_id).ok_or_else(|| {
20223                                ExcelError::new(ExcelErrorKind::NImpl)
20224                                    .with_message("FormulaPlane schedule referenced a stale span")
20225                            })?;
20226                            let sheet_id = {
20227                                let authority = self.graph.formula_authority();
20228                                let span =
20229                                    authority.plane.spans.get(span_ref).ok_or_else(|| {
20230                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
20231                                            "FormulaPlane schedule referenced a stale span",
20232                                        )
20233                                    })?;
20234                                span.sheet_id
20235                            };
20236                            let current_sheet = self.graph.sheet_name(sheet_id);
20237                            let authority = self.graph.formula_authority();
20238                            let evaluator = SpanEvaluator::new_with_cancel(
20239                                &authority.plane,
20240                                self,
20241                                current_sheet,
20242                                self.graph.data_store(),
20243                                self.graph.sheet_reg(),
20244                                self.active_cancel_flag.as_ref(),
20245                            );
20246                            #[cfg(test)]
20247                            let mut last_group_report = None;
20248                            let mut selected_group_work = 0_u64;
20249                            while work_index < work_items.len() {
20250                                let FormulaProducerId::Span(group_span_id) =
20251                                    work_items[work_index].producer
20252                                else {
20253                                    break;
20254                                };
20255                                let group_span_ref =
20256                                    *span_refs_by_id.get(&group_span_id).ok_or_else(|| {
20257                                        ExcelError::new(ExcelErrorKind::NImpl).with_message(
20258                                            "FormulaPlane schedule referenced a stale span",
20259                                        )
20260                                    })?;
20261                                let group_sheet_id = {
20262                                    let authority = self.graph.formula_authority();
20263                                    let span =
20264                                        authority.plane.spans.get(group_span_ref).ok_or_else(
20265                                            || {
20266                                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
20267                                                    "FormulaPlane schedule referenced a stale span",
20268                                                )
20269                                            },
20270                                        )?;
20271                                    span.sheet_id
20272                                };
20273                                if group_sheet_id != sheet_id {
20274                                    break;
20275                                }
20276
20277                                let dirty = producer_dirty_to_span_dirty(
20278                                    work_items[work_index].dirty.clone(),
20279                                    group_span_ref,
20280                                );
20281                                let task = SpanEvalTask {
20282                                    span: group_span_ref,
20283                                    dirty,
20284                                    plane_epoch,
20285                                };
20286                                self.cancellation_checkpoint(
20287                                    "Evaluation cancelled during FormulaPlane span",
20288                                )?;
20289                                let report = match evaluator.evaluate_task(&task, &mut sink) {
20290                                    Ok(report) => report,
20291                                    // #388: fail closed on array results. Drop
20292                                    // this layer's buffer unflushed, demote the
20293                                    // span after the borrow of the authority
20294                                    // ends, and replan onto the legacy path.
20295                                    Err(
20296                                        crate::formula_plane::span_eval::SpanEvalError::ArrayResultRequiresSpill,
20297                                    ) => {
20298                                        array_result_span = Some(group_span_ref);
20299                                        break 'layers;
20300                                    }
20301                                    Err(
20302                                        crate::formula_plane::span_eval::SpanEvalError::Cancelled,
20303                                    ) => {
20304                                        return Err(ExcelError::new(ExcelErrorKind::Cancelled)
20305                                            .with_message(
20306                                                "Evaluation cancelled during FormulaPlane span",
20307                                            ));
20308                                    }
20309                                    Err(err) => {
20310                                        return Err(ExcelError::new(ExcelErrorKind::NImpl)
20311                                            .with_message(format!(
20312                                                "FormulaPlane span evaluation failed: {err:?}"
20313                                            )));
20314                                    }
20315                                };
20316                                #[cfg(test)]
20317                                {
20318                                    last_group_report = Some(report.clone());
20319                                }
20320                                selected_group_work = selected_group_work
20321                                    .saturating_add(report.span_eval_placement_count);
20322                                computed_vertices = computed_vertices
20323                                    .saturating_add(report.span_eval_placement_count as usize);
20324                                work_index = work_index.saturating_add(1);
20325                            }
20326                            // Charge the exact selected placement regions before
20327                            // their transaction-local value buffer is published.
20328                            self.charge_bounded_work(selected_group_work)?;
20329                            #[cfg(test)]
20330                            {
20331                                if let Some(report) = last_group_report {
20332                                    self.last_formula_plane_span_eval_report = Some(report);
20333                                }
20334                            }
20335                        }
20336                        FormulaProducerId::Legacy(_) => {
20337                            // Batch the contiguous run of legacy work items into a
20338                            // synthetic layer and evaluate it through the same
20339                            // coalesced effects pipeline as the legacy scheduler.
20340                            // Items in one mixed layer have no edges between them
20341                            // (same invariant the legacy Kahn layers rely on), so
20342                            // batching preserves ordering semantics while
20343                            // amortizing per-write overlay mirroring that makes
20344                            // one-vertex-at-a-time evaluation ~30x slower.
20345                            let mut vertices = Vec::new();
20346                            while work_index < work_items.len() {
20347                                let FormulaProducerId::Legacy(vertex_id) =
20348                                    work_items[work_index].producer
20349                                else {
20350                                    break;
20351                                };
20352                                vertices.push(vertex_id);
20353                                work_index = work_index.saturating_add(1);
20354                            }
20355                            let legacy_layer = crate::engine::scheduler::Layer { vertices };
20356                            let evaluated = if let Some(delta) = delta.as_deref_mut() {
20357                                if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1 {
20358                                    self.evaluate_layer_parallel_with_delta(&legacy_layer, delta)?
20359                                } else {
20360                                    self.evaluate_layer_sequential_with_delta(&legacy_layer, delta)?
20361                                }
20362                            } else if self.thread_pool.is_some() && legacy_layer.vertices.len() > 1
20363                            {
20364                                self.evaluate_layer_parallel(&legacy_layer)?
20365                            } else {
20366                                self.evaluate_layer_sequential(&legacy_layer)?
20367                            };
20368                            computed_vertices = computed_vertices.saturating_add(evaluated);
20369                        }
20370                    }
20371                }
20372                self.resource_checkpoint(0)?;
20373                self.cancellation_checkpoint("Evaluation cancelled before mixed layer flush")?;
20374                if let Some(delta) = delta.as_deref_mut() {
20375                    self.record_computed_write_buffer_delta(&buffer, delta);
20376                }
20377                self.flush_authoritative_computed_write_buffer(&mut buffer)?;
20378            }
20379
20380            if let Some(span_ref) = array_result_span {
20381                // #388: nothing from the aborted layer was published. Demote the
20382                // span so its placements become legacy vertices (dirty on
20383                // ingest) and rebuild the mixed schedule; the legacy evaluator
20384                // then applies the spill planner to the array result. Each
20385                // iteration removes one span from the plane, so this terminates.
20386                array_result_demotions = array_result_demotions.saturating_add(1);
20387                if array_result_demotions >= MAX_ARRAY_RESULT_DEMOTIONS {
20388                    return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
20389                        "FormulaPlane array-result span demotion did not converge".to_string(),
20390                    ));
20391                }
20392                self.demote_array_result_span(span_ref)?;
20393                if self.graph.formula_authority().active_span_count() == 0 {
20394                    if let Some(target_roots) = runtime_target_roots.as_deref() {
20395                        let refreshed =
20396                            self.resolve_target_producers_from_existing_roots(target_roots)?;
20397                        let result =
20398                            self.evaluate_legacy_target_roots(&refreshed, delta.as_deref_mut())?;
20399                        self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &[])?;
20400                        return Ok(result);
20401                    }
20402                    return self.evaluate_all_legacy_and_ack_dirty(formula_dirty);
20403                }
20404                include_dirty_regions = true;
20405                continue 'virtual_replan;
20406            }
20407
20408            let mut changed_virtual =
20409                self.changed_virtual_dep_vertices(&legacy_vertices, &old_virtual_dependencies);
20410            let new_dynamic_regions = self.dynamic_virtual_regions(&legacy_vertices);
20411            let mut dynamic_candidates = old_dynamic_regions
20412                .keys()
20413                .chain(new_dynamic_regions.keys())
20414                .copied()
20415                .collect::<FxHashSet<_>>();
20416            for vertex in dynamic_candidates.drain() {
20417                if old_dynamic_regions.get(&vertex) != new_dynamic_regions.get(&vertex)
20418                    && !changed_virtual.contains(&vertex)
20419                {
20420                    changed_virtual.push(vertex);
20421                }
20422            }
20423            if let Some(telemetry) = virtual_telemetry.as_mut() {
20424                telemetry.changed_vdeps_total = telemetry
20425                    .changed_vdeps_total
20426                    .saturating_add(changed_virtual.len());
20427            }
20428            self.resource_checkpoint(0)?;
20429            self.graph.clear_dirty_flags(&legacy_vertices);
20430            if !changed_virtual.is_empty() {
20431                for vertex in &changed_virtual {
20432                    self.graph.set_dirty(*vertex, true);
20433                }
20434                let can_replan = runtime_replan_iterations < MAX_RUNTIME_REPLAN;
20435                let can_attempt_workbook =
20436                    !can_replan && runtime_target_roots.is_some() && !workbook_exact_attempted;
20437                if can_replan || can_attempt_workbook {
20438                    if can_replan {
20439                        if let Some(roots) = runtime_target_roots.as_mut() {
20440                            self.extend_target_roots_with_dynamic_regions(
20441                                roots,
20442                                old_dynamic_regions
20443                                    .values()
20444                                    .flatten()
20445                                    .chain(new_dynamic_regions.values().flatten()),
20446                            )?;
20447                        }
20448                        runtime_replan_iterations = runtime_replan_iterations.saturating_add(1);
20449                    } else {
20450                        runtime_target_roots = None;
20451                        workbook_exact_attempted = true;
20452                    }
20453                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
20454                        if can_replan {
20455                            stats.runtime_replan_rounds =
20456                                stats.runtime_replan_rounds.saturating_add(1);
20457                        }
20458                        stats.runtime_widening_rounds =
20459                            stats.runtime_widening_rounds.saturating_add(1);
20460                        if can_attempt_workbook {
20461                            stats.workbook_exact_attempts = 1;
20462                        }
20463                    }
20464                    self.cached_mixed_topology = None;
20465                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
20466                        ledger
20467                            .account_mixed_cache(0)
20468                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
20469                    }
20470                    include_dirty_regions = true;
20471                    continue 'virtual_replan;
20472                }
20473                if let Some(telemetry) = virtual_telemetry.as_mut() {
20474                    telemetry.replan_iterations = runtime_replan_iterations;
20475                    telemetry.bailout_reason = Some("max_replan");
20476                    self.last_virtual_dep_telemetry = telemetry.clone();
20477                }
20478                return Err(self.replan_exhausted_error(
20479                    MAX_RUNTIME_REPLAN,
20480                    "mixed dynamic dependency evaluation",
20481                ));
20482            }
20483            if let Some(mut telemetry) = virtual_telemetry {
20484                telemetry.replan_iterations = runtime_replan_iterations;
20485                telemetry.bailout_reason = Some(if changed_virtual.is_empty() {
20486                    "converged"
20487                } else {
20488                    "max_replan"
20489                });
20490                self.last_virtual_dep_telemetry = telemetry;
20491            }
20492            // Drop dirty flags on any newly-scheduled FP runtime cells whose graph
20493            // vertices weren't in the dirty subset (e.g. recently-introduced span
20494            // result cells); legacy clear_dirty_flags is safe over the full set.
20495            self.redirty_for_next_recalc();
20496            self.checked_ack_formula_dirty_sublease_observed(formula_dirty, &owned_dirty_events)?;
20497            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
20498            return Ok(EvalResult {
20499                computed_vertices,
20500                cycle_errors: prepass_cycle_errors,
20501                elapsed: start.elapsed(),
20502            });
20503        }
20504    }
20505
20506    fn effective_mixed_cache_limits(&self) -> (usize, usize, usize) {
20507        let budgets = &self.evaluation_resource_budgets;
20508        let candidate_limit = budgets
20509            .optimization
20510            .mixed_cache_candidates
20511            .map_or(self.config.max_formula_plane_cache_candidates, |limit| {
20512                limit.min(self.config.max_formula_plane_cache_candidates)
20513            });
20514        let edge_limit = budgets
20515            .optimization
20516            .mixed_cache_edges
20517            .map_or(self.config.max_formula_plane_cache_edges, |limit| {
20518                limit.min(self.config.max_formula_plane_cache_edges)
20519            });
20520        let mut byte_limit =
20521            u64::try_from(self.config.max_formula_plane_cache_bytes).unwrap_or(u64::MAX);
20522        if let Some(limit) = budgets.retained.total_bytes {
20523            byte_limit = byte_limit.min(limit);
20524        }
20525        if let Some(limit) = budgets.retained.mixed_cache_bytes {
20526            byte_limit = byte_limit.min(limit);
20527        }
20528        (
20529            candidate_limit,
20530            edge_limit,
20531            usize::try_from(byte_limit).unwrap_or(usize::MAX),
20532        )
20533    }
20534
20535    fn mixed_topology_cache_key(&self) -> MixedTopologyCacheKey {
20536        let (max_candidates, max_edges, max_memory_bytes) = self.effective_mixed_cache_limits();
20537        MixedTopologyCacheKey {
20538            engine_topology_epoch: self.topology_epoch,
20539            graph_topology_revision: self.graph.topology_revision(),
20540            authority_indexes_epoch: self.graph.formula_authority().indexes_epoch(),
20541            legacy_island_revision: self.graph.topology_revision(),
20542            boundary_index_revision: self.graph.formula_authority().indexes_epoch(),
20543            function_semantic_epoch: self.function_semantic_epoch_seen,
20544            function_provider_revision: self.function_provider_revision_seen,
20545            max_candidates,
20546            max_edges,
20547            max_memory_bytes,
20548        }
20549    }
20550
20551    fn mixed_topology_index_preflight_bytes(&self) -> u64 {
20552        use crate::formula_plane::region_index::AxisRange;
20553
20554        const FIXED_BYTES: u64 = 8 * 1024;
20555        const ENTRY_BYTES: u64 = 512;
20556        const INDEX_UNIT_BYTES: u64 = 1024;
20557        const BINDING_BYTES: u64 = 128;
20558        let index_units = |region: Region| -> u64 {
20559            match region.normalized().axis_ranges() {
20560                (AxisRange::Point(_), AxisRange::Point(_)) => 2,
20561                (AxisRange::Span(row_start, row_end), AxisRange::Span(col_start, col_end)) => {
20562                    let row_buckets = u64::from(row_end / 64 - row_start / 64).saturating_add(1);
20563                    let col_buckets = u64::from(col_end / 16 - col_start / 16).saturating_add(1);
20564                    row_buckets.saturating_mul(col_buckets).saturating_mul(2)
20565                }
20566                _ => 1,
20567            }
20568        };
20569
20570        let authority = self.graph.formula_authority();
20571        let span_refs = authority.active_span_refs();
20572        let legacy_vertices = self.graph.formula_vertices();
20573        let mut entry_count = 0_u64;
20574        let mut units = 0_u64;
20575        for span_ref in &span_refs {
20576            if let Some(span) = authority.plane.spans.get(*span_ref) {
20577                let result_region = Region::from_domain(span.result_region.domain());
20578                entry_count = entry_count.saturating_add(1);
20579                units = units.saturating_add(index_units(result_region));
20580                if let Some(summary_id) = span.read_summary_id
20581                    && let Some(summary) = authority.plane.span_read_summaries.get(summary_id)
20582                {
20583                    for dependency in &summary.dependencies {
20584                        entry_count = entry_count.saturating_add(1);
20585                        units = units.saturating_add(index_units(dependency.read_region));
20586                    }
20587                }
20588            }
20589        }
20590        for &vertex in &legacy_vertices {
20591            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex) {
20592                let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
20593                entry_count = entry_count.saturating_add(1);
20594                units = units.saturating_add(index_units(point));
20595            }
20596            for dependency in self.graph.get_dependencies(vertex) {
20597                if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
20598                    let point = Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
20599                    entry_count = entry_count.saturating_add(1);
20600                    units = units.saturating_add(index_units(point));
20601                }
20602            }
20603            let vertex_sheet = self.graph.get_vertex_sheet_id(vertex);
20604            if let Some(ranges) = self.graph.get_range_dependencies(vertex) {
20605                for range in ranges {
20606                    if let Ok(Some(region)) =
20607                        self.shared_range_to_region_pattern(range, vertex_sheet)
20608                    {
20609                        entry_count = entry_count.saturating_add(1);
20610                        units = units.saturating_add(index_units(region));
20611                    }
20612                }
20613            }
20614        }
20615        FIXED_BYTES
20616            .saturating_add(entry_count.saturating_mul(ENTRY_BYTES))
20617            .saturating_add(units.saturating_mul(INDEX_UNIT_BYTES))
20618            .saturating_add((span_refs.len() as u64).saturating_mul(BINDING_BYTES))
20619    }
20620
20621    fn prove_sheet_disjoint_legacy_island(
20622        &self,
20623        span_results: &FormulaProducerResultIndex,
20624        span_reads: &FormulaConsumerReadIndex,
20625        legacy_vertices: &[VertexId],
20626    ) -> Option<LegacyIslandPlan> {
20627        if !self.legacy_island_structural_summaries_trusted
20628            || self.graph.named_ranges_iter().next().is_some()
20629            || self.graph.sheet_named_ranges_iter().next().is_some()
20630            || self.graph.spill_registry_counts() != (0, 0)
20631            || legacy_vertices.iter().copied().any(|vertex| {
20632                self.graph.is_dynamic(vertex)
20633                    || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
20634            })
20635        {
20636            return None;
20637        }
20638        let span_result_sheets = span_results
20639            .entries()
20640            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
20641            .map(|entry| entry.result_region.sheet_id())
20642            .collect::<FxHashSet<_>>();
20643        let span_read_sheets = span_reads
20644            .entries()
20645            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Span(_)))
20646            .map(|entry| entry.read_region.sheet_id())
20647            .collect::<FxHashSet<_>>();
20648
20649        let span_boundary_sheets = span_result_sheets
20650            .union(&span_read_sheets)
20651            .copied()
20652            .collect::<FxHashSet<_>>();
20653        for vertex in legacy_vertices {
20654            let cell = self.graph.get_cell_ref_for_vertex(*vertex)?;
20655            if span_boundary_sheets.contains(&cell.sheet_id) {
20656                return None;
20657            }
20658            for dependency in self.graph.get_dependencies(*vertex) {
20659                let dependency_cell = self.graph.get_cell_ref_for_vertex(dependency)?;
20660                if span_boundary_sheets.contains(&dependency_cell.sheet_id) {
20661                    return None;
20662                }
20663            }
20664            if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
20665                for range in ranges {
20666                    let region = self
20667                        .shared_range_to_region_pattern(range, cell.sheet_id)
20668                        .ok()??;
20669                    if span_boundary_sheets.contains(&region.sheet_id()) {
20670                        return None;
20671                    }
20672                }
20673            }
20674        }
20675
20676        let mut membership = legacy_vertices.to_vec();
20677        membership.sort_unstable();
20678        if membership.is_empty() {
20679            return None;
20680        }
20681        let mut sheet_ids = membership
20682            .iter()
20683            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
20684            .map(|cell| cell.sheet_id)
20685            .collect::<Vec<_>>();
20686        sheet_ids.sort_unstable();
20687        sheet_ids.dedup();
20688        let island_id = membership
20689            .iter()
20690            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
20691                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
20692            });
20693        let retained_bytes = membership
20694            .capacity()
20695            .saturating_mul(std::mem::size_of::<VertexId>())
20696            .saturating_add(
20697                sheet_ids
20698                    .capacity()
20699                    .saturating_mul(std::mem::size_of::<SheetId>()),
20700            );
20701        Some(LegacyIslandPlan {
20702            membership,
20703            dirty_vertices: Vec::new(),
20704            sheet_ids,
20705            island_id,
20706            retained_bytes,
20707            boundary_relationships: 0,
20708            omitted_relationships: legacy_vertices.len(),
20709            omitted_direct_relationships: 0,
20710        })
20711    }
20712
20713    fn contract_legacy_islands(
20714        &self,
20715        full_results: &FormulaProducerResultIndex,
20716        full_reads: &FormulaConsumerReadIndex,
20717        legacy_reads_complete: bool,
20718        candidate_cap: usize,
20719    ) -> Option<(
20720        FormulaProducerResultIndex,
20721        FormulaConsumerReadIndex,
20722        LegacyIslandPlan,
20723    )> {
20724        use crate::formula_plane::producer::ProjectionResult;
20725        use crate::formula_plane::region_index::BoundedRegionQueryResult;
20726
20727        if !self.legacy_island_structural_summaries_trusted
20728            || !legacy_reads_complete
20729            || self.graph.named_ranges_iter().next().is_some()
20730            || self.graph.sheet_named_ranges_iter().next().is_some()
20731            || self.graph.spill_registry_counts() != (0, 0)
20732        {
20733            return None;
20734        }
20735        let legacy_vertices = self.graph.formula_vertices();
20736        if legacy_vertices.iter().copied().any(|vertex| {
20737            self.graph.is_dynamic(vertex)
20738                || self.graph.get_vertex_kind(vertex) == VertexKind::FormulaArray
20739        }) {
20740            return None;
20741        }
20742
20743        let mut reads_by_consumer: FxHashMap<VertexId, Vec<Region>> = FxHashMap::default();
20744        for entry in full_reads.entries() {
20745            if let FormulaProducerId::Legacy(vertex) = entry.consumer {
20746                reads_by_consumer
20747                    .entry(vertex)
20748                    .or_default()
20749                    .push(entry.read_region);
20750            }
20751        }
20752
20753        let mut connected = FxHashSet::default();
20754        let mut queue = VecDeque::new();
20755        let mut observed_candidates = 0usize;
20756        let mut boundary_relationships = 0usize;
20757        let add_connected = |vertex: VertexId,
20758                             connected: &mut FxHashSet<VertexId>,
20759                             queue: &mut VecDeque<VertexId>| {
20760            if connected.insert(vertex) {
20761                queue.push_back(vertex);
20762            }
20763        };
20764
20765        // Span results -> legacy consumers.
20766        for result in full_results.entries() {
20767            if !matches!(result.producer, FormulaProducerId::Span(_)) {
20768                continue;
20769            }
20770            let remaining = candidate_cap.saturating_sub(observed_candidates);
20771            let query = full_reads.query_changed_region_bounded(result.result_region, remaining);
20772            let BoundedRegionQueryResult::Complete(query) = query else {
20773                return None;
20774            };
20775            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
20776            for matched in query.matches {
20777                if let FormulaProducerId::Legacy(vertex) = matched.value.consumer {
20778                    match matched.value.dirty {
20779                        ProjectionResult::NoIntersection => {}
20780                        ProjectionResult::Unsupported(_) => return None,
20781                        ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
20782                            boundary_relationships = boundary_relationships.saturating_add(1);
20783                            add_connected(vertex, &mut connected, &mut queue);
20784                        }
20785                    }
20786                }
20787            }
20788        }
20789
20790        // Span reads -> legacy producers.
20791        for read in full_reads.entries() {
20792            if !matches!(read.consumer, FormulaProducerId::Span(_)) {
20793                continue;
20794            }
20795            let remaining = candidate_cap.saturating_sub(observed_candidates);
20796            let query = full_results.query_bounded(read.read_region, remaining);
20797            let BoundedRegionQueryResult::Complete(query) = query else {
20798                return None;
20799            };
20800            observed_candidates = observed_candidates.saturating_add(query.stats.candidate_count);
20801            for matched in query.matches {
20802                if let FormulaProducerId::Legacy(vertex) = matched.value.producer {
20803                    boundary_relationships = boundary_relationships.saturating_add(1);
20804                    add_connected(vertex, &mut connected, &mut queue);
20805                }
20806            }
20807        }
20808
20809        // Undirected closure through legacy relationships. Every vertex in this
20810        // closure remains in the mixed scheduler, preserving the only detector
20811        // for cycles whose path crosses a virtual span member.
20812        while let Some(vertex) = queue.pop_front() {
20813            let producer = FormulaProducerId::Legacy(vertex);
20814            if let Some(result_region) = full_results.producer_result_region(producer) {
20815                let remaining = candidate_cap.saturating_sub(observed_candidates);
20816                let query = full_reads.query_changed_region_bounded(result_region, remaining);
20817                let BoundedRegionQueryResult::Complete(query) = query else {
20818                    return None;
20819                };
20820                observed_candidates =
20821                    observed_candidates.saturating_add(query.stats.candidate_count);
20822                for matched in query.matches {
20823                    if let FormulaProducerId::Legacy(consumer) = matched.value.consumer {
20824                        match matched.value.dirty {
20825                            ProjectionResult::NoIntersection => {}
20826                            ProjectionResult::Unsupported(_) => return None,
20827                            ProjectionResult::Exact(_) | ProjectionResult::Conservative { .. } => {
20828                                add_connected(consumer, &mut connected, &mut queue);
20829                            }
20830                        }
20831                    }
20832                }
20833            }
20834            for read_region in reads_by_consumer.get(&vertex).into_iter().flatten() {
20835                let remaining = candidate_cap.saturating_sub(observed_candidates);
20836                let query = full_results.query_bounded(*read_region, remaining);
20837                let BoundedRegionQueryResult::Complete(query) = query else {
20838                    return None;
20839                };
20840                observed_candidates =
20841                    observed_candidates.saturating_add(query.stats.candidate_count);
20842                for matched in query.matches {
20843                    if let FormulaProducerId::Legacy(precedent) = matched.value.producer {
20844                        add_connected(precedent, &mut connected, &mut queue);
20845                    }
20846                }
20847            }
20848        }
20849
20850        let mut membership = legacy_vertices
20851            .iter()
20852            .copied()
20853            .filter(|vertex| !connected.contains(vertex))
20854            .collect::<Vec<_>>();
20855        membership.sort_unstable();
20856        if membership.is_empty() {
20857            return None;
20858        }
20859
20860        let mut producer_results = FormulaProducerResultIndex::default();
20861        let mut consumer_reads = FormulaConsumerReadIndex::default();
20862        let retained_span_results = full_results
20863            .entries()
20864            .filter(|entry| matches!(entry.producer, FormulaProducerId::Span(_)))
20865            .count();
20866        producer_results
20867            .try_reserve(retained_span_results.saturating_add(connected.len()))
20868            .ok()?;
20869        for entry in full_results.entries() {
20870            let retain = match entry.producer {
20871                FormulaProducerId::Span(_) => true,
20872                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
20873            };
20874            if retain {
20875                producer_results.insert_producer(entry.producer, entry.result_region);
20876            }
20877        }
20878        for entry in full_reads.entries() {
20879            let retain = match entry.consumer {
20880                FormulaProducerId::Span(_) => true,
20881                FormulaProducerId::Legacy(vertex) => connected.contains(&vertex),
20882            };
20883            if retain {
20884                consumer_reads.try_reserve(1).ok()?;
20885                consumer_reads.insert_read(
20886                    entry.consumer,
20887                    entry.read_region,
20888                    entry.consumer_result_region,
20889                    entry.projection,
20890                );
20891            }
20892        }
20893
20894        let mut sheet_ids = membership
20895            .iter()
20896            .filter_map(|vertex| self.graph.get_cell_ref_for_vertex(*vertex))
20897            .map(|cell| cell.sheet_id)
20898            .collect::<Vec<_>>();
20899        sheet_ids.sort_unstable();
20900        sheet_ids.dedup();
20901        let island_id = membership
20902            .iter()
20903            .fold(0xcbf29ce484222325_u64, |hash, vertex| {
20904                (hash ^ u64::from(vertex.0)).wrapping_mul(0x100000001b3)
20905            });
20906        let retained_bytes = membership
20907            .capacity()
20908            .saturating_mul(std::mem::size_of::<VertexId>())
20909            .saturating_add(
20910                sheet_ids
20911                    .capacity()
20912                    .saturating_mul(std::mem::size_of::<SheetId>()),
20913            );
20914        let omitted_relationships = full_reads
20915            .entries()
20916            .filter(|entry| matches!(entry.consumer, FormulaProducerId::Legacy(vertex) if !connected.contains(&vertex)))
20917            .count();
20918        let membership_set = membership.iter().copied().collect::<FxHashSet<_>>();
20919        let omitted_direct_relationships = membership
20920            .iter()
20921            .map(|vertex| {
20922                self.graph
20923                    .get_dependencies(*vertex)
20924                    .into_iter()
20925                    .filter(|dependency| membership_set.contains(dependency))
20926                    .count()
20927            })
20928            .sum();
20929        Some((
20930            producer_results,
20931            consumer_reads,
20932            LegacyIslandPlan {
20933                membership,
20934                dirty_vertices: Vec::new(),
20935                sheet_ids,
20936                island_id,
20937                retained_bytes,
20938                boundary_relationships,
20939                omitted_relationships,
20940                omitted_direct_relationships,
20941            },
20942        ))
20943    }
20944
20945    fn compile_formula_plane_mixed_topology(
20946        &mut self,
20947        key: MixedTopologyCacheKey,
20948    ) -> Result<FormulaPlaneTopologyCompileResult, ExcelError> {
20949        #[cfg(test)]
20950        {
20951            self.mixed_topology_index_builds_for_test =
20952                self.mixed_topology_index_builds_for_test.saturating_add(1);
20953        }
20954        let authority = self.graph.formula_authority();
20955        let mut producer_results = FormulaProducerResultIndex::default();
20956        let mut consumer_reads = FormulaConsumerReadIndex::default();
20957        let mut legacy_reads_complete = true;
20958        let span_refs = authority.active_span_refs();
20959        let legacy_vertices = self.graph.formula_vertices();
20960        let _producer_count = span_refs
20961            .len()
20962            .checked_add(legacy_vertices.len())
20963            .ok_or_else(|| {
20964                ExcelError::new(ExcelErrorKind::NImpl)
20965                    .with_message("FormulaPlane cache producer count overflow")
20966            })?;
20967        producer_results.try_reserve(_producer_count).map_err(|_| {
20968            ExcelError::new(ExcelErrorKind::NImpl)
20969                .with_message("FormulaPlane cache producer index reservation failed")
20970        })?;
20971        let span_refs_by_id = span_refs
20972            .iter()
20973            .copied()
20974            .map(|span_ref| (span_ref.id, span_ref))
20975            .collect::<BTreeMap<_, _>>();
20976        for span_ref in &span_refs {
20977            let span = authority.plane.spans.get(*span_ref).ok_or_else(|| {
20978                ExcelError::new(ExcelErrorKind::NImpl)
20979                    .with_message("FormulaPlane active span ref is stale")
20980            })?;
20981            let result_region = Region::from_domain(span.result_region.domain());
20982            producer_results.insert_producer(FormulaProducerId::Span(span.id), result_region);
20983            let Some(read_summary_id) = span.read_summary_id else {
20984                return Err(ExcelError::new(ExcelErrorKind::NImpl)
20985                    .with_message("FormulaPlane active span is missing read summary"));
20986            };
20987            let Some(read_summary) = authority.plane.span_read_summaries.get(read_summary_id)
20988            else {
20989                return Err(ExcelError::new(ExcelErrorKind::NImpl)
20990                    .with_message("FormulaPlane active span has stale read summary"));
20991            };
20992            if read_summary.result_region != result_region {
20993                return Err(ExcelError::new(ExcelErrorKind::NImpl)
20994                    .with_message("FormulaPlane active span read summary is stale"));
20995            }
20996            for dependency in &read_summary.dependencies {
20997                consumer_reads.try_reserve(1).map_err(|_| {
20998                    ExcelError::new(ExcelErrorKind::NImpl)
20999                        .with_message("FormulaPlane cache read-index reservation failed")
21000                })?;
21001                consumer_reads.insert_read(
21002                    FormulaProducerId::Span(span.id),
21003                    dependency.read_region,
21004                    read_summary.result_region,
21005                    dependency.projection,
21006                );
21007            }
21008        }
21009
21010        let sheet_disjoint_island = self.prove_sheet_disjoint_legacy_island(
21011            &producer_results,
21012            &consumer_reads,
21013            &legacy_vertices,
21014        );
21015        if sheet_disjoint_island.is_none() {
21016            for vertex in &legacy_vertices {
21017                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
21018                    continue;
21019                };
21020                producer_results.insert_producer(
21021                    FormulaProducerId::Legacy(*vertex),
21022                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col()),
21023                );
21024            }
21025            for vertex in &legacy_vertices {
21026                let Some(cell) = self.graph.get_cell_ref_for_vertex(*vertex) else {
21027                    continue;
21028                };
21029                let result_region =
21030                    Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21031                let mut seen = rustc_hash::FxHashSet::default();
21032                for dep in self.graph.get_dependencies(*vertex) {
21033                    let Some(dep_cell) = self.graph.get_cell_ref_for_vertex(dep) else {
21034                        // Named/table/source vertices do not have a trustworthy
21035                        // regional boundary summary. Preserve the global planner.
21036                        legacy_reads_complete = false;
21037                        continue;
21038                    };
21039                    let read_region = Region::point(
21040                        dep_cell.sheet_id,
21041                        dep_cell.coord.row(),
21042                        dep_cell.coord.col(),
21043                    );
21044                    if seen.insert(read_region) {
21045                        consumer_reads.try_reserve(1).map_err(|_| {
21046                            ExcelError::new(ExcelErrorKind::NImpl)
21047                                .with_message("FormulaPlane cache read-index reservation failed")
21048                        })?;
21049                        consumer_reads.insert_read(
21050                            FormulaProducerId::Legacy(*vertex),
21051                            read_region,
21052                            result_region,
21053                            DirtyProjectionRule::WholeResult,
21054                        );
21055                    }
21056                }
21057                if let Some(ranges) = self.graph.get_range_dependencies(*vertex) {
21058                    for range in ranges {
21059                        let Some(read_region) =
21060                            self.shared_range_to_region_pattern(range, cell.sheet_id)?
21061                        else {
21062                            legacy_reads_complete = false;
21063                            continue;
21064                        };
21065                        if seen.insert(read_region) {
21066                            consumer_reads.try_reserve(1).map_err(|_| {
21067                                ExcelError::new(ExcelErrorKind::NImpl).with_message(
21068                                    "FormulaPlane cache read-index reservation failed",
21069                                )
21070                            })?;
21071                            consumer_reads.insert_read(
21072                                FormulaProducerId::Legacy(*vertex),
21073                                read_region,
21074                                result_region,
21075                                DirtyProjectionRule::WholeResult,
21076                            );
21077                        }
21078                    }
21079                }
21080                let (virtual_dependencies, _) = VirtualDepBuilder::new(self).build(&[*vertex]);
21081                for dependency in virtual_dependencies
21082                    .get(vertex)
21083                    .into_iter()
21084                    .flatten()
21085                    .copied()
21086                {
21087                    let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) else {
21088                        continue;
21089                    };
21090                    let read_region =
21091                        Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21092                    if seen.insert(read_region) {
21093                        consumer_reads.try_reserve(1).map_err(|_| {
21094                            ExcelError::new(ExcelErrorKind::NImpl)
21095                                .with_message("FormulaPlane cache virtual read reservation failed")
21096                        })?;
21097                        consumer_reads.insert_read(
21098                            FormulaProducerId::Legacy(*vertex),
21099                            read_region,
21100                            result_region,
21101                            DirtyProjectionRule::WholeResult,
21102                        );
21103                    }
21104                }
21105                for read_region in DynamicRefVirtualDepProvider::get_virtual_regions(self, *vertex)
21106                {
21107                    if seen.insert(read_region) {
21108                        consumer_reads.try_reserve(1).map_err(|_| {
21109                            ExcelError::new(ExcelErrorKind::NImpl)
21110                                .with_message("FormulaPlane dynamic-region reservation failed")
21111                        })?;
21112                        consumer_reads.insert_read(
21113                            FormulaProducerId::Legacy(*vertex),
21114                            read_region,
21115                            result_region,
21116                            DirtyProjectionRule::WholeResult,
21117                        );
21118                    }
21119                }
21120            }
21121        }
21122
21123        // Boundary discovery uses complete, request-local indexes. Only the
21124        // span-connected closure is retained and compiled; disconnected legacy
21125        // membership is contracted into the legacy scheduler primitive.
21126        let (producer_results, consumer_reads, island) = if let Some(island) = sheet_disjoint_island
21127        {
21128            (producer_results, consumer_reads, island)
21129        } else {
21130            self.contract_legacy_islands(
21131                &producer_results,
21132                &consumer_reads,
21133                legacy_reads_complete,
21134                key.max_candidates,
21135            )
21136            .unwrap_or((
21137                producer_results,
21138                consumer_reads,
21139                LegacyIslandPlan::default(),
21140            ))
21141        };
21142
21143        let retained_memory_bytes = std::mem::size_of::<CachedMixedTopology>()
21144            .checked_add(
21145                producer_results
21146                    .estimated_memory_bytes()
21147                    .unwrap_or(usize::MAX),
21148            )
21149            .and_then(|bytes| {
21150                bytes.checked_add(
21151                    consumer_reads
21152                        .estimated_memory_bytes()
21153                        .unwrap_or(usize::MAX),
21154                )
21155            })
21156            .and_then(|bytes| bytes.checked_add(estimated_span_bindings_bytes(&span_refs_by_id)?))
21157            .and_then(|bytes| bytes.checked_add(island.retained_bytes))
21158            .unwrap_or(usize::MAX);
21159        let config = MixedTopologyConfig {
21160            max_candidates: key.max_candidates,
21161            max_edges: key.max_edges,
21162            max_memory_bytes: key.max_memory_bytes,
21163            retained_memory_bytes,
21164        };
21165        let mut compiled = compile_mixed_topology(&producer_results, &consumer_reads, &config);
21166        // `producers_observed` remains a discovery counter, not merely the
21167        // post-contraction mixed-node count. Preserve that established meaning
21168        // while exposing omissions separately in route telemetry.
21169        match &mut compiled {
21170            MixedTopologyCompileResult::Cached(topology) => {
21171                topology.stats.producers = topology
21172                    .stats
21173                    .producers
21174                    .saturating_add(island.membership.len());
21175                topology.stats.candidates = topology
21176                    .stats
21177                    .candidates
21178                    .saturating_add(island.omitted_direct_relationships);
21179                topology.stats.relationships = topology
21180                    .stats
21181                    .relationships
21182                    .saturating_add(island.omitted_direct_relationships);
21183            }
21184            MixedTopologyCompileResult::CacheSkipped { observed, .. } => {
21185                observed.producers = observed.producers.saturating_add(island.membership.len());
21186                observed.candidates = observed
21187                    .candidates
21188                    .saturating_add(island.omitted_direct_relationships);
21189                observed.relationships = observed
21190                    .relationships
21191                    .saturating_add(island.omitted_direct_relationships);
21192            }
21193        }
21194        let plane_epoch = authority.plane.epoch().0;
21195        Ok(match compiled {
21196            MixedTopologyCompileResult::Cached(topology) => {
21197                FormulaPlaneTopologyCompileResult::Cached(CachedMixedTopology {
21198                    key,
21199                    topology,
21200                    producer_results,
21201                    consumer_reads,
21202                    span_refs_by_id,
21203                    plane_epoch,
21204                    island,
21205                })
21206            }
21207            MixedTopologyCompileResult::CacheSkipped { reason, observed } => {
21208                FormulaPlaneTopologyCompileResult::CacheSkipped(SkippedMixedTopology {
21209                    reason,
21210                    observed,
21211                    producer_results,
21212                    consumer_reads,
21213                    span_refs_by_id,
21214                    plane_epoch,
21215                    island,
21216                })
21217            }
21218        })
21219    }
21220
21221    fn include_non_grid_legacy_work(
21222        &self,
21223        schedule: MixedSchedule,
21224        vertices: &[VertexId],
21225        initial_fallbacks: &[MixedScheduleFallback],
21226        producer_results: &FormulaProducerResultIndex,
21227        consumer_reads: &FormulaConsumerReadIndex,
21228        max_candidates: usize,
21229        max_edges: usize,
21230    ) -> MixedSchedule {
21231        if vertices.is_empty() && initial_fallbacks.is_empty() {
21232            return schedule;
21233        }
21234
21235        // Regional ordering cannot represent dynamic/virtual reads, unresolved
21236        // range shapes, spill domains, or reads from multi-cell producers.
21237        let mut candidate_count = 0usize;
21238        let mut preflight_fallbacks = initial_fallbacks.to_vec();
21239        for &vertex in vertices {
21240            let producer = FormulaProducerId::Legacy(vertex);
21241            let named_formula_is_dynamic =
21242                self.graph
21243                    .named_range_by_vertex(vertex)
21244                    .is_some_and(|named| match &named.definition {
21245                        crate::engine::named_range::NamedDefinition::Formula { ast, .. } => {
21246                            self.graph.is_ast_dynamic(ast)
21247                        }
21248                        _ => false,
21249                    });
21250            if self.graph.is_dynamic(vertex) || named_formula_is_dynamic {
21251                preflight_fallbacks.push(MixedScheduleFallback {
21252                    producer,
21253                    reason: MixedScheduleFallbackReason::UnsupportedProjection,
21254                });
21255                continue;
21256            }
21257
21258            let context_sheet = self.graph.get_vertex_sheet_id(vertex);
21259            for range in self
21260                .graph
21261                .get_range_dependencies(vertex)
21262                .into_iter()
21263                .flatten()
21264            {
21265                let Ok(Some(region)) = self.shared_range_to_region_pattern(range, context_sheet)
21266                else {
21267                    preflight_fallbacks.push(MixedScheduleFallback {
21268                        producer,
21269                        reason: MixedScheduleFallbackReason::UnsupportedProjection,
21270                    });
21271                    break;
21272                };
21273
21274                let (rows, cols) = region.axis_ranges();
21275                let (row_start, row_end) = rows.query_bounds();
21276                let (col_start, col_end) = cols.query_bounds();
21277                if !self
21278                    .graph
21279                    .spill_anchors_in_region(
21280                        region.sheet_id(),
21281                        row_start,
21282                        col_start,
21283                        row_end,
21284                        col_end,
21285                    )
21286                    .is_empty()
21287                {
21288                    preflight_fallbacks.push(MixedScheduleFallback {
21289                        producer,
21290                        reason: MixedScheduleFallbackReason::UnsupportedProjection,
21291                    });
21292                    break;
21293                }
21294
21295                let remaining = max_candidates.saturating_sub(candidate_count);
21296                match producer_results.query_bounded(region, remaining) {
21297                    BoundedRegionQueryResult::Incomplete { .. } => {
21298                        preflight_fallbacks.push(MixedScheduleFallback {
21299                            producer,
21300                            reason: MixedScheduleFallbackReason::MaxCandidatesExceeded,
21301                        });
21302                        break;
21303                    }
21304                    BoundedRegionQueryResult::Complete(query) => {
21305                        candidate_count = candidate_count.saturating_add(query.matches.len());
21306                        if query
21307                            .matches
21308                            .iter()
21309                            .any(|matched| matched.value.result_region.as_point().is_none())
21310                        {
21311                            preflight_fallbacks.push(MixedScheduleFallback {
21312                                producer,
21313                                reason: MixedScheduleFallbackReason::UnsupportedProjection,
21314                            });
21315                            break;
21316                        }
21317                    }
21318                }
21319            }
21320        }
21321
21322        if !preflight_fallbacks.is_empty() {
21323            let mut schedule = schedule;
21324            schedule.stats.input_work_items = schedule
21325                .stats
21326                .input_work_items
21327                .saturating_add(vertices.len())
21328                .saturating_add(initial_fallbacks.len());
21329            schedule.stats.consumer_candidate_count = schedule
21330                .stats
21331                .consumer_candidate_count
21332                .saturating_add(candidate_count);
21333            schedule.stats.max_candidates_exceeded_count =
21334                schedule.stats.max_candidates_exceeded_count.saturating_add(
21335                    preflight_fallbacks
21336                        .iter()
21337                        .filter(|fallback| {
21338                            fallback.reason == MixedScheduleFallbackReason::MaxCandidatesExceeded
21339                        })
21340                        .count(),
21341                );
21342            schedule.fallbacks.extend(preflight_fallbacks);
21343            return schedule;
21344        }
21345
21346        let mut work_by_producer = BTreeMap::new();
21347        for item in schedule
21348            .layers
21349            .iter()
21350            .flat_map(|layer| layer.work.iter())
21351            .cloned()
21352        {
21353            work_by_producer.insert(item.producer, item);
21354        }
21355        for &vertex in vertices {
21356            let producer = FormulaProducerId::Legacy(vertex);
21357            work_by_producer
21358                .entry(producer)
21359                .or_insert(FormulaProducerWork {
21360                    producer,
21361                    dirty: ProducerDirtyDomain::Whole,
21362                });
21363        }
21364
21365        if work_by_producer.len() > max_candidates {
21366            let mut schedule = schedule;
21367            schedule.stats.input_work_items = schedule
21368                .stats
21369                .input_work_items
21370                .saturating_add(vertices.len());
21371            schedule.stats.max_candidates_exceeded_count = schedule
21372                .stats
21373                .max_candidates_exceeded_count
21374                .saturating_add(1);
21375            schedule.fallbacks.push(MixedScheduleFallback {
21376                producer: FormulaProducerId::Legacy(vertices[0]),
21377                reason: MixedScheduleFallbackReason::MaxCandidatesExceeded,
21378            });
21379            return schedule;
21380        }
21381
21382        let producer_set = work_by_producer.keys().copied().collect::<FxHashSet<_>>();
21383        let mut outgoing = BTreeMap::<FormulaProducerId, BTreeSet<FormulaProducerId>>::new();
21384        let mut indegree = work_by_producer
21385            .keys()
21386            .copied()
21387            .map(|producer| (producer, 0usize))
21388            .collect::<BTreeMap<_, _>>();
21389        let mut derived_edges = 0usize;
21390        let mut duplicate_edges = 0usize;
21391        let try_add_edge = |source: FormulaProducerId,
21392                            target: FormulaProducerId,
21393                            outgoing: &mut BTreeMap<_, BTreeSet<_>>,
21394                            indegree: &mut BTreeMap<_, usize>,
21395                            derived_edges: &mut usize,
21396                            duplicate_edges: &mut usize|
21397         -> bool {
21398            if source == target
21399                || !producer_set.contains(&source)
21400                || !producer_set.contains(&target)
21401            {
21402                return true;
21403            }
21404            if outgoing
21405                .get(&source)
21406                .is_some_and(|targets| targets.contains(&target))
21407            {
21408                *duplicate_edges = duplicate_edges.saturating_add(1);
21409                return true;
21410            }
21411            if *derived_edges >= max_edges {
21412                return false;
21413            }
21414            outgoing.entry(source).or_default().insert(target);
21415            *indegree.entry(target).or_default() += 1;
21416            *derived_edges = derived_edges.saturating_add(1);
21417            true
21418        };
21419
21420        let mut overflow_reason = None;
21421        'regional_edges: for &source in &producer_set {
21422            let Some(result_region) = producer_results.producer_result_region(source) else {
21423                continue;
21424            };
21425            let remaining = max_candidates.saturating_sub(candidate_count);
21426            let query = match consumer_reads.query_changed_region_bounded(result_region, remaining)
21427            {
21428                BoundedRegionQueryResult::Incomplete { .. } => {
21429                    overflow_reason = Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21430                    break 'regional_edges;
21431                }
21432                BoundedRegionQueryResult::Complete(query) => query,
21433            };
21434            candidate_count = candidate_count.saturating_add(query.matches.len());
21435            for matched in query.matches {
21436                if !matches!(matched.value.dirty, ProjectionResult::NoIntersection)
21437                    && !try_add_edge(
21438                        source,
21439                        matched.value.consumer,
21440                        &mut outgoing,
21441                        &mut indegree,
21442                        &mut derived_edges,
21443                        &mut duplicate_edges,
21444                    )
21445                {
21446                    overflow_reason = Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21447                    break 'regional_edges;
21448                }
21449            }
21450        }
21451
21452        if overflow_reason.is_none() {
21453            'legacy_edges: for &consumer in &producer_set {
21454                let FormulaProducerId::Legacy(vertex) = consumer else {
21455                    continue;
21456                };
21457                for dependency in self.graph.get_dependencies(vertex) {
21458                    let direct = FormulaProducerId::Legacy(dependency);
21459                    if !try_add_edge(
21460                        direct,
21461                        consumer,
21462                        &mut outgoing,
21463                        &mut indegree,
21464                        &mut derived_edges,
21465                        &mut duplicate_edges,
21466                    ) {
21467                        overflow_reason = Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21468                        break 'legacy_edges;
21469                    }
21470
21471                    if let Some(cell) = self.graph.get_cell_ref_for_vertex(dependency) {
21472                        let point =
21473                            Region::point(cell.sheet_id, cell.coord.row(), cell.coord.col());
21474                        let remaining = max_candidates.saturating_sub(candidate_count);
21475                        let query = match producer_results.query_bounded(point, remaining) {
21476                            BoundedRegionQueryResult::Incomplete { .. } => {
21477                                overflow_reason =
21478                                    Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21479                                break 'legacy_edges;
21480                            }
21481                            BoundedRegionQueryResult::Complete(query) => query,
21482                        };
21483                        candidate_count = candidate_count.saturating_add(query.matches.len());
21484                        for matched in query.matches {
21485                            if !try_add_edge(
21486                                matched.value.producer,
21487                                consumer,
21488                                &mut outgoing,
21489                                &mut indegree,
21490                                &mut derived_edges,
21491                                &mut duplicate_edges,
21492                            ) {
21493                                overflow_reason =
21494                                    Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21495                                break 'legacy_edges;
21496                            }
21497                        }
21498                    }
21499                }
21500                let context_sheet = self.graph.get_vertex_sheet_id(vertex);
21501                for range in self
21502                    .graph
21503                    .get_range_dependencies(vertex)
21504                    .into_iter()
21505                    .flatten()
21506                {
21507                    if let Ok(Some(region)) =
21508                        self.shared_range_to_region_pattern(range, context_sheet)
21509                    {
21510                        let remaining = max_candidates.saturating_sub(candidate_count);
21511                        let query = match producer_results.query_bounded(region, remaining) {
21512                            BoundedRegionQueryResult::Incomplete { .. } => {
21513                                overflow_reason =
21514                                    Some(MixedScheduleFallbackReason::MaxCandidatesExceeded);
21515                                break 'legacy_edges;
21516                            }
21517                            BoundedRegionQueryResult::Complete(query) => query,
21518                        };
21519                        candidate_count = candidate_count.saturating_add(query.matches.len());
21520                        for matched in query.matches {
21521                            if !try_add_edge(
21522                                matched.value.producer,
21523                                consumer,
21524                                &mut outgoing,
21525                                &mut indegree,
21526                                &mut derived_edges,
21527                                &mut duplicate_edges,
21528                            ) {
21529                                overflow_reason =
21530                                    Some(MixedScheduleFallbackReason::MaxEdgesExceeded);
21531                                break 'legacy_edges;
21532                            }
21533                        }
21534                    }
21535                }
21536            }
21537        }
21538
21539        if let Some(reason) = overflow_reason {
21540            let mut schedule = schedule;
21541            schedule.stats.input_work_items = schedule
21542                .stats
21543                .input_work_items
21544                .saturating_add(vertices.len());
21545            schedule.stats.consumer_candidate_count = schedule
21546                .stats
21547                .consumer_candidate_count
21548                .saturating_add(candidate_count);
21549            schedule.stats.duplicate_edges_skipped = schedule
21550                .stats
21551                .duplicate_edges_skipped
21552                .saturating_add(duplicate_edges);
21553            match reason {
21554                MixedScheduleFallbackReason::MaxEdgesExceeded => {
21555                    schedule.stats.max_edges_exceeded_count =
21556                        schedule.stats.max_edges_exceeded_count.saturating_add(1);
21557                }
21558                MixedScheduleFallbackReason::MaxCandidatesExceeded => {
21559                    schedule.stats.max_candidates_exceeded_count = schedule
21560                        .stats
21561                        .max_candidates_exceeded_count
21562                        .saturating_add(1);
21563                }
21564                _ => unreachable!("non-grid reconstruction overflow reason is capped"),
21565            }
21566            schedule.fallbacks.push(MixedScheduleFallback {
21567                producer: FormulaProducerId::Legacy(vertices[0]),
21568                reason,
21569            });
21570            return schedule;
21571        }
21572
21573        let mut ready = indegree
21574            .iter()
21575            .filter_map(|(&producer, &degree)| (degree == 0).then_some(producer))
21576            .collect::<BTreeSet<_>>();
21577        let mut layers = Vec::new();
21578        let mut scheduled_count = 0usize;
21579        while !ready.is_empty() {
21580            let current = std::mem::take(&mut ready);
21581            let mut next = BTreeSet::new();
21582            let mut work = Vec::with_capacity(current.len());
21583            for producer in current {
21584                scheduled_count = scheduled_count.saturating_add(1);
21585                work.push(
21586                    work_by_producer
21587                        .get(&producer)
21588                        .expect("ready producer has work")
21589                        .clone(),
21590                );
21591                for &target in outgoing.get(&producer).into_iter().flatten() {
21592                    let degree = indegree.get_mut(&target).expect("edge target has indegree");
21593                    *degree -= 1;
21594                    if *degree == 0 {
21595                        next.insert(target);
21596                    }
21597                }
21598            }
21599            layers.push(MixedLayer { work });
21600            ready = next;
21601        }
21602
21603        let mut fallbacks = schedule.fallbacks;
21604        let mut stats = schedule.stats;
21605        if scheduled_count != work_by_producer.len() {
21606            for (&producer, &degree) in &indegree {
21607                if degree != 0 {
21608                    fallbacks.push(MixedScheduleFallback {
21609                        producer,
21610                        reason: MixedScheduleFallbackReason::CycleDetected,
21611                    });
21612                }
21613            }
21614            stats.cycle_count = stats.cycle_count.saturating_add(1);
21615        }
21616        stats.input_work_items = stats.input_work_items.saturating_add(vertices.len());
21617        stats.merged_work_items = work_by_producer.len();
21618        stats.unique_producers = work_by_producer.len();
21619        stats.consumer_candidate_count = stats
21620            .consumer_candidate_count
21621            .saturating_add(candidate_count);
21622        stats.edges_added = stats.edges_added.saturating_add(derived_edges);
21623        stats.duplicate_edges_skipped = stats
21624            .duplicate_edges_skipped
21625            .saturating_add(duplicate_edges);
21626        stats.layers = layers.len();
21627        MixedSchedule {
21628            layers,
21629            stats,
21630            fallbacks,
21631        }
21632    }
21633
21634    fn build_formula_plane_mixed_schedule(
21635        &mut self,
21636        formula_dirty: &FormulaDirtyLease,
21637        retry_whole_spans: &[FormulaSpanRef],
21638        include_dirty_regions: bool,
21639    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21640        self.build_formula_plane_mixed_schedule_inner(
21641            formula_dirty,
21642            retry_whole_spans,
21643            include_dirty_regions,
21644            None,
21645        )
21646    }
21647
21648    fn build_formula_plane_target_schedule(
21649        &mut self,
21650        formula_dirty: &FormulaDirtyLease,
21651        retry_whole_spans: &[FormulaSpanRef],
21652        include_dirty_regions: bool,
21653        target_roots: &[crate::engine::target_preparation::TargetProducer],
21654    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21655        self.build_formula_plane_mixed_schedule_inner(
21656            formula_dirty,
21657            retry_whole_spans,
21658            include_dirty_regions,
21659            Some(target_roots),
21660        )
21661    }
21662
21663    fn build_formula_plane_mixed_schedule_inner(
21664        &mut self,
21665        formula_dirty: &FormulaDirtyLease,
21666        retry_whole_spans: &[FormulaSpanRef],
21667        include_dirty_regions: bool,
21668        target_roots: Option<&[crate::engine::target_preparation::TargetProducer]>,
21669    ) -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21670        let baseline = self.graph.baseline_stats();
21671        self.charge_bounded_work(
21672            baseline
21673                .graph_vertex_count
21674                .saturating_add(baseline.graph_edge_count) as u64,
21675        )?;
21676        let key = self.mixed_topology_cache_key();
21677        let active_refs = self
21678            .graph
21679            .formula_authority()
21680            .active_span_refs()
21681            .into_iter()
21682            .map(|span_ref| (span_ref.id, span_ref))
21683            .collect::<BTreeMap<_, _>>();
21684        let has_dynamic_legacy = self
21685            .graph
21686            .formula_vertices()
21687            .into_iter()
21688            .any(|vertex| self.graph.is_dynamic(vertex));
21689        let cache_hit = !has_dynamic_legacy
21690            && self
21691                .cached_mixed_topology
21692                .as_ref()
21693                .is_some_and(|cached| cached.key == key && cached.span_refs_by_id == active_refs);
21694        let mut skipped_topology = None;
21695        let mut index_scratch_reserved = 0_u64;
21696        let mut demand_scratch_reserved = 0_u64;
21697        let topology_started = crate::instant::FzInstant::now();
21698        let (cache_outcome, strategy, compile_stats) = if cache_hit {
21699            self.mixed_topology_cache_hits = self.mixed_topology_cache_hits.saturating_add(1);
21700            self.mixed_topology_cache_skip_streak = 0;
21701            let cached = self
21702                .cached_mixed_topology
21703                .as_ref()
21704                .expect("cache hit has topology");
21705            let stats = cached.topology.stats.clone();
21706            let strategy = if cached.topology.is_complete() {
21707                FormulaPlaneTopologyStrategy::Cached
21708            } else {
21709                FormulaPlaneTopologyStrategy::ExactPagedIndexed
21710            };
21711            if let Some(ledger) = self.active_resource_ledger.as_mut() {
21712                ledger
21713                    .account_mixed_cache(stats.estimated_memory_bytes as u64)
21714                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21715            }
21716            (FormulaPlaneTopologyCacheOutcome::Hit, strategy, stats)
21717        } else {
21718            // A key/cap miss makes the old retained topology stale. Drop it before building its
21719            // replacement so the request never accounts or owns both generations concurrently.
21720            self.cached_mixed_topology = None;
21721            if let Some(ledger) = self.active_resource_ledger.as_mut() {
21722                ledger
21723                    .account_mixed_cache(0)
21724                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21725            }
21726
21727            let preflight = self.mixed_topology_index_preflight_bytes();
21728            if let Some(ledger) = self.active_resource_ledger.as_mut() {
21729                ledger
21730                    .reserve_schedule_discovery(preflight)
21731                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21732            }
21733            index_scratch_reserved = preflight;
21734            let compiled = match self.compile_formula_plane_mixed_topology(key.clone()) {
21735                Ok(compiled) => compiled,
21736                Err(error) => {
21737                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
21738                        let _ = ledger.release_scratch(index_scratch_reserved);
21739                    }
21740                    return Err(error);
21741                }
21742            };
21743            let actual_index_bytes = match &compiled {
21744                FormulaPlaneTopologyCompileResult::Cached(compiled) => compiled
21745                    .producer_results
21746                    .estimated_memory_bytes()
21747                    .and_then(|bytes| {
21748                        bytes.checked_add(compiled.consumer_reads.estimated_memory_bytes()?)
21749                    })
21750                    .and_then(|bytes| {
21751                        bytes.checked_add(estimated_span_bindings_bytes(&compiled.span_refs_by_id)?)
21752                    })
21753                    .and_then(|bytes| bytes.checked_add(compiled.island.retained_bytes)),
21754                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => skipped
21755                    .producer_results
21756                    .estimated_memory_bytes()
21757                    .and_then(|bytes| {
21758                        bytes.checked_add(skipped.consumer_reads.estimated_memory_bytes()?)
21759                    })
21760                    .and_then(|bytes| {
21761                        bytes.checked_add(estimated_span_bindings_bytes(&skipped.span_refs_by_id)?)
21762                    })
21763                    .and_then(|bytes| bytes.checked_add(skipped.island.retained_bytes)),
21764            }
21765            .unwrap_or(usize::MAX) as u64;
21766            debug_assert!(
21767                actual_index_bytes <= index_scratch_reserved,
21768                "mixed topology index preflight underestimated actual allocation"
21769            );
21770            if actual_index_bytes > index_scratch_reserved {
21771                let additional = actual_index_bytes - index_scratch_reserved;
21772                let reserve_result = self
21773                    .active_resource_ledger
21774                    .as_mut()
21775                    .map_or(Ok(()), |ledger| {
21776                        ledger.reserve_schedule_discovery(additional)
21777                    });
21778                if let Err(error) = reserve_result {
21779                    drop(compiled);
21780                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
21781                        let _ = ledger.release_scratch(index_scratch_reserved);
21782                    }
21783                    return Err(error.into_excel_error());
21784                }
21785            } else if actual_index_bytes < index_scratch_reserved
21786                && let Some(ledger) = self.active_resource_ledger.as_mut()
21787            {
21788                ledger
21789                    .release_scratch(index_scratch_reserved - actual_index_bytes)
21790                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21791            }
21792            index_scratch_reserved = actual_index_bytes;
21793
21794            self.mixed_topology_cache_builds = self.mixed_topology_cache_builds.saturating_add(1);
21795            match compiled {
21796                FormulaPlaneTopologyCompileResult::Cached(compiled) if has_dynamic_legacy => {
21797                    let stats = compiled.topology.stats.clone();
21798                    self.mixed_topology_cache_skip_streak =
21799                        self.mixed_topology_cache_skip_streak.saturating_add(1);
21800                    skipped_topology = Some(SkippedMixedTopology {
21801                        reason: MixedScheduleFallbackReason::CacheMemoryExceeded,
21802                        observed: stats.clone(),
21803                        producer_results: compiled.producer_results,
21804                        consumer_reads: compiled.consumer_reads,
21805                        span_refs_by_id: compiled.span_refs_by_id,
21806                        plane_epoch: compiled.plane_epoch,
21807                        island: compiled.island,
21808                    });
21809                    (
21810                        FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy,
21811                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
21812                        stats,
21813                    )
21814                }
21815                FormulaPlaneTopologyCompileResult::Cached(compiled) => {
21816                    let stats = compiled.topology.stats.clone();
21817                    let strategy = if compiled.topology.is_complete() {
21818                        FormulaPlaneTopologyStrategy::CompiledAndCached
21819                    } else {
21820                        FormulaPlaneTopologyStrategy::ExactPagedIndexed
21821                    };
21822                    let retained_result = self
21823                        .active_resource_ledger
21824                        .as_mut()
21825                        .map_or(Ok(()), |ledger| {
21826                            ledger.account_mixed_cache(stats.estimated_memory_bytes as u64)
21827                        });
21828                    if let Err(error) = retained_result {
21829                        if let Some(ledger) = self.active_resource_ledger.as_mut() {
21830                            let _ = ledger.release_scratch(index_scratch_reserved);
21831                        }
21832                        return Err(error.into_excel_error());
21833                    }
21834                    self.mixed_topology_cache_skip_streak = 0;
21835                    self.cached_mixed_topology = Some(compiled);
21836                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
21837                        ledger
21838                            .release_scratch(index_scratch_reserved)
21839                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21840                    }
21841                    index_scratch_reserved = 0;
21842                    (FormulaPlaneTopologyCacheOutcome::Built, strategy, stats)
21843                }
21844                FormulaPlaneTopologyCompileResult::CacheSkipped(skipped) => {
21845                    debug_assert!(self.cached_mixed_topology.is_none());
21846                    self.mixed_topology_cache_overflows =
21847                        self.mixed_topology_cache_overflows.saturating_add(1);
21848                    self.mixed_topology_cache_skip_streak =
21849                        self.mixed_topology_cache_skip_streak.saturating_add(1);
21850                    let stats = skipped.observed.clone();
21851                    skipped_topology = Some(skipped);
21852                    (
21853                        FormulaPlaneTopologyCacheOutcome::SkippedOverflow,
21854                        FormulaPlaneTopologyStrategy::ExactPagedIndexed,
21855                        stats,
21856                    )
21857                }
21858            }
21859        };
21860        self.observe_topology(
21861            cache_outcome,
21862            strategy,
21863            &compile_stats,
21864            topology_started.elapsed(),
21865        );
21866        if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
21867            stats.topology.cache_skip_streak = self.mixed_topology_cache_skip_streak;
21868            stats.topology.candidate_cap = Some(key.max_candidates as u64);
21869            stats.topology.edge_cap = Some(key.max_edges as u64);
21870            stats.topology.retained_byte_cap = Some(key.max_memory_bytes as u64);
21871            if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedDynamicLegacy {
21872                stats.topology.operator_guidance =
21873                    Some("dynamic legacy dependencies require an exact request topology rebuild");
21874            } else if cache_outcome == FormulaPlaneTopologyCacheOutcome::SkippedOverflow {
21875                stats.topology.operator_guidance =
21876                    Some(if compile_stats.candidate_overflow_count > 0 {
21877                        "raise the mixed-cache candidate limit to avoid exact request rebuilds"
21878                    } else if compile_stats.edge_overflow_count > 0 {
21879                        "raise the mixed-cache edge limit to avoid exact request rebuilds"
21880                    } else {
21881                        "raise the retained mixed-cache byte limit to avoid exact request rebuilds"
21882                    });
21883            }
21884        }
21885
21886        let reuse_partial_topology = cache_outcome == FormulaPlaneTopologyCacheOutcome::Hit;
21887        let schedule_result = (|| -> Result<FormulaPlaneMixedScheduleBuild, ExcelError> {
21888            let (
21889                producer_results,
21890                consumer_reads,
21891                span_refs_by_id,
21892                plane_epoch,
21893                cached_topology,
21894                partial_topology,
21895                island,
21896            ) = if let Some(skipped) = skipped_topology.as_ref() {
21897                (
21898                    &skipped.producer_results,
21899                    &skipped.consumer_reads,
21900                    &skipped.span_refs_by_id,
21901                    skipped.plane_epoch,
21902                    None,
21903                    None,
21904                    &skipped.island,
21905                )
21906            } else {
21907                let cached = self
21908                    .cached_mixed_topology
21909                    .as_ref()
21910                    .expect("mixed topology available for request");
21911                let (complete, partial) = if cached.topology.is_complete() {
21912                    (Some(&cached.topology), None)
21913                } else {
21914                    (None, Some(&cached.topology))
21915                };
21916                (
21917                    &cached.producer_results,
21918                    &cached.consumer_reads,
21919                    &cached.span_refs_by_id,
21920                    cached.plane_epoch,
21921                    complete,
21922                    partial,
21923                    &cached.island,
21924                )
21925            };
21926            let demand = if let Some(target_roots) = target_roots {
21927                use crate::engine::target_preparation::TargetProducer;
21928                let mut roots = Vec::new();
21929                let mut graph_roots = Vec::new();
21930                for root in target_roots {
21931                    match *root {
21932                        TargetProducer::Span { span_ref, demanded } => {
21933                            roots.push((FormulaProducerId::Span(span_ref.id), demanded))
21934                        }
21935                        TargetProducer::Legacy(vertex) => {
21936                            graph_roots.push(vertex);
21937                            if let Some(region) = producer_results
21938                                .producer_result_region(FormulaProducerId::Legacy(vertex))
21939                            {
21940                                roots.push((FormulaProducerId::Legacy(vertex), region));
21941                            }
21942                        }
21943                        TargetProducer::Symbol(vertex) => graph_roots.push(vertex),
21944                        TargetProducer::ValueOnly(_) => {}
21945                    }
21946                }
21947                if !graph_roots.is_empty() {
21948                    let (legacy_demand, _) = self.build_demand_subgraph(&graph_roots);
21949                    for vertex in legacy_demand {
21950                        if let Some(region) = producer_results
21951                            .producer_result_region(FormulaProducerId::Legacy(vertex))
21952                        {
21953                            roots.push((FormulaProducerId::Legacy(vertex), region));
21954                        }
21955                    }
21956                }
21957                let roots = roots;
21958                let closure_base = (producer_results.len() as u64)
21959                    .saturating_mul(128)
21960                    .saturating_add(4096);
21961                let scratch_estimates =
21962                    exact_demand_scratch_estimates(consumer_reads.len(), closure_base);
21963                let native_allowed = native_exact_demand_allowed(
21964                    self.active_resource_ledger
21965                        .as_ref()
21966                        .and_then(ResourceLedger::disk_scratch_policy),
21967                    !cfg!(target_arch = "wasm32"),
21968                );
21969                let (selected_strategy, reservation) = if cached_topology.is_some() {
21970                    (None, closure_base)
21971                } else {
21972                    let (strategy, reservation) =
21973                        select_exact_demand_strategy(scratch_estimates, native_allowed, |bytes| {
21974                            self.can_reserve_topology_scratch(bytes)
21975                        });
21976                    (Some(strategy), reservation)
21977                };
21978                if let Some(ledger) = self.active_resource_ledger.as_mut() {
21979                    ledger
21980                        .reserve_schedule_discovery(reservation)
21981                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
21982                }
21983                demand_scratch_reserved = reservation;
21984                #[allow(unused_mut)]
21985                let mut native_disk_bytes = 0_u64;
21986                let (demand, actual_strategy, passes) = match selected_strategy {
21987                    None => (
21988                        build_demand_closure_cached(
21989                            roots,
21990                            producer_results,
21991                            cached_topology.expect("cached demand strategy has topology"),
21992                        ),
21993                        None,
21994                        0,
21995                    ),
21996                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
21997                        let (demand, passes) = build_demand_closure_paged(
21998                            roots,
21999                            producer_results,
22000                            consumer_reads,
22001                            |units| {
22002                                Self::resource_loop_checkpoint(
22003                                    &mut self.active_resource_ledger,
22004                                    units,
22005                                )
22006                            },
22007                        )?;
22008                        (demand, selected_strategy, passes)
22009                    }
22010                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
22011                        let (demand, passes) = build_demand_closure_in_memory_runs(
22012                            roots,
22013                            producer_results,
22014                            consumer_reads,
22015                            |units| {
22016                                Self::resource_loop_checkpoint(
22017                                    &mut self.active_resource_ledger,
22018                                    units,
22019                                )
22020                            },
22021                        )?;
22022                        (demand, selected_strategy, passes)
22023                    }
22024                    #[cfg(not(target_arch = "wasm32"))]
22025                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
22026                        let request_id = self
22027                            .active_evaluation_resource_request
22028                            .as_ref()
22029                            .map_or(0, |stats| stats.request_id);
22030                        match NativeTopologyScratch::create(request_id) {
22031                            Ok(mut native) => match build_demand_closure_native(
22032                                roots.clone(),
22033                                producer_results,
22034                                consumer_reads,
22035                                native.file.as_mut().expect("native scratch file"),
22036                                native
22037                                    .auxiliary_file
22038                                    .as_mut()
22039                                    .expect("native auxiliary scratch file"),
22040                                |units| {
22041                                    Self::resource_loop_checkpoint(
22042                                        &mut self.active_resource_ledger,
22043                                        units,
22044                                    )
22045                                },
22046                            ) {
22047                                Ok((demand, passes, bytes)) => {
22048                                    native_disk_bytes = bytes;
22049                                    (demand, selected_strategy, passes)
22050                                }
22051                                Err(NativeExactDemandError::Work(error)) => return Err(error),
22052                                Err(NativeExactDemandError::Io(_)) => {
22053                                    let (demand, passes) = build_demand_closure_repeated_passes(
22054                                        roots,
22055                                        producer_results,
22056                                        consumer_reads,
22057                                        |units| {
22058                                            Self::resource_loop_checkpoint(
22059                                                &mut self.active_resource_ledger,
22060                                                units,
22061                                            )
22062                                        },
22063                                    )?;
22064                                    (
22065                                        demand,
22066                                        Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22067                                        passes,
22068                                    )
22069                                }
22070                            },
22071                            Err(_) => {
22072                                let (demand, passes) = build_demand_closure_repeated_passes(
22073                                    roots,
22074                                    producer_results,
22075                                    consumer_reads,
22076                                    |units| {
22077                                        Self::resource_loop_checkpoint(
22078                                            &mut self.active_resource_ledger,
22079                                            units,
22080                                        )
22081                                    },
22082                                )?;
22083                                (
22084                                    demand,
22085                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22086                                    passes,
22087                                )
22088                            }
22089                        }
22090                    }
22091                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
22092                        let (demand, passes) = build_demand_closure_repeated_passes(
22093                            roots,
22094                            producer_results,
22095                            consumer_reads,
22096                            |units| {
22097                                Self::resource_loop_checkpoint(
22098                                    &mut self.active_resource_ledger,
22099                                    units,
22100                                )
22101                            },
22102                        )?;
22103                        (demand, selected_strategy, passes)
22104                    }
22105                    _ => unreachable!("exact demand strategy selection is exhaustive"),
22106                };
22107                let strategy_memory = match actual_strategy {
22108                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed) => {
22109                        scratch_estimates.paged.saturating_sub(closure_base)
22110                    }
22111                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns) => {
22112                        scratch_estimates.runs.saturating_sub(closure_base)
22113                    }
22114                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch) => {
22115                        scratch_estimates.native.saturating_sub(closure_base)
22116                    }
22117                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses) => {
22118                        scratch_estimates.repeated.saturating_sub(closure_base)
22119                    }
22120                    None => 0,
22121                    _ => 0,
22122                };
22123                let actual_scratch = demand
22124                    .estimated_memory_bytes()
22125                    .saturating_add(strategy_memory);
22126                if actual_scratch > demand_scratch_reserved {
22127                    if let Some(ledger) = self.active_resource_ledger.as_mut() {
22128                        ledger
22129                            .reserve_schedule_discovery(
22130                                actual_scratch.saturating_sub(demand_scratch_reserved),
22131                            )
22132                            .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22133                    }
22134                } else if actual_scratch < demand_scratch_reserved
22135                    && let Some(ledger) = self.active_resource_ledger.as_mut()
22136                {
22137                    ledger
22138                        .release_scratch(demand_scratch_reserved - actual_scratch)
22139                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22140                }
22141                demand_scratch_reserved = actual_scratch;
22142                if let Some(strategy) = actual_strategy
22143                    && let Some(stats) = self.active_evaluation_resource_request.as_mut()
22144                {
22145                    if strategy.severity() > stats.topology.strategy.severity() {
22146                        stats.topology.strategy = strategy;
22147                    }
22148                    stats.topology.exact_pass_count = stats
22149                        .topology
22150                        .exact_pass_count
22151                        .saturating_add(passes.max(1));
22152                    stats.topology.native_topology_disk_bytes = stats
22153                        .topology
22154                        .native_topology_disk_bytes
22155                        .saturating_add(native_disk_bytes);
22156                }
22157                Some(demand)
22158            } else {
22159                None
22160            };
22161
22162            let dirty_legacy = self.graph.get_evaluation_vertices();
22163            let dirty_legacy_set = dirty_legacy.iter().copied().collect::<FxHashSet<_>>();
22164            let island_target_demand = target_roots.map(|roots| {
22165                use crate::engine::target_preparation::TargetProducer;
22166                let graph_roots = roots
22167                    .iter()
22168                    .filter_map(|root| match *root {
22169                        TargetProducer::Legacy(vertex) | TargetProducer::Symbol(vertex) => {
22170                            Some(vertex)
22171                        }
22172                        TargetProducer::Span { .. } | TargetProducer::ValueOnly(_) => None,
22173                    })
22174                    .collect::<Vec<_>>();
22175                self.build_demand_subgraph(&graph_roots)
22176                    .0
22177                    .into_iter()
22178                    .collect::<FxHashSet<_>>()
22179            });
22180            let mut island_plan = island.clone();
22181            island_plan.dirty_vertices = island_plan
22182                .membership
22183                .iter()
22184                .copied()
22185                .filter(|vertex| dirty_legacy_set.contains(vertex))
22186                .filter(|vertex| {
22187                    island_target_demand
22188                        .as_ref()
22189                        .is_none_or(|demand| demand.contains(vertex))
22190                })
22191                .collect();
22192            let mut work = Vec::new();
22193            let mut scheduled_legacy_vertices = Vec::new();
22194            let mut non_grid_legacy_vertices = Vec::new();
22195            let mut non_grid_legacy_fallbacks = Vec::new();
22196            let demanded_dirty = |producer: FormulaProducerId,
22197                                  dirty: ProducerDirtyDomain|
22198             -> Option<ProducerDirtyDomain> {
22199                let Some(demand) = demand.as_ref() else {
22200                    return Some(dirty);
22201                };
22202                let result = producer_results.producer_result_region(producer)?;
22203                dirty.intersect_demanded(result, demand.regions(producer))
22204            };
22205            for (span_ref, region) in formula_dirty.span_regions() {
22206                let producer = FormulaProducerId::Span(span_ref.id);
22207                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
22208                    && let Some(dirty) =
22209                        demanded_dirty(producer, ProducerDirtyDomain::Regions(vec![region]))
22210                {
22211                    work.push(FormulaProducerWork { producer, dirty });
22212                }
22213            }
22214            for span_ref in formula_dirty
22215                .whole_spans()
22216                .chain(retry_whole_spans.iter().copied())
22217            {
22218                let producer = FormulaProducerId::Span(span_ref.id);
22219                if span_refs_by_id.get(&span_ref.id) == Some(&span_ref)
22220                    && let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole)
22221                {
22222                    work.push(FormulaProducerWork { producer, dirty });
22223                }
22224            }
22225            for vertex in dirty_legacy {
22226                let producer = FormulaProducerId::Legacy(vertex);
22227                if producer_results.producer_result_region(producer).is_some() {
22228                    if let Some(dirty) = demanded_dirty(producer, ProducerDirtyDomain::Whole) {
22229                        scheduled_legacy_vertices.push(vertex);
22230                        work.push(FormulaProducerWork { producer, dirty });
22231                    }
22232                } else if matches!(
22233                    self.graph.get_vertex_kind(vertex),
22234                    VertexKind::NamedScalar | VertexKind::NamedArray
22235                ) {
22236                    if island_target_demand
22237                        .as_ref()
22238                        .is_none_or(|demand| demand.contains(&vertex))
22239                    {
22240                        // NamedScalar/NamedArray producers intentionally have no
22241                        // grid result region. Keep them out of regional dirty
22242                        // projection, then merge them back through graph ordering
22243                        // after the regional schedule is built.
22244                        non_grid_legacy_vertices.push(vertex);
22245                    } else {
22246                        non_grid_legacy_fallbacks.push(MixedScheduleFallback {
22247                            producer,
22248                            reason: MixedScheduleFallbackReason::MissingProducerResultRegion,
22249                        });
22250                    }
22251                }
22252            }
22253
22254            let pending_changed_regions = formula_dirty
22255                .regions()
22256                .chain(formula_dirty.span_regions().map(|(_, region)| region))
22257                .collect::<Vec<_>>();
22258            if include_dirty_regions && !pending_changed_regions.is_empty() {
22259                use crate::formula_plane::producer::compute_dirty_closure_checked;
22260                let closure = compute_dirty_closure_checked(
22261                    consumer_reads,
22262                    pending_changed_regions.iter().copied(),
22263                    |producer| producer_results.producer_result_region(producer),
22264                    |units| Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units),
22265                )?;
22266                if closure.incomplete {
22267                    // The fixed-point guard discards partial closure output. Complete conservatively
22268                    // in the same topology/schedule path; it must not become a new demotion route.
22269                    work.clear();
22270                    scheduled_legacy_vertices.clear();
22271                    if let Some(demand) = demand.as_ref() {
22272                        for producer in demand.producers() {
22273                            if let Some(dirty) =
22274                                demanded_dirty(producer, ProducerDirtyDomain::Whole)
22275                            {
22276                                if let FormulaProducerId::Legacy(vertex) = producer {
22277                                    scheduled_legacy_vertices.push(vertex);
22278                                }
22279                                work.push(FormulaProducerWork { producer, dirty });
22280                            }
22281                        }
22282                    } else {
22283                        work.extend(span_refs_by_id.keys().copied().map(|id| {
22284                            FormulaProducerWork {
22285                                producer: FormulaProducerId::Span(id),
22286                                dirty: ProducerDirtyDomain::Whole,
22287                            }
22288                        }));
22289                        for vertex in self.graph.formula_vertices() {
22290                            let producer = FormulaProducerId::Legacy(vertex);
22291                            if producer_results.producer_result_region(producer).is_some() {
22292                                scheduled_legacy_vertices.push(vertex);
22293                                work.push(FormulaProducerWork {
22294                                    producer,
22295                                    dirty: ProducerDirtyDomain::Whole,
22296                                });
22297                            }
22298                        }
22299                    }
22300                    if let Some(stats) = self.active_evaluation_resource_request.as_mut() {
22301                        stats.topology.incomplete_reason =
22302                            Some(crate::engine::EvaluationIncompleteReason::DirtyClosureWork);
22303                    }
22304                } else if let Some(demand) = demand.as_ref() {
22305                    for item in closure.work {
22306                        if let Some(dirty) = demanded_dirty(item.producer, item.dirty) {
22307                            work.push(FormulaProducerWork {
22308                                producer: item.producer,
22309                                dirty,
22310                            });
22311                        }
22312                    }
22313                } else {
22314                    work.extend(closure.work);
22315                }
22316                if !closure.incomplete && !closure.fallbacks.is_empty() {
22317                    let mut already_whole = work
22318                        .iter()
22319                        .filter_map(|item| match (item.producer, &item.dirty) {
22320                            (FormulaProducerId::Span(id), ProducerDirtyDomain::Whole) => Some(id),
22321                            _ => None,
22322                        })
22323                        .collect::<rustc_hash::FxHashSet<_>>();
22324                    for fallback in closure.fallbacks {
22325                        if let FormulaProducerId::Span(id) = fallback.consumer
22326                            && already_whole.insert(id)
22327                        {
22328                            work.push(FormulaProducerWork {
22329                                producer: FormulaProducerId::Span(id),
22330                                dirty: ProducerDirtyDomain::Whole,
22331                            });
22332                        }
22333                    }
22334                }
22335            }
22336
22337            scheduled_legacy_vertices.extend(non_grid_legacy_vertices.iter().copied());
22338
22339            let owned_dirty_events = if let Some(demand) = demand.as_ref() {
22340                use crate::formula_plane::producer::compute_dirty_closure_checked;
22341                let mut owned = Vec::new();
22342                for (index, event) in formula_dirty.events() {
22343                    let (seed_regions, initial) = match event {
22344                        FormulaDirtyEventSnapshot::Region(region) => (vec![region], None),
22345                        FormulaDirtyEventSnapshot::SpanRegion { span_ref, region } => (
22346                            vec![region],
22347                            Some(FormulaProducerWork {
22348                                producer: FormulaProducerId::Span(span_ref.id),
22349                                dirty: ProducerDirtyDomain::Regions(vec![region]),
22350                            }),
22351                        ),
22352                        FormulaDirtyEventSnapshot::WholeSpan(span_ref) => {
22353                            let producer = FormulaProducerId::Span(span_ref.id);
22354                            let Some(result) = producer_results.producer_result_region(producer)
22355                            else {
22356                                continue;
22357                            };
22358                            (
22359                                vec![result],
22360                                Some(FormulaProducerWork {
22361                                    producer,
22362                                    dirty: ProducerDirtyDomain::Whole,
22363                                }),
22364                            )
22365                        }
22366                    };
22367                    let closure = compute_dirty_closure_checked(
22368                        consumer_reads,
22369                        seed_regions,
22370                        |producer| producer_results.producer_result_region(producer),
22371                        |units| {
22372                            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, units)
22373                        },
22374                    )?;
22375                    if closure.incomplete || !closure.fallbacks.is_empty() {
22376                        continue;
22377                    }
22378                    let fully_demanded =
22379                        initial
22380                            .into_iter()
22381                            .chain(closure.work.into_iter())
22382                            .all(|item| {
22383                                let Some(result) =
22384                                    producer_results.producer_result_region(item.producer)
22385                                else {
22386                                    return false;
22387                                };
22388                                item.dirty
22389                                    .result_regions(result)
22390                                    .into_iter()
22391                                    .all(|dirty_region| {
22392                                        demand
22393                                            .regions(item.producer)
22394                                            .iter()
22395                                            .any(|demanded| demanded.contains_region(dirty_region))
22396                                    })
22397                            });
22398                    if fully_demanded {
22399                        owned.push(index);
22400                    }
22401                }
22402                owned
22403            } else {
22404                (0..formula_dirty.len()).collect()
22405            };
22406
22407            Self::resource_loop_checkpoint(&mut self.active_resource_ledger, work.len() as u64)?;
22408            let producer_count = producer_results.len();
22409            let dirty_units = work.iter().fold(0usize, |total, item| {
22410                total.saturating_add(match &item.dirty {
22411                    ProducerDirtyDomain::Whole => 0,
22412                    ProducerDirtyDomain::Cells(cells) => cells.len(),
22413                    ProducerDirtyDomain::Regions(regions) => regions.len(),
22414                })
22415            });
22416            let work_bytes = work
22417                .len()
22418                .saturating_mul(std::mem::size_of::<FormulaProducerWork>() + 64)
22419                .saturating_add(
22420                    dirty_units.saturating_mul(std::mem::size_of::<Region>().max(
22421                        std::mem::size_of::<crate::formula_plane::region_index::RegionKey>(),
22422                    )),
22423                ) as u64;
22424            let binding_bytes =
22425                estimated_span_bindings_bytes(span_refs_by_id).unwrap_or(usize::MAX) as u64;
22426            let schedule_bytes = work_bytes
22427                .saturating_mul(4)
22428                .saturating_add((producer_count as u64).saturating_mul(192))
22429                .saturating_add(binding_bytes)
22430                .saturating_add(1024);
22431            let maximum_edges =
22432                producer_count.saturating_mul(producer_count.saturating_sub(1)) as u64;
22433            // Producer/read indexes are either retained by the cache or covered by the live index
22434            // preflight reservation. Strategy reservations therefore cover only additional scheduling
22435            // structures and must not charge those indexes a second time.
22436            let paged_bytes = schedule_bytes
22437                .saturating_add(maximum_edges.saturating_mul(160))
22438                .saturating_add(16 * 1024);
22439            let runs_bytes = schedule_bytes
22440                .saturating_add(maximum_edges.saturating_mul(48))
22441                .saturating_add(8192);
22442            let native_bytes = schedule_bytes
22443                .saturating_add((producer_count as u64).saturating_mul(128))
22444                .saturating_add(2048);
22445            let repeated_bytes = schedule_bytes
22446                .saturating_add((producer_count as u64).saturating_mul(96))
22447                .saturating_add(1024);
22448            let native_allowed = self
22449                .active_resource_ledger
22450                .as_ref()
22451                .and_then(ResourceLedger::disk_scratch_policy)
22452                == Some(crate::engine::DiskScratchPolicy::NativeTemporary)
22453                && !cfg!(target_arch = "wasm32");
22454
22455            let cached_schedule_bytes = schedule_bytes
22456                .saturating_add(
22457                    cached_topology
22458                        .map_or(0, |topology| topology.stats.relationships as u64)
22459                        .saturating_mul(160),
22460                )
22461                .saturating_add(1024);
22462            let use_cached_schedule = cached_topology.is_some()
22463                && self.can_reserve_topology_scratch(cached_schedule_bytes);
22464            let (selected_strategy, scratch) = if use_cached_schedule {
22465                (None, cached_schedule_bytes)
22466            } else if self.can_reserve_topology_scratch(paged_bytes) {
22467                (
22468                    Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed),
22469                    paged_bytes,
22470                )
22471            } else if self.can_reserve_topology_scratch(runs_bytes) {
22472                (
22473                    Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns),
22474                    runs_bytes,
22475                )
22476            } else if native_allowed && self.can_reserve_topology_scratch(native_bytes) {
22477                (
22478                    Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
22479                    native_bytes,
22480                )
22481            } else if self.can_reserve_topology_scratch(repeated_bytes) {
22482                (
22483                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22484                    repeated_bytes,
22485                )
22486            } else {
22487                if let Some(ledger) = self.active_resource_ledger.as_mut() {
22488                    ledger
22489                        .reserve_schedule_discovery(repeated_bytes)
22490                        .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22491                }
22492                unreachable!("failed scratch reservation must return a typed error")
22493            };
22494            if let Some(ledger) = self.active_resource_ledger.as_mut() {
22495                ledger
22496                    .reserve_schedule_discovery(scratch)
22497                    .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22498            }
22499            let request_id = self
22500                .active_evaluation_resource_request
22501                .as_ref()
22502                .map_or(0, |stats| stats.request_id);
22503            let build_result = (|| -> Result<_, ExcelError> {
22504                let (schedule, actual_strategy, passes, native_disk_bytes) =
22505                    match (cached_topology, selected_strategy) {
22506                        (Some(topology), None) => (
22507                            schedule_dirty_work(work, producer_results, topology, 256),
22508                            None,
22509                            0,
22510                            0,
22511                        ),
22512                        (_, Some(FormulaPlaneTopologyStrategy::ExactPagedIndexed)) => {
22513                            let (schedule, passes) = schedule_dirty_work_paged_hybrid(
22514                                work,
22515                                producer_results,
22516                                consumer_reads,
22517                                partial_topology.filter(|_| reuse_partial_topology),
22518                                256,
22519                                |units| {
22520                                    Self::resource_loop_checkpoint(
22521                                        &mut self.active_resource_ledger,
22522                                        units,
22523                                    )
22524                                },
22525                            )?;
22526                            (schedule, selected_strategy, passes.max(1), 0)
22527                        }
22528                        (_, Some(FormulaPlaneTopologyStrategy::ExactInMemoryRuns)) => {
22529                            let (schedule, passes) = schedule_dirty_work_in_memory_runs(
22530                                work,
22531                                producer_results,
22532                                consumer_reads,
22533                                256,
22534                                |units| {
22535                                    Self::resource_loop_checkpoint(
22536                                        &mut self.active_resource_ledger,
22537                                        units,
22538                                    )
22539                                },
22540                            )?;
22541                            (schedule, selected_strategy, passes.max(1), 0)
22542                        }
22543                        #[cfg(not(target_arch = "wasm32"))]
22544                        (_, Some(FormulaPlaneTopologyStrategy::ExactNativeScratch)) => {
22545                            let native = NativeTopologyScratch::create(request_id);
22546                            if let Ok(mut native) = native {
22547                                match schedule_dirty_work_native(
22548                                    work.clone(),
22549                                    producer_results,
22550                                    consumer_reads,
22551                                    256,
22552                                    native.file.as_mut().expect("native scratch file"),
22553                                    |units| {
22554                                        Self::resource_loop_checkpoint(
22555                                            &mut self.active_resource_ledger,
22556                                            units,
22557                                        )
22558                                    },
22559                                ) {
22560                                    Ok((schedule, passes, topology_bytes)) => (
22561                                        schedule,
22562                                        Some(FormulaPlaneTopologyStrategy::ExactNativeScratch),
22563                                        passes.max(1),
22564                                        topology_bytes,
22565                                    ),
22566                                    Err(NativeExactScheduleError::Work(error)) => {
22567                                        return Err(error);
22568                                    }
22569                                    Err(NativeExactScheduleError::Io(_)) => {
22570                                        let (schedule, passes) =
22571                                            schedule_dirty_work_repeated_passes(
22572                                                work,
22573                                                producer_results,
22574                                                consumer_reads,
22575                                                256,
22576                                                |units| {
22577                                                    Self::resource_loop_checkpoint(
22578                                                        &mut self.active_resource_ledger,
22579                                                        units,
22580                                                    )
22581                                                },
22582                                            )?;
22583                                        (
22584                                            schedule,
22585                                            Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22586                                            passes.max(1),
22587                                            0,
22588                                        )
22589                                    }
22590                                }
22591                            } else {
22592                                let (schedule, passes) = schedule_dirty_work_repeated_passes(
22593                                    work,
22594                                    producer_results,
22595                                    consumer_reads,
22596                                    256,
22597                                    |units| {
22598                                        Self::resource_loop_checkpoint(
22599                                            &mut self.active_resource_ledger,
22600                                            units,
22601                                        )
22602                                    },
22603                                )?;
22604                                (
22605                                    schedule,
22606                                    Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses),
22607                                    passes.max(1),
22608                                    0,
22609                                )
22610                            }
22611                        }
22612                        (_, Some(FormulaPlaneTopologyStrategy::ExactRepeatedPasses)) => {
22613                            let (schedule, passes) = schedule_dirty_work_repeated_passes(
22614                                work,
22615                                producer_results,
22616                                consumer_reads,
22617                                256,
22618                                |units| {
22619                                    Self::resource_loop_checkpoint(
22620                                        &mut self.active_resource_ledger,
22621                                        units,
22622                                    )
22623                                },
22624                            )?;
22625                            (schedule, selected_strategy, passes.max(1), 0)
22626                        }
22627                        _ => unreachable!(
22628                            "topology cache and exact strategy selection are exhaustive"
22629                        ),
22630                    };
22631                Ok((
22632                    schedule,
22633                    actual_strategy,
22634                    passes,
22635                    native_disk_bytes,
22636                    span_refs_by_id.clone(),
22637                ))
22638            })();
22639            let (mut schedule, exact_strategy, pass_count, native_disk_bytes, result_span_refs) =
22640                match build_result {
22641                    Ok(output) => output,
22642                    Err(error) => {
22643                        if let Some(ledger) = self.active_resource_ledger.as_mut() {
22644                            let _ = ledger.release_scratch(scratch);
22645                        }
22646                        return Err(error);
22647                    }
22648                };
22649            // Keep the selected strategy's scratch reservation live while the
22650            // bounded symbol merge allocates its replacement edge map.
22651            schedule = self.include_non_grid_legacy_work(
22652                schedule,
22653                &non_grid_legacy_vertices,
22654                &non_grid_legacy_fallbacks,
22655                producer_results,
22656                consumer_reads,
22657                key.max_candidates,
22658                key.max_edges,
22659            );
22660            self.active_resource_ledger
22661                .as_mut()
22662                .map_or(Ok(()), |ledger| ledger.release_scratch(scratch))
22663                .map_err(crate::engine::ResourceLedgerError::into_excel_error)?;
22664            if let Some(strategy) = exact_strategy
22665                && let Some(stats) = self.active_evaluation_resource_request.as_mut()
22666            {
22667                if strategy.severity() > stats.topology.strategy.severity() {
22668                    stats.topology.strategy = strategy;
22669                }
22670                stats.topology.exact_pass_count = pass_count;
22671                stats.topology.native_topology_disk_bytes = stats
22672                    .topology
22673                    .native_topology_disk_bytes
22674                    .saturating_add(native_disk_bytes);
22675            }
22676            Ok((
22677                schedule,
22678                result_span_refs,
22679                plane_epoch,
22680                scheduled_legacy_vertices,
22681                owned_dirty_events,
22682                island_plan,
22683            ))
22684        })();
22685        let scratch_release_result = self
22686            .active_resource_ledger
22687            .as_mut()
22688            .map_or(Ok(()), |ledger| {
22689                ledger
22690                    .release_scratch(index_scratch_reserved.saturating_add(demand_scratch_reserved))
22691            })
22692            .map_err(crate::engine::ResourceLedgerError::into_excel_error);
22693        match schedule_result {
22694            Ok(output) => {
22695                scratch_release_result?;
22696                Ok(output)
22697            }
22698            Err(error) => {
22699                let _ = scratch_release_result;
22700                Err(error)
22701            }
22702        }
22703    }
22704}
22705
22706impl<R> Engine<R>
22707where
22708    R: EvaluationContext,
22709{
22710    /// The [`Region`] a compressed range dependency covers.
22711    ///
22712    /// `context_sheet` is the sheet the dependency's formula lives on, which is
22713    /// what a `Current` locator means. It must be supplied by the caller: this
22714    /// method has no way to recover it, and substituting the workbook default
22715    /// sheet pointed the region at an unrelated sheet (issue #110). `None` means
22716    /// "no region pattern"; callers treat that as an incomplete summary and fall
22717    /// back to the conservative global path.
22718    fn shared_range_to_region_pattern(
22719        &self,
22720        range: &crate::reference::SharedRangeRef<'static>,
22721        context_sheet: SheetId,
22722    ) -> Result<Option<Region>, ExcelError> {
22723        assert!(
22724            matches!(range.sheet, crate::reference::SharedSheetLocator::Id(_)),
22725            "PROBE: non-Id locator reached shared_range_to_region_pattern: {:?}",
22726            range.sheet
22727        );
22728        let Ok(sheet_id) = self.resolve_sheet_locator(&range.sheet, context_sheet) else {
22729            // Unresolvable sheet name: no trustworthy region.
22730            return Ok(None);
22731        };
22732        match (
22733            range.start_row,
22734            range.end_row,
22735            range.start_col,
22736            range.end_col,
22737        ) {
22738            (Some(sr), Some(er), Some(sc), Some(ec)) => Ok(Some(Region::rect(
22739                sheet_id, sr.index, er.index, sc.index, ec.index,
22740            ))),
22741            (None, None, Some(sc), Some(ec)) if sc.index == ec.index => {
22742                Ok(Some(Region::whole_col(sheet_id, sc.index)))
22743            }
22744            (Some(sr), Some(er), None, None) if sr.index == er.index => {
22745                Ok(Some(Region::whole_row(sheet_id, sr.index)))
22746            }
22747            _ => Ok(None),
22748        }
22749    }
22750
22751    /// Evaluate all dirty/volatile vertices
22752    pub fn evaluate_all(&mut self) -> Result<EvalResult, ExcelError> {
22753        self.observe_evaluation_resource_request(EvaluationRequestKind::Full, |engine| {
22754            engine.evaluate_all_unobserved()
22755        })
22756    }
22757
22758    fn evaluate_all_unobserved(&mut self) -> Result<EvalResult, ExcelError> {
22759        debug_assert!(
22760            !self.graph.deferred_dirty_active(),
22761            "deferred-dirty scope leaked into evaluate_all: a begin_deferred_dirty \
22762             was not balanced by end_deferred_dirty"
22763        );
22764        self.observe_function_semantic_epoch()?;
22765        self.lookup_index_cache.reset_counters();
22766        let _source_cache = self.source_cache_session();
22767        self.validate_deterministic_mode()?;
22768        if self.config.defer_graph_building {
22769            // Build graph for all staged formulas before evaluating
22770            self.build_graph_all()?;
22771        }
22772        self.evaluate_all_coordinator()
22773    }
22774
22775    /// Central FormulaPlane-aware coordinator for `evaluate_all`. In
22776    /// `AuthoritativeExperimental` mode every call enters the FormulaPlane
22777    /// coordinator; the coordinator itself composes with private legacy
22778    /// primitives for legacy-only work.
22779    fn evaluate_all_coordinator(&mut self) -> Result<EvalResult, ExcelError> {
22780        self.transition_off_mode_spans_to_legacy()?;
22781        self.begin_evaluation_request();
22782        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental {
22783            return self.evaluate_authoritative_formula_plane_all();
22784        }
22785        self.evaluate_all_legacy_impl()
22786    }
22787
22788    /// Walk a schedule's units in condensation order: stamp each cyclic SCC
22789    /// at its position and evaluate each layer (parallel when enabled).
22790    ///
22791    /// Returns `(computed_vertices, cycle_count)` where `cycle_count` is the
22792    /// number of Cycle units walked (the former `schedule.cycles.len()`).
22793    fn legacy_pass_run_units(
22794        &mut self,
22795        schedule: &crate::engine::scheduler::Schedule,
22796    ) -> Result<(usize, usize), ExcelError> {
22797        let mut computed_vertices = 0;
22798        let mut cycle_count = 0;
22799        for &unit in &schedule.units {
22800            match unit {
22801                ScheduleUnit::Cycle(i) => {
22802                    if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0 {
22803                        cycle_count += 1;
22804                    }
22805                }
22806                ScheduleUnit::Layer(i) => {
22807                    let layer = schedule.unit_layer(i);
22808                    if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22809                        computed_vertices += self.evaluate_layer_parallel(layer)?;
22810                    } else {
22811                        computed_vertices += self.evaluate_layer_sequential(layer)?;
22812                    }
22813                }
22814            }
22815        }
22816        Ok((computed_vertices, cycle_count))
22817    }
22818
22819    /// Execute a proven span-disconnected legacy island without beginning or
22820    /// finalising an evaluation request. The FormulaPlane coordinator retains
22821    /// sole ownership of the dirty lease, volatile redirty, acknowledgement,
22822    /// clock sample, and recalc-epoch advance.
22823    fn evaluate_legacy_island_non_finalizing(
22824        &mut self,
22825        vertices: &[VertexId],
22826        mut delta: Option<&mut DeltaCollector>,
22827    ) -> Result<(usize, usize), ExcelError> {
22828        if vertices.is_empty() {
22829            return Ok((0, 0));
22830        }
22831        let (schedule, _, _) = self.create_evaluation_schedule(vertices)?;
22832        let mut computed = 0usize;
22833        let mut cycles = 0usize;
22834        for &unit in &schedule.units {
22835            self.cancellation_checkpoint("Evaluation cancelled during legacy island")?;
22836            match unit {
22837                ScheduleUnit::Cycle(index) => {
22838                    if self.handle_cycle_unit(
22839                        schedule.unit_cycle(index),
22840                        delta.as_deref_mut(),
22841                        None,
22842                        None,
22843                    )? > 0
22844                    {
22845                        cycles = cycles.saturating_add(1);
22846                    }
22847                }
22848                ScheduleUnit::Layer(index) => {
22849                    let layer = schedule.unit_layer(index);
22850                    let evaluated = if let Some(delta) = delta.as_deref_mut() {
22851                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22852                            self.evaluate_layer_parallel_with_delta(layer, delta)?
22853                        } else {
22854                            self.evaluate_layer_sequential_with_delta(layer, delta)?
22855                        }
22856                    } else if self.thread_pool.is_some() && layer.vertices.len() > 1 {
22857                        self.evaluate_layer_parallel(layer)?
22858                    } else {
22859                        self.evaluate_layer_sequential(layer)?
22860                    };
22861                    computed = computed.saturating_add(evaluated);
22862                }
22863            }
22864        }
22865        self.graph.clear_dirty_flags(vertices);
22866        Ok((computed, cycles))
22867    }
22868
22869    /// Legacy `evaluate_all` body, reachable from the FormulaPlane coordinator
22870    /// when no active spans exist or FormulaPlane authority is not in
22871    /// `AuthoritativeExperimental` mode. This is now an internal primitive; it
22872    /// must not be invoked directly from public APIs.
22873    ///
22874    /// Does NOT call `begin_evaluation_request` (cycle-telemetry reset +
22875    /// per-recalc clock sample): the FormulaPlane coordinator composes this
22876    /// primitive *after* `evaluate_legacy_cycle_prepass` may have accumulated
22877    /// counts (G8 demotion path), and both sub-passes belong to ONE request /
22878    /// one clock sample; request begin happens at the public entry points /
22879    /// coordinators instead.
22880    fn evaluate_all_legacy_impl(&mut self) -> Result<EvalResult, ExcelError> {
22881        self.reset_virtual_dep_telemetry_if_disabled();
22882        #[cfg(feature = "tracing")]
22883        let _span_eval = tracing::info_span!("evaluate_all").entered();
22884        let start = crate::instant::FzInstant::now();
22885        let mut computed_vertices = 0;
22886        let mut cycle_errors = 0;
22887        let mut replan_iterations = 0;
22888        const MAX_REPLAN: usize = 5;
22889        let mut telemetry = self
22890            .config
22891            .enable_virtual_dep_telemetry
22892            .then(|| self.start_virtual_dep_telemetry());
22893
22894        loop {
22895            let to_evaluate = self.graph.get_evaluation_vertices();
22896            if to_evaluate.is_empty() {
22897                if let Some(t) = telemetry.as_mut()
22898                    && t.bailout_reason.is_none()
22899                {
22900                    t.bailout_reason = Some("no_work");
22901                }
22902                break;
22903            }
22904
22905            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
22906            if let Some(t) = telemetry.as_mut() {
22907                Self::accumulate_schedule_meta(t, &meta);
22908            }
22909
22910            let (pass_computed, pass_cycles) = self.legacy_pass_run_units(&schedule)?;
22911            computed_vertices += pass_computed;
22912            cycle_errors += pass_cycles;
22913
22914            // Check if dynamic dependencies changed
22915            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
22916            if let Some(t) = telemetry.as_mut() {
22917                t.changed_vdeps_total += changed_vertices.len();
22918            }
22919
22920            self.resource_checkpoint(0)?;
22921            self.graph.clear_dirty_flags(&to_evaluate);
22922            for v in &changed_vertices {
22923                self.graph.set_dirty(*v, true);
22924            }
22925
22926            if changed_vertices.is_empty() {
22927                if let Some(t) = telemetry.as_mut() {
22928                    t.bailout_reason = Some("converged");
22929                }
22930                break;
22931            }
22932            if replan_iterations >= MAX_REPLAN {
22933                if let Some(mut t) = telemetry.take() {
22934                    t.bailout_reason = Some("max_replan");
22935                    t.replan_iterations = replan_iterations;
22936                    self.last_virtual_dep_telemetry = t;
22937                }
22938                return Err(
22939                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
22940                );
22941            }
22942
22943            replan_iterations += 1;
22944        }
22945
22946        if let Some(mut t) = telemetry {
22947            t.replan_iterations = replan_iterations;
22948            self.last_virtual_dep_telemetry = t;
22949        }
22950
22951        // Re-dirty volatile vertices for the next evaluation cycle
22952        self.redirty_for_next_recalc();
22953
22954        // Advance recalc epoch after a full evaluation pass finishes
22955        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
22956
22957        Ok(EvalResult {
22958            computed_vertices,
22959            cycle_errors,
22960            elapsed: start.elapsed(),
22961        })
22962    }
22963
22964    pub fn evaluate_all_with_target_delta(
22965        &mut self,
22966    ) -> Result<(EvalResult, crate::engine::TargetEvalDelta), ExcelError> {
22967        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
22968            engine.observe_function_semantic_epoch()?;
22969            let mut collector = DeltaCollector::new(DeltaMode::Cells);
22970            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
22971            Ok((result, collector.finish_target()))
22972        })
22973    }
22974
22975    pub fn evaluate_all_with_delta(&mut self) -> Result<(EvalResult, EvalDelta), ExcelError> {
22976        self.evaluate_all_with_delta_policy(EvalDeltaCompatibilityPolicy::Unlimited)
22977    }
22978
22979    pub fn evaluate_all_with_delta_policy(
22980        &mut self,
22981        policy: EvalDeltaCompatibilityPolicy,
22982    ) -> Result<(EvalResult, EvalDelta), ExcelError> {
22983        self.observe_evaluation_resource_request(EvaluationRequestKind::FullWithDelta, |engine| {
22984            engine.observe_function_semantic_epoch()?;
22985            let mut collector = DeltaCollector::new(DeltaMode::Cells);
22986            let result = engine.evaluate_all_with_delta_collector(&mut collector)?;
22987            Ok((result, collector.finish_with_policy(policy)?))
22988        })
22989    }
22990
22991    fn evaluate_all_with_delta_collector(
22992        &mut self,
22993        delta: &mut DeltaCollector,
22994    ) -> Result<EvalResult, ExcelError> {
22995        let _source_cache = self.source_cache_session();
22996        if self.config.defer_graph_building {
22997            self.build_graph_all()?;
22998        }
22999        self.transition_off_mode_spans_to_legacy()?;
23000        self.begin_evaluation_request();
23001        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
23002            && self.graph.formula_authority().active_span_count() > 0
23003        {
23004            return self.evaluate_authoritative_formula_plane(None, Some(delta));
23005        }
23006        self.reset_virtual_dep_telemetry_if_disabled();
23007        #[cfg(feature = "tracing")]
23008        let _span_eval = tracing::info_span!("evaluate_all_with_delta").entered();
23009        let start = crate::instant::FzInstant::now();
23010        let mut computed_vertices = 0;
23011        let mut cycle_errors = 0;
23012
23013        let mut replan_iterations = 0;
23014        const MAX_REPLAN: usize = 5;
23015        let mut telemetry = self
23016            .config
23017            .enable_virtual_dep_telemetry
23018            .then(|| self.start_virtual_dep_telemetry());
23019
23020        loop {
23021            let to_evaluate = self.graph.get_evaluation_vertices();
23022            if to_evaluate.is_empty() {
23023                if let Some(t) = telemetry.as_mut()
23024                    && t.bailout_reason.is_none()
23025                {
23026                    t.bailout_reason = Some("no_work");
23027                }
23028                break;
23029            }
23030
23031            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
23032            if let Some(t) = telemetry.as_mut() {
23033                Self::accumulate_schedule_meta(t, &meta);
23034            }
23035
23036            for &unit in &schedule.units {
23037                match unit {
23038                    ScheduleUnit::Cycle(i) => {
23039                        if self.handle_cycle_unit(
23040                            schedule.unit_cycle(i),
23041                            Some(delta),
23042                            None,
23043                            None,
23044                        )? > 0
23045                        {
23046                            cycle_errors += 1;
23047                        }
23048                    }
23049                    ScheduleUnit::Layer(i) => {
23050                        let layer = schedule.unit_layer(i);
23051                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
23052                            computed_vertices +=
23053                                self.evaluate_layer_parallel_with_delta(layer, delta)?;
23054                        } else {
23055                            computed_vertices +=
23056                                self.evaluate_layer_sequential_with_delta(layer, delta)?;
23057                        }
23058                    }
23059                }
23060            }
23061
23062            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
23063            if let Some(t) = telemetry.as_mut() {
23064                t.changed_vdeps_total += changed_vertices.len();
23065            }
23066            self.resource_checkpoint(0)?;
23067            self.graph.clear_dirty_flags(&to_evaluate);
23068            for v in &changed_vertices {
23069                self.graph.set_dirty(*v, true);
23070            }
23071
23072            if changed_vertices.is_empty() {
23073                if let Some(t) = telemetry.as_mut() {
23074                    t.bailout_reason = Some("converged");
23075                }
23076                break;
23077            }
23078            if replan_iterations >= MAX_REPLAN {
23079                if let Some(mut t) = telemetry.take() {
23080                    t.bailout_reason = Some("max_replan");
23081                    t.replan_iterations = replan_iterations;
23082                    self.last_virtual_dep_telemetry = t;
23083                }
23084                return Err(
23085                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
23086                );
23087            }
23088            replan_iterations += 1;
23089        }
23090
23091        if let Some(mut t) = telemetry {
23092            t.replan_iterations = replan_iterations;
23093            self.last_virtual_dep_telemetry = t;
23094        }
23095
23096        self.redirty_for_next_recalc();
23097        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
23098
23099        Ok(EvalResult {
23100            computed_vertices,
23101            cycle_errors,
23102            elapsed: start.elapsed(),
23103        })
23104    }
23105
23106    /// Convenience: demand-driven evaluation of a single cell by sheet name and row/col.
23107    ///
23108    /// This will evaluate only the minimal set of dirty / volatile precedents required
23109    /// to bring the target cell up-to-date (as if a user asked for that single value),
23110    /// rather than scheduling a full workbook recalc. If the cell is already clean and
23111    /// non-volatile, no vertices will be recomputed.
23112    ///
23113    /// Returns the (possibly newly computed) value stored for the cell afterwards.
23114    /// Empty cells return None. Errors are surfaced via the Result type.
23115    pub fn evaluate_cell(
23116        &mut self,
23117        sheet: &str,
23118        row: u32,
23119        col: u32,
23120    ) -> Result<Option<LiteralValue>, ExcelError> {
23121        self.observe_evaluation_resource_request(EvaluationRequestKind::Cell, |engine| {
23122            engine.evaluate_cell_unobserved(sheet, row, col)
23123        })
23124    }
23125
23126    fn evaluate_cell_unobserved(
23127        &mut self,
23128        sheet: &str,
23129        row: u32,
23130        col: u32,
23131    ) -> Result<Option<LiteralValue>, ExcelError> {
23132        if row == 0 || col == 0 {
23133            return Err(ExcelError::new(ExcelErrorKind::Ref)
23134                .with_message("Row and column must be >= 1".to_string()));
23135        }
23136
23137        let result = self.evaluate_cells(&[(sheet, row, col)])?;
23138
23139        match result.len() {
23140            0 => Ok(None),
23141            1 => {
23142                let v = result.into_iter().next().unwrap();
23143                Ok(v)
23144            }
23145            _ => unreachable!("evaluate_cells returned unexpected length"),
23146        }
23147    }
23148
23149    /// Convenience: demand-driven evaluation of multiple cells; accepts a slice of
23150    /// (sheet, row, col) triples. The union of required dirty / volatile precedents
23151    /// is computed once and evaluated, which is typically faster than calling
23152    /// `evaluate_cell` repeatedly for a related set of targets.
23153    ///
23154    /// Returns the resulting values for each requested target in the same order.
23155    pub fn evaluate_cells(
23156        &mut self,
23157        targets: &[(&str, u32, u32)],
23158    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23159        self.observe_evaluation_resource_request(EvaluationRequestKind::Cells, |engine| {
23160            engine.evaluate_cells_unobserved(targets)
23161        })
23162    }
23163
23164    fn evaluate_cells_unobserved(
23165        &mut self,
23166        targets: &[(&str, u32, u32)],
23167    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23168        self.observe_function_semantic_epoch()?;
23169        debug_assert!(
23170            !self.graph.deferred_dirty_active(),
23171            "deferred-dirty scope leaked into evaluate_cells: a begin_deferred_dirty \
23172             was not balanced by end_deferred_dirty"
23173        );
23174        self.validate_deterministic_mode()?;
23175        if targets.is_empty() {
23176            return Ok(Vec::new());
23177        }
23178        let typed_targets = self.legacy_coordinate_targets(targets);
23179        self.evaluate_mixed_targets(&typed_targets, None)?;
23180        Ok(targets
23181            .iter()
23182            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
23183            .collect())
23184    }
23185
23186    pub fn evaluate_cells_cancellable(
23187        &mut self,
23188        targets: &[(&str, u32, u32)],
23189        cancel: crate::engine::CancelToken,
23190    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23191        self.observe_evaluation_resource_request(
23192            EvaluationRequestKind::CellsCancellable,
23193            |engine| {
23194                engine.observe_function_semantic_epoch()?;
23195                engine.active_cancel_flag = Some(cancel.clone());
23196                let res = engine.evaluate_cells_cancellable_impl(targets, cancel.as_flag());
23197                engine.active_cancel_flag = None;
23198                res
23199            },
23200        )
23201    }
23202
23203    fn evaluate_cells_cancellable_impl(
23204        &mut self,
23205        targets: &[(&str, u32, u32)],
23206        cancel_flag: &AtomicBool,
23207    ) -> Result<Vec<Option<LiteralValue>>, ExcelError> {
23208        self.validate_deterministic_mode()?;
23209        if targets.is_empty() {
23210            return Ok(Vec::new());
23211        }
23212        if cancel_flag.load(Ordering::Relaxed) {
23213            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23214                .with_message("Evaluation cancelled before target preparation"));
23215        }
23216        let typed_targets = self.legacy_coordinate_targets(targets);
23217        self.evaluate_mixed_targets(&typed_targets, None)?;
23218        Ok(targets
23219            .iter()
23220            .map(|(sheet, row, col)| self.get_cell_value(sheet, *row, *col))
23221            .collect())
23222    }
23223
23224    pub fn evaluate_cells_with_target_delta(
23225        &mut self,
23226        targets: &[(&str, u32, u32)],
23227    ) -> Result<(Vec<Option<LiteralValue>>, crate::engine::TargetEvalDelta), ExcelError> {
23228        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
23229            engine.observe_function_semantic_epoch()?;
23230            engine.validate_deterministic_mode()?;
23231            if targets.is_empty() {
23232                return Ok((Vec::new(), crate::engine::TargetEvalDelta::default()));
23233            }
23234            let mut collector = DeltaCollector::new(DeltaMode::Cells);
23235            engine.evaluate_until_with_delta_collector(targets, &mut collector)?;
23236            let values = targets
23237                .iter()
23238                .map(|(sheet, row, col)| engine.get_cell_value(sheet, *row, *col))
23239                .collect();
23240            Ok((values, collector.finish_target()))
23241        })
23242    }
23243
23244    pub fn evaluate_cells_with_delta(
23245        &mut self,
23246        targets: &[(&str, u32, u32)],
23247    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23248        self.evaluate_cells_with_delta_policy(targets, EvalDeltaCompatibilityPolicy::Unlimited)
23249    }
23250
23251    pub fn evaluate_cells_with_delta_policy(
23252        &mut self,
23253        targets: &[(&str, u32, u32)],
23254        policy: EvalDeltaCompatibilityPolicy,
23255    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23256        self.observe_evaluation_resource_request(EvaluationRequestKind::CellsWithDelta, |engine| {
23257            engine.evaluate_cells_with_delta_unobserved(targets, policy)
23258        })
23259    }
23260
23261    fn evaluate_cells_with_delta_unobserved(
23262        &mut self,
23263        targets: &[(&str, u32, u32)],
23264        policy: EvalDeltaCompatibilityPolicy,
23265    ) -> Result<(Vec<Option<LiteralValue>>, EvalDelta), ExcelError> {
23266        self.observe_function_semantic_epoch()?;
23267        self.validate_deterministic_mode()?;
23268        if targets.is_empty() {
23269            return Ok((Vec::new(), EvalDelta::default()));
23270        }
23271        let mut collector = DeltaCollector::new(DeltaMode::Cells);
23272        self.evaluate_until_with_delta_collector(targets, &mut collector)?;
23273        let values = targets
23274            .iter()
23275            .map(|(s, r, c)| self.get_cell_value(s, *r, *c))
23276            .collect();
23277        Ok((values, collector.finish_with_policy(policy)?))
23278    }
23279
23280    /// Get the evaluation plan for target cells without actually evaluating them
23281    pub fn get_eval_plan(&self, targets: &[(&str, u32, u32)]) -> Result<EvalPlan, ExcelError> {
23282        if targets.is_empty() {
23283            return Ok(EvalPlan {
23284                total_vertices_to_evaluate: 0,
23285                layers: Vec::new(),
23286                cycles_detected: 0,
23287                dirty_count: 0,
23288                volatile_count: 0,
23289                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23290                estimated_parallel_layers: 0,
23291                target_cells: Vec::new(),
23292            });
23293        }
23294        if self.config.defer_graph_building && self.has_staged_formulas() {
23295            return Err(ExcelError::new(ExcelErrorKind::Value).with_message(
23296                "Evaluation plan requested with deferred graph; build first or call evaluate_*",
23297            ));
23298        }
23299
23300        // Convert targets to A1 notation for consistency
23301        let addresses: Vec<String> = targets
23302            .iter()
23303            .map(|(s, r, c)| format!("{}!{}{}", s, Self::col_to_letters(*c), r))
23304            .collect();
23305
23306        // Parse target cell addresses
23307        let mut target_addrs = Vec::new();
23308        for (sheet, row, col) in targets {
23309            if let Some(sheet_id) = self.graph.sheet_id(sheet) {
23310                let coord = Coord::from_excel(*row, *col, true, true);
23311                target_addrs.push(CellRef::new(sheet_id, coord));
23312            }
23313        }
23314
23315        // Find vertex IDs for targets
23316        let mut target_vertex_ids = Vec::new();
23317        for addr in &target_addrs {
23318            if let Some(vertex_id) = self.graph.get_vertex_id_for_address(addr) {
23319                target_vertex_ids.push(*vertex_id);
23320            }
23321        }
23322
23323        if target_vertex_ids.is_empty() {
23324            return Ok(EvalPlan {
23325                total_vertices_to_evaluate: 0,
23326                layers: Vec::new(),
23327                cycles_detected: 0,
23328                dirty_count: 0,
23329                volatile_count: 0,
23330                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23331                estimated_parallel_layers: 0,
23332                target_cells: addresses,
23333            });
23334        }
23335
23336        // Build demand subgraph with virtual edges (same as evaluate_until)
23337        let (precedents_to_eval, vdeps) = self.build_demand_subgraph(&target_vertex_ids);
23338
23339        if precedents_to_eval.is_empty() {
23340            return Ok(EvalPlan {
23341                total_vertices_to_evaluate: 0,
23342                layers: Vec::new(),
23343                cycles_detected: 0,
23344                dirty_count: 0,
23345                volatile_count: 0,
23346                parallel_enabled: self.config.enable_parallel && self.thread_pool.is_some(),
23347                estimated_parallel_layers: 0,
23348                target_cells: addresses,
23349            });
23350        }
23351
23352        // Count dirty and volatile vertices
23353        let mut dirty_count = 0;
23354        let mut volatile_count = 0;
23355        for &vertex_id in &precedents_to_eval {
23356            if self.graph.is_dirty(vertex_id) {
23357                dirty_count += 1;
23358            }
23359            if self.graph.is_volatile(vertex_id) {
23360                volatile_count += 1;
23361            }
23362        }
23363
23364        // Create schedule for the minimal subgraph honoring virtual edges
23365        let scheduler = Scheduler::new(&self.graph);
23366        let schedule = scheduler.create_schedule_with_virtual(&precedents_to_eval, &vdeps)?;
23367
23368        // Build layer information
23369        let mut layers = Vec::new();
23370        let mut estimated_parallel_layers = 0;
23371        let parallel_enabled = self.config.enable_parallel && self.thread_pool.is_some();
23372
23373        for layer in &schedule.layers {
23374            let parallel_eligible = parallel_enabled && layer.vertices.len() > 1;
23375            if parallel_eligible {
23376                estimated_parallel_layers += 1;
23377            }
23378
23379            // Get sample cell addresses (up to 5)
23380            let sample_cells: Vec<String> = layer
23381                .vertices
23382                .iter()
23383                .take(5)
23384                .filter_map(|&vertex_id| {
23385                    self.graph
23386                        .get_cell_ref_for_vertex(vertex_id)
23387                        .map(|cell_ref| {
23388                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
23389                            format!(
23390                                "{}!{}{}",
23391                                sheet_name,
23392                                Self::col_to_letters(cell_ref.coord.col().saturating_add(1)),
23393                                cell_ref.coord.row() + 1
23394                            )
23395                        })
23396                })
23397                .collect();
23398
23399            layers.push(LayerInfo {
23400                vertex_count: layer.vertices.len(),
23401                parallel_eligible,
23402                sample_cells,
23403            });
23404        }
23405
23406        Ok(EvalPlan {
23407            total_vertices_to_evaluate: precedents_to_eval.len(),
23408            layers,
23409            cycles_detected: schedule.cycles.len(),
23410            dirty_count,
23411            volatile_count,
23412            parallel_enabled,
23413            estimated_parallel_layers,
23414            target_cells: addresses,
23415        })
23416    }
23417    /// Helper to create a schedule, integrating virtual dependencies automatically.
23418    fn create_evaluation_schedule(
23419        &mut self,
23420        to_evaluate: &[VertexId],
23421    ) -> Result<EvaluationScheduleBuildOutput, ExcelError> {
23422        #[cfg(any(test, feature = "benchmark_internal"))]
23423        {
23424            self.recalc_reuse_probe.get_mut().unwrap().schedule_requests += 1;
23425        }
23426        // Fold pending edge deltas once per schedule build so traversal uses
23427        // the zero-allocation CSR slices (#125).
23428        self.graph.flush_pending_edge_deltas();
23429        if self.can_use_static_schedule_cache(to_evaluate) {
23430            if let Some(cached) = self.cached_static_schedule.as_ref()
23431                && cached.topology_epoch == self.topology_epoch
23432                && cached.candidate_vertices.as_slice() == to_evaluate
23433            {
23434                let meta = ScheduleBuildMeta {
23435                    candidate_vertices: to_evaluate.len(),
23436                    vdeps_vertices: 0,
23437                    vdeps_edges: 0,
23438                    builder_elapsed_ms: 0,
23439                    used_virtual_schedule: false,
23440                    schedule_cache_hit: true,
23441                    schedule_cache_eligible: true,
23442                };
23443                #[cfg(any(test, feature = "benchmark_internal"))]
23444                {
23445                    let mut probe = self.recalc_reuse_probe.lock().unwrap();
23446                    probe.schedule_cache_hits += 1;
23447                    probe.schedule_shared_handles += 1;
23448                }
23449                return Ok((
23450                    EvaluationSchedule::Shared(Arc::clone(&cached.schedule)),
23451                    FxHashMap::default(),
23452                    meta,
23453                ));
23454            }
23455
23456            let (schedule, vdeps, mut meta) =
23457                self.create_evaluation_schedule_uncached(to_evaluate)?;
23458            meta.schedule_cache_hit = false;
23459            meta.schedule_cache_eligible = true;
23460            #[cfg(any(test, feature = "benchmark_internal"))]
23461            {
23462                self.recalc_reuse_probe
23463                    .get_mut()
23464                    .unwrap()
23465                    .schedule_cache_misses += 1;
23466            }
23467            let schedule = if vdeps.is_empty() {
23468                // Clone previously discarded builder spare capacity. Keep that compact
23469                // retained payload while sharing it with the current request.
23470                let mut schedule = schedule;
23471                schedule.units.shrink_to_fit();
23472                for layer in &mut schedule.layers {
23473                    layer.vertices.shrink_to_fit();
23474                }
23475                schedule.layers.shrink_to_fit();
23476                for cycle in &mut schedule.cycles {
23477                    cycle.shrink_to_fit();
23478                }
23479                schedule.cycles.shrink_to_fit();
23480                let schedule = Arc::new(schedule);
23481                #[cfg(any(test, feature = "benchmark_internal"))]
23482                {
23483                    self.recalc_reuse_probe
23484                        .get_mut()
23485                        .unwrap()
23486                        .schedule_shared_handles += 1;
23487                }
23488                self.cached_static_schedule = Some(CachedScheduleEntry {
23489                    topology_epoch: self.topology_epoch,
23490                    candidate_vertices: to_evaluate.to_vec(),
23491                    schedule: Arc::clone(&schedule),
23492                });
23493                EvaluationSchedule::Shared(schedule)
23494            } else {
23495                EvaluationSchedule::Owned(schedule)
23496            };
23497            return Ok((schedule, vdeps, meta));
23498        }
23499
23500        let (schedule, vdeps, mut meta) = self.create_evaluation_schedule_uncached(to_evaluate)?;
23501        meta.schedule_cache_hit = false;
23502        meta.schedule_cache_eligible = false;
23503        #[cfg(any(test, feature = "benchmark_internal"))]
23504        {
23505            self.recalc_reuse_probe
23506                .get_mut()
23507                .unwrap()
23508                .schedule_cache_ineligible += 1;
23509        }
23510        Ok((EvaluationSchedule::Owned(schedule), vdeps, meta))
23511    }
23512
23513    fn create_evaluation_schedule_uncached(
23514        &self,
23515        to_evaluate: &[VertexId],
23516    ) -> Result<ScheduleBuildOutput, ExcelError> {
23517        #[cfg(any(test, feature = "benchmark_internal"))]
23518        {
23519            self.recalc_reuse_probe.lock().unwrap().schedule_builds += 1;
23520        }
23521        let builder = VirtualDepBuilder::new(self);
23522        let (vdeps, augmented, builder_elapsed_ms, vdeps_edges) =
23523            if self.config.enable_virtual_dep_telemetry {
23524                let build_started = crate::instant::FzInstant::now();
23525                let (vdeps, augmented) = builder.build(to_evaluate);
23526                let builder_elapsed_ms = build_started.elapsed().as_millis();
23527                let vdeps_edges = vdeps.values().map(|deps| deps.len()).sum::<usize>();
23528                (vdeps, augmented, builder_elapsed_ms, vdeps_edges)
23529            } else {
23530                let (vdeps, augmented) = builder.build(to_evaluate);
23531                (vdeps, augmented, 0, 0)
23532            };
23533
23534        let mut final_evaluate = to_evaluate.to_vec();
23535        if !augmented.is_empty() {
23536            final_evaluate.extend(augmented);
23537            final_evaluate.sort_unstable();
23538            final_evaluate.dedup();
23539        }
23540
23541        let use_virtual = !vdeps.is_empty();
23542
23543        let scheduler = Scheduler::new(&self.graph);
23544        let schedule = if use_virtual {
23545            scheduler.create_schedule_with_virtual(&final_evaluate, &vdeps)?
23546        } else {
23547            scheduler.create_schedule(&final_evaluate)?
23548        };
23549
23550        let meta = ScheduleBuildMeta {
23551            candidate_vertices: to_evaluate.len(),
23552            vdeps_vertices: vdeps.len(),
23553            vdeps_edges,
23554            builder_elapsed_ms,
23555            used_virtual_schedule: use_virtual,
23556            schedule_cache_hit: false,
23557            schedule_cache_eligible: false,
23558        };
23559
23560        Ok((schedule, vdeps, meta))
23561    }
23562
23563    fn can_use_static_schedule_cache(&self, to_evaluate: &[VertexId]) -> bool {
23564        !to_evaluate.is_empty()
23565            && to_evaluate.iter().copied().all(|v| {
23566                !self.graph.is_dynamic(v) && self.graph.get_range_dependencies(v).is_none()
23567            })
23568    }
23569
23570    fn start_virtual_dep_telemetry(&self) -> VirtualDepTelemetry {
23571        VirtualDepTelemetry {
23572            fallback_mode_activations: self.virtual_dep_fallback_activations,
23573            ..VirtualDepTelemetry::default()
23574        }
23575    }
23576
23577    fn accumulate_schedule_meta(telemetry: &mut VirtualDepTelemetry, meta: &ScheduleBuildMeta) {
23578        telemetry.candidate_vertices_total += meta.candidate_vertices;
23579        telemetry.vdeps_vertices_total += meta.vdeps_vertices;
23580        telemetry.vdeps_edges_total += meta.vdeps_edges;
23581        telemetry.builder_elapsed_ms_total += meta.builder_elapsed_ms;
23582        if meta.schedule_cache_eligible {
23583            if meta.schedule_cache_hit {
23584                telemetry.schedule_cache_hits += 1;
23585                telemetry.reused_schedule_vertices_total += meta.candidate_vertices;
23586            } else {
23587                telemetry.schedule_cache_misses += 1;
23588            }
23589        }
23590        if meta.used_virtual_schedule {
23591            telemetry.schedule_virtual_passes += 1;
23592        } else {
23593            telemetry.schedule_static_passes += 1;
23594        }
23595    }
23596
23597    fn dynamic_virtual_regions(&self, vertices: &[VertexId]) -> FxHashMap<VertexId, Vec<Region>> {
23598        vertices
23599            .iter()
23600            .copied()
23601            .filter(|vertex| self.graph.is_dynamic(*vertex))
23602            .filter_map(|vertex| {
23603                let regions = DynamicRefVirtualDepProvider::get_virtual_regions(self, vertex);
23604                (!regions.is_empty()).then_some((vertex, regions))
23605            })
23606            .collect()
23607    }
23608
23609    fn extend_target_roots_with_dynamic_regions<'a>(
23610        &self,
23611        roots: &mut Vec<crate::engine::target_preparation::TargetProducer>,
23612        regions: impl Iterator<Item = &'a Region>,
23613    ) -> Result<(), ExcelError> {
23614        use crate::engine::target_preparation::TargetProducer;
23615        let request_id = self
23616            .active_evaluation_resource_request
23617            .as_ref()
23618            .map(|request| request.request_id);
23619        let root_count = roots.len();
23620        let mut extended = OrderedTargetProducers::from_ordered(std::mem::take(roots))
23621            .map_err(|_| target_root_allocation_error(root_count, request_id))?;
23622        for region in regions {
23623            for vertex in self.graph.formula_vertices() {
23624                let Some(position) = self.graph.vertex_grid_addr(vertex) else {
23625                    continue;
23626                };
23627                let key = crate::formula_plane::region_index::RegionKey {
23628                    sheet_id: self.graph.get_vertex_sheet_id(vertex),
23629                    row: position.row(),
23630                    col: position.col(),
23631                };
23632                if region.contains_key(key) {
23633                    extended.push(TargetProducer::Legacy(vertex)).map_err(|_| {
23634                        target_root_allocation_error(extended.len() + 1, request_id)
23635                    })?;
23636                }
23637            }
23638            let authority = self.graph.formula_authority();
23639            for span_ref in authority.active_span_refs() {
23640                let Some(span) = authority.plane.spans.get(span_ref) else {
23641                    continue;
23642                };
23643                let Some(demanded) =
23644                    Region::from_domain(span.result_region.domain()).intersection(*region)
23645                else {
23646                    continue;
23647                };
23648                extended
23649                    .push(TargetProducer::Span { span_ref, demanded })
23650                    .map_err(|_| target_root_allocation_error(extended.len() + 1, request_id))?;
23651            }
23652        }
23653        *roots = extended.into_vec();
23654        Ok(())
23655    }
23656
23657    fn changed_virtual_dep_vertices(
23658        &mut self,
23659        to_evaluate: &[VertexId],
23660        old_vdeps: &FxHashMap<VertexId, Vec<VertexId>>,
23661    ) -> Vec<VertexId> {
23662        #[cfg(test)]
23663        if self.force_virtual_dep_changes_remaining_for_test > 0
23664            && let Some(vertex) = to_evaluate.first().copied()
23665        {
23666            self.force_virtual_dep_changes_remaining_for_test -= 1;
23667            return vec![vertex];
23668        }
23669        if !to_evaluate
23670            .iter()
23671            .copied()
23672            .any(|v| self.graph.is_dynamic(v))
23673        {
23674            return Vec::new();
23675        }
23676
23677        let builder = VirtualDepBuilder::new(self);
23678        let (new_vdeps, _) = builder.build(to_evaluate);
23679
23680        let mut candidates = FxHashSet::default();
23681        candidates.extend(old_vdeps.keys().copied());
23682        candidates.extend(new_vdeps.keys().copied());
23683
23684        let mut changed = Vec::new();
23685        for v in candidates {
23686            if old_vdeps.get(&v) != new_vdeps.get(&v) {
23687                changed.push(v);
23688            }
23689        }
23690        changed
23691    }
23692
23693    /// Build a demand-driven subgraph for the given targets, including ephemeral edges for
23694    /// compressed ranges, and returning the set of dirty/volatile precedents and virtual deps.
23695    fn build_demand_subgraph(
23696        &self,
23697        target_vertices: &[VertexId],
23698    ) -> (
23699        Vec<VertexId>,
23700        rustc_hash::FxHashMap<VertexId, Vec<VertexId>>,
23701    ) {
23702        #[cfg(feature = "tracing")]
23703        let _span =
23704            tracing::info_span!("demand_subgraph", targets = target_vertices.len()).entered();
23705        use rustc_hash::{FxHashMap, FxHashSet};
23706
23707        let mut to_evaluate: FxHashSet<VertexId> = FxHashSet::default();
23708        let mut visited: FxHashSet<VertexId> = FxHashSet::default();
23709        let mut stack: Vec<VertexId> = Vec::new();
23710        let mut vdeps: FxHashMap<VertexId, Vec<VertexId>> = FxHashMap::default(); // incoming deps per vertex
23711        #[cfg(any(test, feature = "benchmark_internal"))]
23712        let (mut probe_vertices, mut probe_clean_formulas, mut probe_edges) = (0, 0, 0);
23713
23714        for &t in target_vertices {
23715            stack.push(t);
23716        }
23717
23718        while let Some(v) = stack.pop() {
23719            if !visited.insert(v) {
23720                continue;
23721            }
23722            if !self.graph.vertex_exists(v) {
23723                continue;
23724            }
23725            #[cfg(any(test, feature = "benchmark_internal"))]
23726            {
23727                probe_vertices += 1;
23728                if matches!(
23729                    self.graph.get_vertex_kind(v),
23730                    VertexKind::FormulaScalar | VertexKind::FormulaArray
23731                ) && !self.graph.is_dirty(v)
23732                    && !self.graph.is_volatile(v)
23733                {
23734                    probe_clean_formulas += 1;
23735                }
23736            }
23737            // Schedule dirty/volatile formulas. Also schedule pass-through
23738            // Named*/Range vertices so the scheduler honours the
23739            // topological position of any formula cells that sit underneath
23740            // them — without these in `vertex_set` the scheduler skips the
23741            // edges that route a target through a named-range vertex into
23742            // its underlying cells, and the underlying cells then end up
23743            // in the same (or an earlier) layer as the target.
23744            match self.graph.get_vertex_kind(v) {
23745                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
23746                    if self.graph.is_dirty(v) || self.graph.is_volatile(v) {
23747                        to_evaluate.insert(v);
23748                    }
23749                }
23750                VertexKind::NamedScalar
23751                | VertexKind::NamedArray
23752                | VertexKind::Range
23753                | VertexKind::InfiniteRange => {
23754                    to_evaluate.insert(v);
23755                }
23756                _ => {}
23757            }
23758
23759            // Explicit dependencies (graph edges). We push *every* dep onto
23760            // the stack — not just formulas — because intermediate vertices
23761            // (NamedScalar, NamedArray, Range) are pass-through nodes whose
23762            // own dependencies point at the actual formula cells. Filtering
23763            // by kind here previously caused DN-range refs to be dropped
23764            // from the demand subgraph, so a target like
23765            // ``=SUM(named_range_pointing_at_dirty_cells)`` would evaluate
23766            // using stale values for those cells. The kind check at the top
23767            // of the loop still gates which vertices end up in
23768            // ``to_evaluate``; only Formula vertices are scheduled.
23769            if let Some(dependencies) = self.graph.dependencies_slice(v) {
23770                for &dep in dependencies {
23771                    #[cfg(any(test, feature = "benchmark_internal"))]
23772                    {
23773                        probe_edges += 1;
23774                    }
23775                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
23776                        stack.push(dep);
23777                    }
23778                }
23779            } else {
23780                for dep in self.graph.get_dependencies(v) {
23781                    #[cfg(any(test, feature = "benchmark_internal"))]
23782                    {
23783                        probe_edges += 1;
23784                    }
23785                    if self.graph.vertex_exists(dep) && !visited.contains(&dep) {
23786                        stack.push(dep);
23787                    }
23788                }
23789            } // Virtual dependencies (compressed ranges + dynamic like INDIRECT)
23790            let builder = VirtualDepBuilder::new(self);
23791            let (vdeps_map, _) = builder.build(&[v]);
23792            if let Some(deps) = vdeps_map.get(&v) {
23793                for &u in deps {
23794                    vdeps.entry(v).or_default().push(u);
23795                    if !visited.contains(&u) {
23796                        stack.push(u);
23797                    }
23798                }
23799            }
23800        }
23801
23802        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
23803        result.sort_unstable();
23804        // Dedup virtual deps
23805        for deps in vdeps.values_mut() {
23806            deps.sort_unstable();
23807            deps.dedup();
23808        }
23809        #[cfg(any(test, feature = "benchmark_internal"))]
23810        {
23811            let mut probe = self.recalc_reuse_probe.lock().unwrap();
23812            probe.demand_builds += 1;
23813            probe.demand_vertices += probe_vertices;
23814            probe.demand_clean_formulas += probe_clean_formulas;
23815            probe.demand_explicit_edges += probe_edges;
23816            probe.demand_virtual_builder_calls += probe_vertices;
23817        }
23818        (result, vdeps)
23819    }
23820
23821    /// Helper: convert 1-based column index to Excel-style letters (1 -> A, 27 -> AA)
23822    fn col_to_letters(col: u32) -> String {
23823        col_letters_from_1based(col).expect("column index must be >= 1")
23824    }
23825
23826    /// Evaluate all dirty/volatile vertices with cancellation support
23827    pub fn evaluate_all_cancellable(
23828        &mut self,
23829        cancel: crate::engine::CancelToken,
23830    ) -> Result<EvalResult, ExcelError> {
23831        self.observe_evaluation_resource_request(EvaluationRequestKind::FullCancellable, |engine| {
23832            engine.observe_function_semantic_epoch()?;
23833            engine.active_cancel_flag = Some(cancel.clone());
23834            let res = engine.evaluate_all_cancellable_impl(cancel.as_flag());
23835            engine.active_cancel_flag = None;
23836            res
23837        })
23838    }
23839
23840    fn evaluate_all_cancellable_impl(
23841        &mut self,
23842        cancel_flag: &AtomicBool,
23843    ) -> Result<EvalResult, ExcelError> {
23844        let _source_cache = self.source_cache_session();
23845        self.validate_deterministic_mode()?;
23846        if self.config.defer_graph_building {
23847            self.build_graph_all()?;
23848        }
23849        if cancel_flag.load(Ordering::Relaxed) {
23850            let message = if self.config.formula_plane_mode
23851                == FormulaPlaneMode::AuthoritativeExperimental
23852                && self.graph.formula_authority().active_span_count() > 0
23853            {
23854                "Evaluation cancelled before FormulaPlane scheduling"
23855            } else {
23856                "Evaluation cancelled before scheduling"
23857            };
23858            return Err(
23859                ExcelError::new(ExcelErrorKind::Cancelled).with_message(message.to_string())
23860            );
23861        }
23862        self.transition_off_mode_spans_to_legacy()?;
23863        self.begin_evaluation_request();
23864        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
23865            && self.graph.formula_authority().active_span_count() > 0
23866        {
23867            if cancel_flag.load(Ordering::Relaxed) {
23868                return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
23869                    "Evaluation cancelled before FormulaPlane scheduling".to_string(),
23870                ));
23871            }
23872            return self.evaluate_authoritative_formula_plane_all();
23873        }
23874        self.reset_virtual_dep_telemetry_if_disabled();
23875        let start = crate::instant::FzInstant::now();
23876        let mut computed_vertices = 0;
23877        let mut cycle_errors = 0;
23878
23879        let mut replan_iterations = 0;
23880        const MAX_REPLAN: usize = 5;
23881        let mut telemetry = self
23882            .config
23883            .enable_virtual_dep_telemetry
23884            .then(|| self.start_virtual_dep_telemetry());
23885
23886        loop {
23887            if cancel_flag.load(Ordering::Relaxed) {
23888                if let Some(mut t) = telemetry {
23889                    t.bailout_reason = Some("cancelled");
23890                    t.replan_iterations = replan_iterations;
23891                    self.last_virtual_dep_telemetry = t;
23892                }
23893                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23894                    .with_message("Evaluation cancelled before scheduling".to_string()));
23895            }
23896
23897            let to_evaluate = self.graph.get_evaluation_vertices();
23898            if to_evaluate.is_empty() {
23899                if let Some(t) = telemetry.as_mut()
23900                    && t.bailout_reason.is_none()
23901                {
23902                    t.bailout_reason = Some("no_work");
23903                }
23904                break;
23905            }
23906
23907            let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
23908            if let Some(t) = telemetry.as_mut() {
23909                Self::accumulate_schedule_meta(t, &meta);
23910            }
23911
23912            // Walk units in condensation order, checking cancellation between
23913            // units (formerly between cycles and between layers).
23914            for &unit in &schedule.units {
23915                match unit {
23916                    ScheduleUnit::Cycle(i) => {
23917                        // Check cancellation between cycles
23918                        if cancel_flag.load(Ordering::Relaxed) {
23919                            if let Some(mut t) = telemetry {
23920                                t.bailout_reason = Some("cancelled");
23921                                t.replan_iterations = replan_iterations;
23922                                self.last_virtual_dep_telemetry = t;
23923                            }
23924                            return Err(ExcelError::new(ExcelErrorKind::Cancelled).with_message(
23925                                "Evaluation cancelled during cycle handling".to_string(),
23926                            ));
23927                        }
23928
23929                        if self.handle_cycle_unit(
23930                            schedule.unit_cycle(i),
23931                            None,
23932                            None,
23933                            Some(cancel_flag),
23934                        )? > 0
23935                        {
23936                            cycle_errors += 1;
23937                        }
23938                    }
23939                    ScheduleUnit::Layer(i) => {
23940                        let layer = schedule.unit_layer(i);
23941                        // Check cancellation between layers
23942                        if cancel_flag.load(Ordering::Relaxed) {
23943                            if let Some(mut t) = telemetry {
23944                                t.bailout_reason = Some("cancelled");
23945                                t.replan_iterations = replan_iterations;
23946                                self.last_virtual_dep_telemetry = t;
23947                            }
23948                            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
23949                                .with_message("Evaluation cancelled between layers".to_string()));
23950                        }
23951
23952                        // Evaluate vertices in this layer (parallel or sequential)
23953                        if self.thread_pool.is_some() && layer.vertices.len() > 1 {
23954                            computed_vertices +=
23955                                self.evaluate_layer_parallel_cancellable(layer, cancel_flag)?;
23956                        } else {
23957                            computed_vertices +=
23958                                self.evaluate_layer_sequential_cancellable(layer, cancel_flag)?;
23959                        }
23960                    }
23961                }
23962            }
23963
23964            let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
23965            if let Some(t) = telemetry.as_mut() {
23966                t.changed_vdeps_total += changed_vertices.len();
23967            }
23968            self.resource_checkpoint(0)?;
23969            self.graph.clear_dirty_flags(&to_evaluate);
23970            for v in &changed_vertices {
23971                self.graph.set_dirty(*v, true);
23972            }
23973
23974            if changed_vertices.is_empty() {
23975                if let Some(t) = telemetry.as_mut() {
23976                    t.bailout_reason = Some("converged");
23977                }
23978                break;
23979            }
23980            if replan_iterations >= MAX_REPLAN {
23981                if let Some(mut t) = telemetry.take() {
23982                    t.bailout_reason = Some("max_replan");
23983                    t.replan_iterations = replan_iterations;
23984                    self.last_virtual_dep_telemetry = t;
23985                }
23986                return Err(
23987                    self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
23988                );
23989            }
23990            replan_iterations += 1;
23991        }
23992
23993        if let Some(mut t) = telemetry {
23994            t.replan_iterations = replan_iterations;
23995            self.last_virtual_dep_telemetry = t;
23996        }
23997
23998        // Re-dirty volatile vertices for the next evaluation cycle
23999        self.redirty_for_next_recalc();
24000        self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
24001
24002        Ok(EvalResult {
24003            computed_vertices,
24004            cycle_errors,
24005            elapsed: start.elapsed(),
24006        })
24007    }
24008
24009    /// Evaluate only the necessary precedents for specific target cells with cancellation support
24010    pub fn evaluate_until_cancellable(
24011        &mut self,
24012        targets: &[&str],
24013        cancel: crate::engine::CancelToken,
24014    ) -> Result<EvalResult, ExcelError> {
24015        self.observe_evaluation_resource_request(
24016            EvaluationRequestKind::TargetedCancellable,
24017            |engine| {
24018                engine.observe_function_semantic_epoch()?;
24019                engine.active_cancel_flag = Some(cancel.clone());
24020                let res = engine.evaluate_until_cancellable_impl(targets, cancel.as_flag());
24021                engine.active_cancel_flag = None;
24022                res
24023            },
24024        )
24025    }
24026
24027    fn evaluate_until_cancellable_impl(
24028        &mut self,
24029        targets: &[&str],
24030        cancel_flag: &AtomicBool,
24031    ) -> Result<EvalResult, ExcelError> {
24032        if cancel_flag.load(Ordering::Relaxed) {
24033            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
24034                .with_message("Evaluation cancelled before target preparation"));
24035        }
24036        let mut typed_targets = Vec::with_capacity(targets.len());
24037        for target in targets {
24038            let (sheet, row, col) = self.parse_a1_notation(target)?;
24039            self.graph.sheet_id_mut(&sheet);
24040            typed_targets.push(crate::engine::EvaluationTarget::Cell { sheet, row, col });
24041        }
24042        self.evaluate_mixed_targets(&typed_targets, None)
24043    }
24044
24045    fn parse_a1_notation(&self, address: &str) -> Result<(String, u32, u32), ExcelError> {
24046        let mut quoted = false;
24047        let mut separator = None;
24048        let bytes = address.as_bytes();
24049        let mut index = 0usize;
24050        while index < bytes.len() {
24051            match bytes[index] {
24052                b'\'' => {
24053                    if quoted && bytes.get(index + 1) == Some(&b'\'') {
24054                        index = index.saturating_add(1);
24055                    } else {
24056                        quoted = !quoted;
24057                    }
24058                }
24059                b'!' if !quoted => separator = Some(index),
24060                _ => {}
24061            }
24062            index = index.saturating_add(1);
24063        }
24064        if quoted {
24065            return Err(ExcelError::new(ExcelErrorKind::Ref)
24066                .with_message(format!("Invalid quoted sheet reference `{address}`")));
24067        }
24068        let (sheet, cell_part) = match separator {
24069            Some(separator) => {
24070                let raw_sheet = &address[..separator];
24071                let sheet = if raw_sheet.starts_with('\'') && raw_sheet.ends_with('\'') {
24072                    raw_sheet[1..raw_sheet.len().saturating_sub(1)].replace("''", "'")
24073                } else {
24074                    raw_sheet.to_string()
24075                };
24076                (sheet, &address[separator + 1..])
24077            }
24078            None => (self.default_sheet_name().to_string(), address),
24079        };
24080
24081        let (row, col, _, _) = parse_a1_1based(cell_part).map_err(|err| {
24082            ExcelError::new(ExcelErrorKind::Ref)
24083                .with_message(format!("Invalid cell reference `{cell_part}`: {err}"))
24084        })?;
24085
24086        Ok((sheet, row, col))
24087    }
24088
24089    /// Determine volatility using this engine's FunctionProvider, falling back to global registry.
24090    fn is_ast_volatile_with_provider(&self, ast: &ASTNode) -> bool {
24091        use formualizer_parse::parser::ASTNodeType;
24092        match &ast.node_type {
24093            ASTNodeType::Function { name, args, .. } => {
24094                if let Some(func) = self
24095                    .get_function("", name)
24096                    .or_else(|| crate::function_registry::get("", name))
24097                    && func.caps().contains(crate::function::FnCaps::VOLATILE)
24098                {
24099                    return true;
24100                }
24101                args.iter()
24102                    .any(|arg| self.is_ast_volatile_with_provider(arg))
24103            }
24104            ASTNodeType::BinaryOp { left, right, .. } => {
24105                self.is_ast_volatile_with_provider(left)
24106                    || self.is_ast_volatile_with_provider(right)
24107            }
24108            ASTNodeType::UnaryOp { expr, .. } => self.is_ast_volatile_with_provider(expr),
24109            ASTNodeType::Array(rows) => rows.iter().any(|row| {
24110                row.iter()
24111                    .any(|cell| self.is_ast_volatile_with_provider(cell))
24112            }),
24113            _ => false,
24114        }
24115    }
24116
24117    /// Find dirty precedents that need evaluation for the given target vertices
24118    fn find_dirty_precedents(&self, target_vertices: &[VertexId]) -> Vec<VertexId> {
24119        let mut to_evaluate = FxHashSet::default();
24120        let mut visited = FxHashSet::default();
24121        let mut stack = Vec::new();
24122
24123        // Start reverse traversal from target vertices
24124        for &target in target_vertices {
24125            stack.push(target);
24126        }
24127
24128        while let Some(vertex_id) = stack.pop() {
24129            if !visited.insert(vertex_id) {
24130                continue; // Already processed
24131            }
24132
24133            if self.graph.vertex_exists(vertex_id) {
24134                // Check if this vertex needs evaluation
24135                let kind = self.graph.get_vertex_kind(vertex_id);
24136                let needs_eval = match kind {
24137                    super::vertex::VertexKind::FormulaScalar
24138                    | super::vertex::VertexKind::FormulaArray => {
24139                        self.graph.is_dirty(vertex_id) || self.graph.is_volatile(vertex_id)
24140                    }
24141                    _ => false, // Values and empty cells don't need evaluation
24142                };
24143
24144                if needs_eval {
24145                    to_evaluate.insert(vertex_id);
24146                }
24147
24148                // Continue traversal to dependencies (precedents)
24149                if let Some(dependencies) = self.graph.dependencies_slice(vertex_id) {
24150                    for &dep_id in dependencies {
24151                        if !visited.contains(&dep_id) {
24152                            stack.push(dep_id);
24153                        }
24154                    }
24155                } else {
24156                    let dependencies = self.graph.get_dependencies(vertex_id);
24157                    for dep_id in dependencies {
24158                        if !visited.contains(&dep_id) {
24159                            stack.push(dep_id);
24160                        }
24161                    }
24162                }
24163            }
24164        }
24165
24166        let mut result: Vec<VertexId> = to_evaluate.into_iter().collect();
24167        result.sort_unstable();
24168        result
24169    }
24170
24171    /// Evaluate a layer sequentially
24172    fn evaluate_layer_sequential(
24173        &mut self,
24174        layer: &super::scheduler::Layer,
24175    ) -> Result<usize, ExcelError> {
24176        self.resource_checkpoint(layer.vertices.len() as u64)?;
24177        self.evaluate_layer_sequential_effects(layer)
24178    }
24179
24180    fn update_vertex_value_with_delta(
24181        &mut self,
24182        vertex_id: VertexId,
24183        new_value: LiteralValue,
24184        delta: &mut DeltaCollector,
24185    ) {
24186        if delta.mode != DeltaMode::Off
24187            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
24188        {
24189            let sheet_name = self.graph.sheet_name(cell.sheet_id);
24190            let old = self
24191                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24192                .unwrap_or(LiteralValue::Empty);
24193            if old != new_value {
24194                delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
24195            }
24196        }
24197        self.graph.update_vertex_value(vertex_id, new_value.clone());
24198        self.mirror_vertex_value_to_overlay(vertex_id, &new_value);
24199    }
24200
24201    fn evaluate_layer_sequential_with_delta(
24202        &mut self,
24203        layer: &super::scheduler::Layer,
24204        delta: &mut DeltaCollector,
24205    ) -> Result<usize, ExcelError> {
24206        self.resource_checkpoint(layer.vertices.len() as u64)?;
24207        self.evaluate_layer_sequential_with_delta_effects(layer, delta)
24208    }
24209
24210    /// Evaluate a layer sequentially with cancellation support
24211    fn evaluate_layer_sequential_cancellable(
24212        &mut self,
24213        layer: &super::scheduler::Layer,
24214        cancel_flag: &AtomicBool,
24215    ) -> Result<usize, ExcelError> {
24216        self.resource_checkpoint(layer.vertices.len() as u64)?;
24217        self.evaluate_layer_sequential_cancellable_effects(layer, cancel_flag)
24218    }
24219
24220    /// Evaluate a layer sequentially with more frequent cancellation checks for demand-driven evaluation
24221    fn evaluate_layer_sequential_cancellable_demand_driven(
24222        &mut self,
24223        layer: &super::scheduler::Layer,
24224        cancel_flag: &AtomicBool,
24225    ) -> Result<usize, ExcelError> {
24226        self.resource_checkpoint(layer.vertices.len() as u64)?;
24227        self.evaluate_layer_sequential_cancellable_demand_driven_effects(layer, cancel_flag)
24228    }
24229
24230    /// Evaluate a layer in parallel using the thread pool
24231    fn evaluate_layer_parallel(
24232        &mut self,
24233        layer: &super::scheduler::Layer,
24234    ) -> Result<usize, ExcelError> {
24235        self.resource_checkpoint(layer.vertices.len() as u64)?;
24236        self.evaluate_layer_parallel_effects(layer)
24237    }
24238
24239    fn evaluate_layer_parallel_with_delta(
24240        &mut self,
24241        layer: &super::scheduler::Layer,
24242        delta: &mut DeltaCollector,
24243    ) -> Result<usize, ExcelError> {
24244        self.resource_checkpoint(layer.vertices.len() as u64)?;
24245        self.evaluate_layer_parallel_with_delta_effects(layer, delta)
24246    }
24247
24248    /// Evaluate a layer in parallel with cancellation support
24249    fn evaluate_layer_parallel_cancellable(
24250        &mut self,
24251        layer: &super::scheduler::Layer,
24252        cancel_flag: &AtomicBool,
24253    ) -> Result<usize, ExcelError> {
24254        self.resource_checkpoint(layer.vertices.len() as u64)?;
24255        self.evaluate_layer_parallel_cancellable_effects(layer, cancel_flag)
24256    }
24257
24258    /// Apply a computed result produced by `evaluate_vertex_immutable()`.
24259    ///
24260    /// This is the parallel equivalent of the "apply" portion of `evaluate_vertex_impl`.
24261    /// We keep apply sequential for correctness (spill commit is inherently stateful).
24262    fn apply_parallel_vertex_result(
24263        &mut self,
24264        vertex_id: VertexId,
24265        result: LiteralValue,
24266        mut delta: Option<&mut DeltaCollector>,
24267        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24268    ) -> Result<(), ExcelError> {
24269        // If this vertex's cell is currently covered by a spill from a different anchor,
24270        // ignore the computed result. The spill's committed values own the grid.
24271        if let Some(cell) = self.graph.get_cell_ref(vertex_id)
24272            && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
24273            && owner != vertex_id
24274        {
24275            return Ok(());
24276        }
24277
24278        let kind = self.graph.get_vertex_kind(vertex_id);
24279
24280        // Only formula vertices spill dynamic arrays into the grid.
24281        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
24282        if is_formula {
24283            let derived_format = self
24284                .derived_format_results
24285                .write()
24286                .unwrap()
24287                .remove(&vertex_id)
24288                .flatten();
24289            if let Some(cell) = self.graph.get_cell_ref(vertex_id) {
24290                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24291                self.write_computed_overlay_format_0based(
24292                    &sheet_name,
24293                    cell.coord.row(),
24294                    cell.coord.col(),
24295                    derived_format,
24296                );
24297            }
24298            match result {
24299                LiteralValue::Array(rows) => {
24300                    self.apply_array_result_from_parallel(
24301                        vertex_id,
24302                        rows,
24303                        delta.as_deref_mut(),
24304                        overwritable_formulas,
24305                    )?;
24306                }
24307                other => {
24308                    self.apply_non_array_result_from_parallel(
24309                        vertex_id,
24310                        other,
24311                        delta.as_deref_mut(),
24312                    );
24313                }
24314            }
24315            return Ok(());
24316        }
24317
24318        // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
24319        if let Some(d) = delta {
24320            self.update_vertex_value_with_delta(vertex_id, result, d);
24321        } else {
24322            self.graph.update_vertex_value(vertex_id, result.clone());
24323            self.mirror_vertex_value_to_overlay(vertex_id, &result);
24324        }
24325        Ok(())
24326    }
24327
24328    fn apply_non_array_result_from_parallel(
24329        &mut self,
24330        vertex_id: VertexId,
24331        value: LiteralValue,
24332        delta: Option<&mut DeltaCollector>,
24333    ) {
24334        // Scalar/error result: store value and ensure any previous spill is cleared.
24335        // This mirrors the sequential behavior in `evaluate_vertex_impl`.
24336        let spill_cells = self
24337            .graph
24338            .spill_cells_for_anchor(vertex_id)
24339            .map(|cells| cells.to_vec())
24340            .unwrap_or_default();
24341
24342        if let Some(d) = delta
24343            && d.mode != DeltaMode::Off
24344            && let Some(anchor) = self.graph.get_cell_ref_for_vertex(vertex_id)
24345        {
24346            if spill_cells.is_empty() {
24347                let old = self
24348                    .read_cell_value(
24349                        self.graph.sheet_name(anchor.sheet_id),
24350                        anchor.coord.row() + 1,
24351                        anchor.coord.col() + 1,
24352                    )
24353                    .unwrap_or(LiteralValue::Empty);
24354                if old != value {
24355                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24356                }
24357            } else {
24358                for cell in spill_cells.iter() {
24359                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
24360                    let old = self
24361                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
24362                        .unwrap_or(LiteralValue::Empty);
24363                    let new = if cell.sheet_id == anchor.sheet_id
24364                        && cell.coord.row() == anchor.coord.row()
24365                        && cell.coord.col() == anchor.coord.col()
24366                    {
24367                        value.clone()
24368                    } else {
24369                        LiteralValue::Empty
24370                    };
24371                    Self::record_cell_if_changed(d, cell, &old, &new);
24372                }
24373            }
24374        }
24375
24376        self.graph.clear_spill_region(vertex_id);
24377        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
24378            self.record_formula_plane_structural_change(scope);
24379        }
24380
24381        if self.config.arrow_storage_enabled
24382            && self.config.delta_overlay_enabled
24383            && self.config.write_formula_overlay_enabled
24384        {
24385            let empty = LiteralValue::Empty;
24386            for cell in spill_cells.iter() {
24387                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
24388                self.mirror_value_to_computed_overlay(
24389                    &sheet_name,
24390                    cell.coord.row() + 1,
24391                    cell.coord.col() + 1,
24392                    &empty,
24393                );
24394            }
24395        }
24396
24397        self.graph.update_vertex_value(vertex_id, value.clone());
24398        self.mirror_vertex_value_to_overlay(vertex_id, &value);
24399    }
24400
24401    fn apply_array_result_from_parallel(
24402        &mut self,
24403        vertex_id: VertexId,
24404        rows: Vec<Vec<LiteralValue>>,
24405        mut delta: Option<&mut DeltaCollector>,
24406        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24407    ) -> Result<(), ExcelError> {
24408        // Keep behavior consistent with the sequential spill path in `evaluate_vertex_impl`.
24409        self.graph
24410            .set_kind(vertex_id, crate::engine::vertex::VertexKind::FormulaArray);
24411
24412        let anchor = self
24413            .graph
24414            .get_cell_ref(vertex_id)
24415            .expect("cell ref for vertex");
24416        let sheet_id = anchor.sheet_id;
24417        let h = rows.len() as u32;
24418        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
24419
24420        // Hard cap to avoid vertex explosion from huge dynamic arrays.
24421        let spill_cells = (h as u64).saturating_mul(w as u64);
24422        if spill_cells > self.config.spill.max_spill_cells as u64 {
24423            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24424            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24425                .with_message("SpillTooLarge")
24426                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24427                    expected_rows: h,
24428                    expected_cols: w,
24429                });
24430            let spill_val = LiteralValue::Error(spill_err.clone());
24431            if let Some(d) = delta.as_deref_mut()
24432                && d.mode != DeltaMode::Off
24433            {
24434                let old = self
24435                    .read_cell_value(
24436                        self.graph.sheet_name(anchor.sheet_id),
24437                        anchor.coord.row() + 1,
24438                        anchor.coord.col() + 1,
24439                    )
24440                    .unwrap_or(LiteralValue::Empty);
24441                if old != spill_val {
24442                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24443                }
24444            }
24445            self.graph.update_vertex_value(vertex_id, spill_val.clone());
24446            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24447            return Ok(());
24448        }
24449
24450        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity)
24451        const PACKED_MAX_ROW: u32 = 1_048_575; // 20-bit max
24452        const PACKED_MAX_COL: u32 = 16_383; // 14-bit max
24453        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
24454        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
24455        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
24456            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24457            let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24458                .with_message("Spill exceeds sheet bounds")
24459                .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24460                    expected_rows: h,
24461                    expected_cols: w,
24462                });
24463            let spill_val = LiteralValue::Error(spill_err.clone());
24464            if let Some(d) = delta.as_deref_mut()
24465                && d.mode != DeltaMode::Off
24466            {
24467                let old = self
24468                    .read_cell_value(
24469                        self.graph.sheet_name(anchor.sheet_id),
24470                        anchor.coord.row() + 1,
24471                        anchor.coord.col() + 1,
24472                    )
24473                    .unwrap_or(LiteralValue::Empty);
24474                if old != spill_val {
24475                    d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24476                }
24477            }
24478            self.graph.update_vertex_value(vertex_id, spill_val.clone());
24479            self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24480            return Ok(());
24481        }
24482
24483        let mut targets = Vec::new();
24484        for r in 0..h {
24485            for c in 0..w {
24486                targets.push(self.graph.make_cell_ref_internal(
24487                    sheet_id,
24488                    anchor.coord.row() + r,
24489                    anchor.coord.col() + c,
24490                ));
24491            }
24492        }
24493
24494        match self.spill_mgr.reserve(
24495            vertex_id,
24496            anchor,
24497            SpillShape { rows: h, cols: w },
24498            SpillMeta {
24499                epoch: self.recalc_epoch,
24500                config: self.config.spill,
24501            },
24502        ) {
24503            Ok(()) => {
24504                if let Err(e) = self.commit_spill_and_mirror(
24505                    vertex_id,
24506                    &targets,
24507                    rows.clone(),
24508                    delta.as_deref_mut(),
24509                    overwritable_formulas,
24510                ) {
24511                    self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24512                    let err_val = LiteralValue::Error(e.clone());
24513                    if let Some(d) = delta.as_deref_mut()
24514                        && d.mode != DeltaMode::Off
24515                    {
24516                        let old = self
24517                            .read_cell_value(
24518                                self.graph.sheet_name(anchor.sheet_id),
24519                                anchor.coord.row() + 1,
24520                                anchor.coord.col() + 1,
24521                            )
24522                            .unwrap_or(LiteralValue::Empty);
24523                        if old != err_val {
24524                            d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24525                        }
24526                    }
24527                    self.graph.update_vertex_value(vertex_id, err_val.clone());
24528                    self.mirror_vertex_value_to_overlay(vertex_id, &err_val);
24529                    return Ok(());
24530                }
24531
24532                // Anchor shows the top-left value, like Excel
24533                let top_left = rows
24534                    .first()
24535                    .and_then(|r| r.first())
24536                    .cloned()
24537                    .unwrap_or(LiteralValue::Empty);
24538                self.graph.update_vertex_value(vertex_id, top_left.clone());
24539                self.mirror_vertex_value_to_overlay(vertex_id, &top_left);
24540                Ok(())
24541            }
24542            Err(e) => {
24543                self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
24544                let spill_err = ExcelError::new(ExcelErrorKind::Spill)
24545                    .with_message(e.message.unwrap_or_else(|| "Spill blocked".to_string()))
24546                    .with_extra(formualizer_common::ExcelErrorExtra::Spill {
24547                        expected_rows: h,
24548                        expected_cols: w,
24549                    });
24550                let spill_val = LiteralValue::Error(spill_err.clone());
24551                if let Some(d) = delta
24552                    && d.mode != DeltaMode::Off
24553                {
24554                    let old = self
24555                        .read_cell_value(
24556                            self.graph.sheet_name(anchor.sheet_id),
24557                            anchor.coord.row() + 1,
24558                            anchor.coord.col() + 1,
24559                        )
24560                        .unwrap_or(LiteralValue::Empty);
24561                    if old != spill_val {
24562                        d.record_cell(anchor.sheet_id, anchor.coord.row(), anchor.coord.col());
24563                    }
24564                }
24565                self.graph.update_vertex_value(vertex_id, spill_val.clone());
24566                self.mirror_vertex_value_to_overlay(vertex_id, &spill_val);
24567                Ok(())
24568            }
24569        }
24570    }
24571
24572    /// Evaluate a single vertex without mutating the graph (for parallel evaluation)
24573    fn evaluate_vertex_immutable(&self, vertex_id: VertexId) -> Result<LiteralValue, ExcelError> {
24574        // Check if vertex exists
24575        if !self.graph.vertex_exists(vertex_id) {
24576            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
24577                .with_message(format!("Vertex not found: {vertex_id:?}")));
24578        }
24579
24580        // Get vertex kind and check if it needs evaluation
24581        let kind = self.graph.get_vertex_kind(vertex_id);
24582        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
24583
24584        let ast_id = match kind {
24585            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
24586                if let Some(ast_id) = self.graph.get_formula_id(vertex_id) {
24587                    ast_id
24588                } else {
24589                    return Ok(LiteralValue::Number(0.0));
24590                }
24591            }
24592            VertexKind::Empty | VertexKind::Cell => {
24593                if let Some(cell_ref) = self.graph.get_cell_ref(vertex_id) {
24594                    let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24595                    let row = cell_ref.coord.row() + 1;
24596                    let col = cell_ref.coord.col() + 1;
24597                    if let Some(v) = self.read_cell_value(sheet_name, row, col) {
24598                        return Ok(v);
24599                    }
24600                }
24601                return Ok(LiteralValue::Number(0.0));
24602            }
24603            VertexKind::NamedScalar => {
24604                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24605                    ExcelError::new(ExcelErrorKind::Name)
24606                        .with_message("Named range metadata missing".to_string())
24607                })?;
24608
24609                return match &named_range.definition {
24610                    NamedDefinition::Cell(cell_ref) => {
24611                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24612                        Ok(self
24613                            .get_cell_value(
24614                                sheet_name,
24615                                cell_ref.coord.row() + 1,
24616                                cell_ref.coord.col() + 1,
24617                            )
24618                            .unwrap_or(LiteralValue::Empty))
24619                    }
24620                    NamedDefinition::Literal(v) => Ok(v.clone()),
24621                    NamedDefinition::Formula { ast, .. } => {
24622                        let context_sheet = match named_range.scope {
24623                            NameScope::Sheet(id) => id,
24624                            NameScope::Workbook => sheet_id,
24625                        };
24626                        let sheet_name = self.graph.sheet_name(context_sheet);
24627                        let cell_ref = self
24628                            .graph
24629                            .get_cell_ref(vertex_id)
24630                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24631                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24632                        interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
24633                    }
24634                    NamedDefinition::Range(_) => Err(ExcelError::new(ExcelErrorKind::Value)
24635                        .with_message("Range-valued name evaluated as scalar".to_string())),
24636                };
24637            }
24638            VertexKind::NamedArray => {
24639                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
24640                    ExcelError::new(ExcelErrorKind::Name)
24641                        .with_message("Named range metadata missing".to_string())
24642                })?;
24643
24644                return match &named_range.definition {
24645                    NamedDefinition::Range(range_ref) => {
24646                        if range_ref.start.sheet_id != range_ref.end.sheet_id {
24647                            return Err(ExcelError::new(ExcelErrorKind::Ref)
24648                                .with_message("Named range cannot span sheets".to_string()));
24649                        }
24650                        let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
24651                        let sr0 = range_ref.start.coord.row();
24652                        let sc0 = range_ref.start.coord.col();
24653                        let er0 = range_ref.end.coord.row();
24654                        let ec0 = range_ref.end.coord.col();
24655                        if sr0 > er0 || sc0 > ec0 {
24656                            return Err(ExcelError::new(ExcelErrorKind::Ref)
24657                                .with_message("Invalid named range bounds".to_string()));
24658                        }
24659
24660                        let h = (er0 - sr0 + 1) as usize;
24661                        let w = (ec0 - sc0 + 1) as usize;
24662                        let cell_count = (h as u64).saturating_mul(w as u64);
24663                        if cell_count > self.config.spill.max_spill_cells as u64 {
24664                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
24665                                "Named range too large to materialize as an array".to_string(),
24666                            ));
24667                        }
24668
24669                        let mut rows = Vec::with_capacity(h);
24670                        for r0 in sr0..=er0 {
24671                            let mut row = Vec::with_capacity(w);
24672                            for c0 in sc0..=ec0 {
24673                                let v = self
24674                                    .get_cell_value(sheet_name, r0 + 1, c0 + 1)
24675                                    .unwrap_or(LiteralValue::Empty);
24676                                row.push(v);
24677                            }
24678                            rows.push(row);
24679                        }
24680                        Ok(LiteralValue::Array(rows))
24681                    }
24682                    NamedDefinition::Cell(cell_ref) => {
24683                        let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
24684                        let row = cell_ref.coord.row() + 1;
24685                        let col = cell_ref.coord.col() + 1;
24686                        let v = self
24687                            .get_cell_value(sheet_name, row, col)
24688                            .unwrap_or(LiteralValue::Empty);
24689                        Ok(LiteralValue::Array(vec![vec![v]]))
24690                    }
24691                    NamedDefinition::Literal(v) => Ok(LiteralValue::Array(vec![vec![v.clone()]])),
24692                    NamedDefinition::Formula { ast, .. } => {
24693                        let context_sheet = match named_range.scope {
24694                            NameScope::Sheet(id) => id,
24695                            NameScope::Workbook => sheet_id,
24696                        };
24697                        let sheet_name = self.graph.sheet_name(context_sheet);
24698                        let cell_ref = self
24699                            .graph
24700                            .get_cell_ref(vertex_id)
24701                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
24702                        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24703                        match interpreter.evaluate_ast(ast) {
24704                            Ok(cv) => {
24705                                let v = cv.into_literal();
24706                                match v {
24707                                    LiteralValue::Array(_) => Ok(v),
24708                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
24709                                }
24710                            }
24711                            Err(err) => Ok(LiteralValue::Error(err)),
24712                        }
24713                    }
24714                };
24715            }
24716            VertexKind::InfiniteRange
24717            | VertexKind::Range
24718            | VertexKind::External
24719            | VertexKind::Table => {
24720                // Not directly evaluatable here.
24721                return Ok(LiteralValue::Number(0.0));
24722            }
24723        };
24724
24725        // The interpreter uses a reference to the engine as the context
24726        let sheet_name = self.graph.sheet_name(sheet_id);
24727        let cell_ref = self
24728            .graph
24729            .get_cell_ref(vertex_id)
24730            .expect("cell ref for vertex");
24731        let interpreter = Interpreter::new_with_cell(self, sheet_name, cell_ref);
24732
24733        interpreter
24734            .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
24735            .map(|cv| {
24736                let format = cv.format_id();
24737                self.derived_format_results
24738                    .write()
24739                    .unwrap()
24740                    .insert(vertex_id, format);
24741                self.record_derived_format(vertex_id, format);
24742                crate::engine::result_finalization::finalize_formula_result(cv.into_literal())
24743            })
24744    }
24745
24746    /// Get access to the shared thread pool for parallel evaluation
24747    pub fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
24748        self.thread_pool.as_ref()
24749    }
24750}
24751
24752#[derive(Default)]
24753struct RowBoundsCache {
24754    snapshot: u64,
24755    // key: (sheet_id, col_idx)
24756    map: rustc_hash::FxHashMap<(u32, usize), (Option<u32>, Option<u32>)>,
24757}
24758
24759impl RowBoundsCache {
24760    fn new(snapshot: u64) -> Self {
24761        Self {
24762            snapshot,
24763            map: Default::default(),
24764        }
24765    }
24766    fn get_row_bounds(
24767        &self,
24768        sheet_id: SheetId,
24769        col_idx: usize,
24770        snapshot: u64,
24771    ) -> Option<(Option<u32>, Option<u32>)> {
24772        if self.snapshot != snapshot {
24773            return None;
24774        }
24775        self.map.get(&(sheet_id as u32, col_idx)).copied()
24776    }
24777    fn put_row_bounds(
24778        &mut self,
24779        sheet_id: SheetId,
24780        col_idx: usize,
24781        snapshot: u64,
24782        bounds: (Option<u32>, Option<u32>),
24783    ) {
24784        if self.snapshot != snapshot {
24785            self.snapshot = snapshot;
24786            self.map.clear();
24787        }
24788        self.map.insert((sheet_id as u32, col_idx), bounds);
24789    }
24790}
24791
24792struct UsedAxisBoundsCache {
24793    snapshot: u64,
24794    row_bounds_by_col_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
24795    col_bounds_by_row_span: rustc_hash::FxHashMap<(SheetId, u32, u32), Option<(u32, u32)>>,
24796    #[cfg(test)]
24797    row_hits: std::sync::atomic::AtomicUsize,
24798    #[cfg(test)]
24799    row_misses: std::sync::atomic::AtomicUsize,
24800    #[cfg(test)]
24801    col_hits: std::sync::atomic::AtomicUsize,
24802    #[cfg(test)]
24803    col_misses: std::sync::atomic::AtomicUsize,
24804}
24805
24806impl UsedAxisBoundsCache {
24807    fn new(snapshot: u64) -> Self {
24808        Self {
24809            snapshot,
24810            row_bounds_by_col_span: Default::default(),
24811            col_bounds_by_row_span: Default::default(),
24812            #[cfg(test)]
24813            row_hits: std::sync::atomic::AtomicUsize::new(0),
24814            #[cfg(test)]
24815            row_misses: std::sync::atomic::AtomicUsize::new(0),
24816            #[cfg(test)]
24817            col_hits: std::sync::atomic::AtomicUsize::new(0),
24818            #[cfg(test)]
24819            col_misses: std::sync::atomic::AtomicUsize::new(0),
24820        }
24821    }
24822
24823    fn reset_for_snapshot(&mut self, snapshot: u64) {
24824        if self.snapshot != snapshot {
24825            self.snapshot = snapshot;
24826            self.row_bounds_by_col_span.clear();
24827            self.col_bounds_by_row_span.clear();
24828        }
24829    }
24830
24831    fn get_row_bounds(
24832        &self,
24833        sheet_id: SheetId,
24834        start_col: u32,
24835        end_col: u32,
24836        snapshot: u64,
24837    ) -> Option<Option<(u32, u32)>> {
24838        if self.snapshot != snapshot {
24839            return None;
24840        }
24841        let cached = self
24842            .row_bounds_by_col_span
24843            .get(&(sheet_id, start_col, end_col))
24844            .copied();
24845        #[cfg(test)]
24846        if cached.is_some() {
24847            self.row_hits.fetch_add(1, Ordering::Relaxed);
24848        }
24849        cached
24850    }
24851
24852    fn put_row_bounds(
24853        &mut self,
24854        sheet_id: SheetId,
24855        start_col: u32,
24856        end_col: u32,
24857        snapshot: u64,
24858        bounds: Option<(u32, u32)>,
24859    ) {
24860        self.reset_for_snapshot(snapshot);
24861        self.row_bounds_by_col_span
24862            .insert((sheet_id, start_col, end_col), bounds);
24863        #[cfg(test)]
24864        self.row_misses.fetch_add(1, Ordering::Relaxed);
24865    }
24866
24867    fn get_col_bounds(
24868        &self,
24869        sheet_id: SheetId,
24870        start_row: u32,
24871        end_row: u32,
24872        snapshot: u64,
24873    ) -> Option<Option<(u32, u32)>> {
24874        if self.snapshot != snapshot {
24875            return None;
24876        }
24877        let cached = self
24878            .col_bounds_by_row_span
24879            .get(&(sheet_id, start_row, end_row))
24880            .copied();
24881        #[cfg(test)]
24882        if cached.is_some() {
24883            self.col_hits.fetch_add(1, Ordering::Relaxed);
24884        }
24885        cached
24886    }
24887
24888    fn put_col_bounds(
24889        &mut self,
24890        sheet_id: SheetId,
24891        start_row: u32,
24892        end_row: u32,
24893        snapshot: u64,
24894        bounds: Option<(u32, u32)>,
24895    ) {
24896        self.reset_for_snapshot(snapshot);
24897        self.col_bounds_by_row_span
24898            .insert((sheet_id, start_row, end_row), bounds);
24899        #[cfg(test)]
24900        self.col_misses.fetch_add(1, Ordering::Relaxed);
24901    }
24902}
24903
24904// Phase 2 shim: in-process spill manager delegating to current graph methods.
24905#[derive(Default)]
24906pub struct ShimSpillManager {
24907    region_locks: RegionLockManager,
24908    pub(crate) active_locks: rustc_hash::FxHashMap<VertexId, u64>,
24909}
24910
24911impl ShimSpillManager {
24912    pub(crate) fn reserve(
24913        &mut self,
24914        owner: VertexId,
24915        anchor_cell: CellRef,
24916        shape: SpillShape,
24917        _meta: SpillMeta,
24918    ) -> Result<(), ExcelError> {
24919        // Derive region from anchor + shape; enforce in-flight exclusivity only.
24920        let region = crate::engine::spill::Region {
24921            sheet_id: anchor_cell.sheet_id as u32,
24922            row_start: anchor_cell.coord.row(),
24923            row_end: anchor_cell
24924                .coord
24925                .row()
24926                .saturating_add(shape.rows)
24927                .saturating_sub(1),
24928            col_start: anchor_cell.coord.col(),
24929            col_end: anchor_cell
24930                .coord
24931                .col()
24932                .saturating_add(shape.cols)
24933                .saturating_sub(1),
24934        };
24935        match self.region_locks.reserve(region, owner) {
24936            Ok(id) => {
24937                if id != 0 {
24938                    self.active_locks.insert(owner, id);
24939                }
24940                Ok(())
24941            }
24942            Err(e) => Err(e),
24943        }
24944    }
24945
24946    /// Release any in-flight region reservation still held for `owner`.
24947    ///
24948    /// Reservations are normally released on commit/rollback, but if an anchor is
24949    /// abandoned without committing (e.g. cycle detection stamps it with #CIRC), a
24950    /// stale reservation could remain. This is a no-op when nothing is held.
24951    pub(crate) fn release_owner(&mut self, owner: VertexId) {
24952        if let Some(id) = self.active_locks.remove(&owner) {
24953            self.region_locks.release(id);
24954        }
24955    }
24956
24957    pub(crate) fn commit_array_with_value_probe<F>(
24958        &mut self,
24959        graph: &mut DependencyGraph,
24960        anchor_vertex: VertexId,
24961        targets: &[CellRef],
24962        rows: Vec<Vec<LiteralValue>>,
24963        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
24964        mut value_probe: F,
24965    ) -> Result<(), ExcelError>
24966    where
24967        F: FnMut(&DependencyGraph, &CellRef) -> Option<LiteralValue>,
24968    {
24969        use formualizer_common::{ExcelErrorExtra, ExcelErrorKind};
24970
24971        // Re-run plan on concrete targets before committing to respect blockers.
24972        // This plan checks formula/spill ownership in the graph, but when the graph value cache
24973        // is disabled (Arrow-canonical mode), it cannot see non-empty value blockers.
24974        let plan_res = graph.plan_spill_region_allowing_formula_overwrite(
24975            anchor_vertex,
24976            targets,
24977            overwritable_formulas,
24978        );
24979        if let Err(e) = plan_res {
24980            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
24981                self.region_locks.release(id);
24982            }
24983            return Err(e);
24984        }
24985
24986        if !graph.value_cache_enabled() {
24987            // Compute expected spill shape from the target rectangle for diagnostics.
24988            let (expected_rows, expected_cols) = if targets.is_empty() {
24989                (0u32, 0u32)
24990            } else {
24991                let mut min_r = u32::MAX;
24992                let mut max_r = 0u32;
24993                let mut min_c = u32::MAX;
24994                let mut max_c = 0u32;
24995                for cell in targets {
24996                    let r = cell.coord.row();
24997                    let c = cell.coord.col();
24998                    min_r = min_r.min(r);
24999                    max_r = max_r.max(r);
25000                    min_c = min_c.min(c);
25001                    max_c = max_c.max(c);
25002                }
25003                (
25004                    max_r.saturating_sub(min_r).saturating_add(1),
25005                    max_c.saturating_sub(min_c).saturating_add(1),
25006                )
25007            };
25008
25009            let anchor_cell = graph
25010                .get_cell_ref(anchor_vertex)
25011                .expect("anchor cell ref for spill commit");
25012
25013            for cell in targets {
25014                // Never treat the anchor as a blocker.
25015                if *cell == anchor_cell {
25016                    continue;
25017                }
25018                // Skip cells already known to be owned by a spill; plan() handled spill conflicts.
25019                if graph.spill_registry_anchor_for_cell(*cell).is_some() {
25020                    continue;
25021                }
25022                // Skip formula vertices in the target region; plan() handled them (or allowed).
25023                if let Some(&vid) = graph.get_vertex_id_for_address(cell)
25024                    && vid != anchor_vertex
25025                {
25026                    match graph.get_vertex_kind(vid) {
25027                        crate::engine::vertex::VertexKind::FormulaScalar
25028                        | crate::engine::vertex::VertexKind::FormulaArray => {
25029                            // plan() already approved allowed overwrites.
25030                            continue;
25031                        }
25032                        _ => {}
25033                    }
25034                }
25035
25036                if let Some(v) = value_probe(graph, cell)
25037                    && !matches!(v, LiteralValue::Empty)
25038                {
25039                    if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25040                        self.region_locks.release(id);
25041                    }
25042                    return Err(ExcelError::new(ExcelErrorKind::Spill)
25043                        .with_message("BlockedByValue")
25044                        .with_extra(ExcelErrorExtra::Spill {
25045                            expected_rows,
25046                            expected_cols,
25047                        }));
25048                }
25049            }
25050        }
25051
25052        let commit_res = graph.commit_spill_region_atomic_with_fault(
25053            anchor_vertex,
25054            targets.to_vec(),
25055            rows,
25056            None,
25057        );
25058        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25059            self.region_locks.release(id);
25060        }
25061        commit_res.map(|_| ())
25062    }
25063
25064    /// Commit a spill and mirror all written cells into Arrow overlay via the owning engine.
25065    pub(crate) fn commit_array_with_overlay<R: EvaluationContext>(
25066        &mut self,
25067        engine: &mut Engine<R>,
25068        anchor_vertex: VertexId,
25069        targets: &[CellRef],
25070        rows: Vec<Vec<LiteralValue>>,
25071        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
25072    ) -> Result<(), ExcelError> {
25073        // Re-run plan on concrete targets before committing to respect blockers.
25074        let plan_res = engine.graph.plan_spill_region_allowing_formula_overwrite(
25075            anchor_vertex,
25076            targets,
25077            overwritable_formulas,
25078        );
25079        if let Err(e) = plan_res {
25080            if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25081                self.region_locks.release(id);
25082            }
25083            return Err(e);
25084        }
25085
25086        let commit_res = engine.graph.commit_spill_region_atomic_with_fault(
25087            anchor_vertex,
25088            targets.to_vec(),
25089            rows.clone(),
25090            None,
25091        );
25092        if let Some(id) = self.active_locks.remove(&anchor_vertex) {
25093            self.region_locks.release(id);
25094        }
25095        commit_res.map(|_| ())?;
25096
25097        // Mirror into Arrow overlay when enabled
25098        if engine.config.arrow_storage_enabled
25099            && engine.config.delta_overlay_enabled
25100            && engine.config.write_formula_overlay_enabled
25101        {
25102            // Expect targets to be a contiguous rectangle row-major starting at some anchor
25103            for (idx, cell) in targets.iter().enumerate() {
25104                let (r_off, c_off) = {
25105                    if rows.is_empty() || rows[0].is_empty() {
25106                        (0usize, 0usize)
25107                    } else {
25108                        let width = rows[0].len();
25109                        (idx / width, idx % width)
25110                    }
25111                };
25112                let v = rows
25113                    .get(r_off)
25114                    .and_then(|r| r.get(c_off))
25115                    .cloned()
25116                    .unwrap_or(LiteralValue::Empty);
25117                let sheet_name = engine.graph.sheet_name(cell.sheet_id).to_string();
25118                engine.mirror_value_to_computed_overlay(
25119                    &sheet_name,
25120                    cell.coord.row() + 1,
25121                    cell.coord.col() + 1,
25122                    &v,
25123                );
25124            }
25125        }
25126        Ok(())
25127    }
25128}
25129
25130impl<R> Engine<R>
25131where
25132    R: EvaluationContext,
25133{
25134    fn resolve_shared_ref(
25135        &self,
25136        reference: &ReferenceType,
25137        current_sheet: &str,
25138    ) -> Result<formualizer_common::SheetRef<'static>, ExcelError> {
25139        use formualizer_common::{
25140            SheetCellRef as SharedCellRef, SheetLocator, SheetRangeRef as SharedRangeRef,
25141            SheetRef as SharedRef,
25142        };
25143
25144        // Preserve anchor flags from the parsed reference when possible.
25145        let sr = match reference {
25146            ReferenceType::Cell {
25147                sheet,
25148                row,
25149                col,
25150                row_abs,
25151                col_abs,
25152            } => {
25153                let row0 = row
25154                    .checked_sub(1)
25155                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25156                let col0 = col
25157                    .checked_sub(1)
25158                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25159                let sheet_loc = match sheet.as_deref() {
25160                    Some(name) => SheetLocator::from_name(name),
25161                    None => SheetLocator::Current,
25162                };
25163                let coord = formualizer_common::RelativeCoord::new(row0, col0, *row_abs, *col_abs);
25164                SharedRef::Cell(SharedCellRef::new(sheet_loc, coord))
25165            }
25166            ReferenceType::Range {
25167                sheet,
25168                start_row,
25169                start_col,
25170                end_row,
25171                end_col,
25172                start_row_abs,
25173                start_col_abs,
25174                end_row_abs,
25175                end_col_abs,
25176            } => {
25177                let sheet_loc = match sheet.as_deref() {
25178                    Some(name) => SheetLocator::from_name(name),
25179                    None => SheetLocator::Current,
25180                };
25181                let sr = start_row
25182                    .map(|r| {
25183                        r.checked_sub(1)
25184                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25185                    })
25186                    .transpose()?;
25187                let sc = start_col
25188                    .map(|c| {
25189                        c.checked_sub(1)
25190                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25191                    })
25192                    .transpose()?;
25193                let er = end_row
25194                    .map(|r| {
25195                        r.checked_sub(1)
25196                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25197                    })
25198                    .transpose()?;
25199                let ec = end_col
25200                    .map(|c| {
25201                        c.checked_sub(1)
25202                            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25203                    })
25204                    .transpose()?;
25205                let range = SharedRangeRef::from_parts(
25206                    sheet_loc,
25207                    sr.map(|idx| formualizer_common::AxisBound::new(idx, *start_row_abs)),
25208                    sc.map(|idx| formualizer_common::AxisBound::new(idx, *start_col_abs)),
25209                    er.map(|idx| formualizer_common::AxisBound::new(idx, *end_row_abs)),
25210                    ec.map(|idx| formualizer_common::AxisBound::new(idx, *end_col_abs)),
25211                )
25212                .map_err(|_| ExcelError::new(ExcelErrorKind::Ref))?;
25213                SharedRef::Range(range)
25214            }
25215            _ => return Err(ExcelError::new(ExcelErrorKind::Ref)),
25216        };
25217
25218        let current_id = self
25219            .graph
25220            .sheet_id(current_sheet)
25221            .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25222
25223        let resolve_loc = |loc: SheetLocator<'_>| -> Result<SheetLocator<'static>, ExcelError> {
25224            match loc {
25225                SheetLocator::Current => Ok(SheetLocator::Id(current_id)),
25226                SheetLocator::Id(id) => Ok(SheetLocator::Id(id)),
25227                SheetLocator::Name(name) => {
25228                    let n = name.as_ref();
25229                    self.graph
25230                        .sheet_id(n)
25231                        .map(SheetLocator::Id)
25232                        .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))
25233                }
25234            }
25235        };
25236
25237        match sr {
25238            SharedRef::Cell(cell) => {
25239                let owned = cell.into_owned();
25240                let sheet = resolve_loc(owned.sheet)?;
25241                Ok(SharedRef::Cell(SharedCellRef::new(sheet, owned.coord)))
25242            }
25243            SharedRef::Range(range) => {
25244                let owned = range.into_owned();
25245                let sheet = resolve_loc(owned.sheet)?;
25246                Ok(SharedRef::Range(SharedRangeRef {
25247                    sheet,
25248                    start_row: owned.start_row,
25249                    start_col: owned.start_col,
25250                    end_row: owned.end_row,
25251                    end_col: owned.end_col,
25252                }))
25253            }
25254        }
25255    }
25256}
25257
25258// Implement the resolver traits for the Engine.
25259// This allows the interpreter to resolve references by querying the engine's graph.
25260impl<R> crate::traits::ReferenceResolver for Engine<R>
25261where
25262    R: EvaluationContext,
25263{
25264    fn resolve_cell_reference(
25265        &self,
25266        sheet: Option<&str>,
25267        row: u32,
25268        col: u32,
25269    ) -> Result<LiteralValue, ExcelError> {
25270        // This context-free trait method has no knowledge of the formula's
25271        // current sheet, so an unqualified (`None`) reference cannot be resolved
25272        // here. Previously this fell back to `default_sheet_name()`, which leaked
25273        // the reference onto an unrelated sheet (issue #110). Interpreter paths
25274        // already qualify references with the current sheet before reaching this
25275        // method (see `Interpreter::implicit_intersection_from_reference`), and
25276        // the sheet-aware scalar path goes through `resolve_cell_reference_value`
25277        // with an explicit `current_sheet`. Returning #REF! for an unqualified
25278        // reference here surfaces the missing context instead of silently
25279        // returning data from the wrong sheet.
25280        let Some(sheet_name) = sheet else {
25281            return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
25282                "Unqualified cell reference resolved without sheet context".to_string(),
25283            ));
25284        };
25285        // Prefer engine's unified accessor which consults Arrow store for base values
25286        // and falls back to graph for formulas and stored values.
25287        if let Some(v) = self.get_cell_value(sheet_name, row, col) {
25288            Ok(v)
25289        } else {
25290            // Excel semantics: empty cell coerces to 0 in numeric contexts
25291            Ok(LiteralValue::Number(0.0))
25292        }
25293    }
25294}
25295
25296impl<R> crate::traits::RangeResolver for Engine<R>
25297where
25298    R: EvaluationContext,
25299{
25300    fn resolve_range_reference(
25301        &self,
25302        sheet: Option<&str>,
25303        sr: Option<u32>,
25304        sc: Option<u32>,
25305        er: Option<u32>,
25306        ec: Option<u32>,
25307    ) -> Result<Box<dyn crate::traits::Range>, ExcelError> {
25308        // For now, delegate range resolution to the external resolver.
25309        // A future optimization could be to handle this within the graph.
25310        self.resolver.resolve_range_reference(sheet, sr, sc, er, ec)
25311    }
25312}
25313
25314impl<R> crate::traits::NamedRangeResolver for Engine<R>
25315where
25316    R: EvaluationContext,
25317{
25318    fn resolve_named_range_reference(
25319        &self,
25320        name: &str,
25321    ) -> Result<Vec<Vec<LiteralValue>>, ExcelError> {
25322        self.resolver.resolve_named_range_reference(name)
25323    }
25324}
25325
25326impl<R> crate::traits::TableResolver for Engine<R>
25327where
25328    R: EvaluationContext,
25329{
25330    fn resolve_table_reference(
25331        &self,
25332        tref: &formualizer_parse::parser::TableReference,
25333    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
25334        self.resolver.resolve_table_reference(tref)
25335    }
25336}
25337
25338impl<R> crate::traits::SourceResolver for Engine<R>
25339where
25340    R: EvaluationContext,
25341{
25342    fn source_scalar_version(&self, name: &str) -> Option<u64> {
25343        self.resolver.source_scalar_version(name)
25344    }
25345
25346    fn resolve_source_scalar(&self, name: &str) -> Result<LiteralValue, ExcelError> {
25347        self.resolver.resolve_source_scalar(name)
25348    }
25349
25350    fn source_table_version(&self, name: &str) -> Option<u64> {
25351        self.resolver.source_table_version(name)
25352    }
25353
25354    fn resolve_source_table(
25355        &self,
25356        name: &str,
25357    ) -> Result<Box<dyn crate::traits::Table>, ExcelError> {
25358        self.resolver.resolve_source_table(name)
25359    }
25360}
25361
25362// The Engine is a Resolver because it implements the constituent traits.
25363impl<R> crate::traits::Resolver for Engine<R> where R: EvaluationContext {}
25364
25365// The Engine provides functions by delegating to its internal resolver.
25366impl<R> crate::traits::FunctionProvider for Engine<R>
25367where
25368    R: EvaluationContext,
25369{
25370    fn planning_semantic_revision(&self) -> Option<u64> {
25371        self.resolver.planning_semantic_revision()
25372    }
25373
25374    fn get_function(
25375        &self,
25376        prefix: &str,
25377        name: &str,
25378    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
25379        self.resolver.get_function(prefix, name)
25380    }
25381
25382    fn get_function_for_planning(
25383        &self,
25384        prefix: &str,
25385        name: &str,
25386    ) -> Option<std::sync::Arc<dyn crate::function::Function>> {
25387        self.resolver.get_function_for_planning(prefix, name)
25388    }
25389}
25390
25391impl<R> Engine<R>
25392where
25393    R: EvaluationContext,
25394{
25395    /// Semantic used coordinates exclude graph-only dependency placeholders.
25396    ///
25397    /// Non-empty base/overlay/computed cells come from Arrow storage, while
25398    /// scalar and array formulas come from graph formula kinds even before
25399    /// their results are materialized. The legacy graph fallback is omitted:
25400    /// `load_packed_to_vertex` entries are either represented by those sources
25401    /// or are `Empty` dependency placeholders, not a third value authority.
25402    pub(crate) fn semantic_used_rows_for_columns(
25403        &self,
25404        sheet: &str,
25405        start_col: u32,
25406        end_col: u32,
25407    ) -> Option<(u32, u32)> {
25408        let arrow_bounds = self
25409            .sheet_store()
25410            .sheet(sheet)
25411            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
25412        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
25413        Self::union_used_bounds(arrow_bounds, formula_bounds)
25414    }
25415
25416    pub(crate) fn semantic_used_cols_for_rows(
25417        &self,
25418        sheet: &str,
25419        start_row: u32,
25420        end_row: u32,
25421    ) -> Option<(u32, u32)> {
25422        let arrow_bounds = self
25423            .sheet_store()
25424            .sheet(sheet)
25425            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
25426        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
25427        Self::union_used_bounds(arrow_bounds, formula_bounds)
25428    }
25429}
25430
25431// Override EvaluationContext to provide thread pool access
25432impl<R> crate::traits::EvaluationContext for Engine<R>
25433where
25434    R: EvaluationContext,
25435{
25436    fn clock(&self) -> &dyn crate::timezone::ClockProvider {
25437        &self.clock
25438    }
25439
25440    fn thread_pool(&self) -> Option<&Arc<rayon::ThreadPool>> {
25441        self.thread_pool.as_ref()
25442    }
25443
25444    fn cancellation_token(&self) -> Option<crate::engine::CancelToken> {
25445        self.active_cancel_flag.clone()
25446    }
25447
25448    fn chunk_hint(&self) -> Option<usize> {
25449        // Use a simple heuristic from configuration (stripe width * height) as a default hint.
25450        let hint =
25451            (self.config.stripe_height as usize).saturating_mul(self.config.stripe_width as usize);
25452        Some(hint.clamp(1024, 1 << 20)) // clamp between 1K and ~1M
25453    }
25454
25455    fn volatile_level(&self) -> crate::traits::VolatileLevel {
25456        self.config.volatile_level
25457    }
25458
25459    fn workbook_seed(&self) -> u64 {
25460        self.config.workbook_seed
25461    }
25462
25463    fn recalc_epoch(&self) -> u64 {
25464        self.recalc_epoch
25465    }
25466
25467    fn workbook_sheet_count(&self) -> Option<usize> {
25468        Some(self.graph.sheet_reg().active_len())
25469    }
25470
25471    fn sheet_index_by_name(&self, sheet: &str) -> Option<usize> {
25472        self.graph.sheet_reg().active_position(sheet)
25473    }
25474
25475    fn current_sheet_index(&self, current_sheet: &str) -> Option<usize> {
25476        self.sheet_index_by_name(current_sheet)
25477    }
25478
25479    fn inspect_reference(
25480        &self,
25481        reference: &ReferenceType,
25482        current_sheet: &str,
25483    ) -> Result<Option<ReferenceInfo>, ExcelError> {
25484        let sheet_info = |sheet_name: &str| -> Result<(SheetId, usize), ExcelError> {
25485            let sheet_id = self
25486                .graph
25487                .sheet_id(sheet_name)
25488                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25489            let sheet_index = self
25490                .graph
25491                .sheet_reg()
25492                .active_position_by_id(sheet_id)
25493                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25494            Ok((sheet_id, sheet_index))
25495        };
25496
25497        let cell_info =
25498            |sheet_name: &str, row: u32, col: u32| -> Result<ReferenceInfo, ExcelError> {
25499                let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
25500                let row0 = row
25501                    .checked_sub(1)
25502                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25503                let col0 = col
25504                    .checked_sub(1)
25505                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25506                Ok(ReferenceInfo {
25507                    first_sheet_index: Some(sheet_index),
25508                    sheet_count: Some(1),
25509                    first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
25510                })
25511            };
25512
25513        let range_info = |sheet_name: &str,
25514                          start_row: Option<u32>,
25515                          start_col: Option<u32>|
25516         -> Result<ReferenceInfo, ExcelError> {
25517            let (sheet_id, sheet_index) = sheet_info(sheet_name)?;
25518            let row = start_row.unwrap_or(1);
25519            let col = start_col.unwrap_or(1);
25520            let row0 = row
25521                .checked_sub(1)
25522                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25523            let col0 = col
25524                .checked_sub(1)
25525                .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25526            Ok(ReferenceInfo {
25527                first_sheet_index: Some(sheet_index),
25528                sheet_count: Some(1),
25529                first_cell: Some(CellRef::new(sheet_id, Coord::new(row0, col0, true, true))),
25530            })
25531        };
25532
25533        let info = match reference {
25534            ReferenceType::Cell {
25535                sheet, row, col, ..
25536            } => {
25537                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
25538                cell_info(sheet_name, *row, *col)?
25539            }
25540            ReferenceType::Range {
25541                sheet,
25542                start_row,
25543                start_col,
25544                ..
25545            } => {
25546                let sheet_name = sheet.as_deref().unwrap_or(current_sheet);
25547                range_info(sheet_name, *start_row, *start_col)?
25548            }
25549            ReferenceType::Cell3D {
25550                sheet_first,
25551                sheet_last,
25552                row,
25553                col,
25554                ..
25555            } => {
25556                let first = cell_info(sheet_first, *row, *col)?;
25557                ReferenceInfo {
25558                    first_sheet_index: first.first_sheet_index,
25559                    sheet_count: self
25560                        .graph
25561                        .sheet_reg()
25562                        .active_span_len(sheet_first, sheet_last),
25563                    first_cell: first.first_cell,
25564                }
25565            }
25566            ReferenceType::Range3D {
25567                sheet_first,
25568                sheet_last,
25569                start_row,
25570                start_col,
25571                ..
25572            } => {
25573                let first = range_info(sheet_first, *start_row, *start_col)?;
25574                ReferenceInfo {
25575                    first_sheet_index: first.first_sheet_index,
25576                    sheet_count: self
25577                        .graph
25578                        .sheet_reg()
25579                        .active_span_len(sheet_first, sheet_last),
25580                    first_cell: first.first_cell,
25581                }
25582            }
25583            ReferenceType::NamedRange(name) => {
25584                let current_id = self
25585                    .graph
25586                    .sheet_id(current_sheet)
25587                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25588                let named = self
25589                    .graph
25590                    .resolve_name_entry(name, current_id)
25591                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25592                match &named.definition {
25593                    NamedDefinition::Cell(cell) => ReferenceInfo {
25594                        first_sheet_index: self
25595                            .graph
25596                            .sheet_reg()
25597                            .active_position_by_id(cell.sheet_id),
25598                        sheet_count: Some(1),
25599                        first_cell: Some(*cell),
25600                    },
25601                    NamedDefinition::Range(range) => ReferenceInfo {
25602                        first_sheet_index: self
25603                            .graph
25604                            .sheet_reg()
25605                            .active_position_by_id(range.start.sheet_id),
25606                        sheet_count: Some(1),
25607                        first_cell: Some(range.start),
25608                    },
25609                    NamedDefinition::Literal(_) | NamedDefinition::Formula { .. } => {
25610                        ReferenceInfo {
25611                            first_sheet_index: None,
25612                            sheet_count: None,
25613                            first_cell: None,
25614                        }
25615                    }
25616                }
25617            }
25618            ReferenceType::Table(tref) => {
25619                let table = self
25620                    .graph
25621                    .resolve_table_entry(&tref.name)
25622                    .ok_or_else(|| ExcelError::new(ExcelErrorKind::Ref))?;
25623                ReferenceInfo {
25624                    first_sheet_index: self
25625                        .graph
25626                        .sheet_reg()
25627                        .active_position_by_id(table.range.start.sheet_id),
25628                    sheet_count: Some(1),
25629                    first_cell: Some(table.range.start),
25630                }
25631            }
25632            ReferenceType::External(_) => return Err(ExcelError::new(ExcelErrorKind::Ref)),
25633        };
25634
25635        Ok(Some(info))
25636    }
25637
25638    fn formula_text_at_cell(&self, cell: CellRef) -> Result<Option<String>, ExcelError> {
25639        let sheet_name = self.graph.sheet_name(cell.sheet_id);
25640        if sheet_name.is_empty() {
25641            return Err(ExcelError::new(ExcelErrorKind::Ref));
25642        }
25643        let row = cell.coord.row() + 1;
25644        let col = cell.coord.col() + 1;
25645
25646        if let Some(entries) = self.staged_formulas.get(sheet_name)
25647            && let Some(text) = entries.get(row, col)
25648        {
25649            return Ok(Some(if text.starts_with('=') {
25650                text.to_owned()
25651            } else {
25652                format!("={text}")
25653            }));
25654        }
25655
25656        let Some((Some(ast), _)) = self.get_cell(sheet_name, row, col) else {
25657            return Ok(None);
25658        };
25659        Ok(Some(formualizer_parse::pretty::canonical_formula(&ast)))
25660    }
25661
25662    fn used_rows_for_columns(
25663        &self,
25664        sheet: &str,
25665        start_col: u32,
25666        end_col: u32,
25667    ) -> Option<(u32, u32)> {
25668        // Union Arrow-backed used-region with formula rows that have not been materialized yet.
25669        let sheet_id = self.graph.sheet_id(sheet)?;
25670        let snap = self.data_snapshot_id();
25671        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
25672            guard
25673                .as_ref()
25674                .and_then(|cache| cache.get_row_bounds(sheet_id, start_col, end_col, snap))
25675        }) {
25676            return cached;
25677        }
25678
25679        let arrow_bounds = self
25680            .sheet_store()
25681            .sheet(sheet)
25682            .and_then(|_| self.arrow_used_row_bounds(sheet, start_col, end_col));
25683        let formula_bounds = self.formula_row_bounds_for_columns(sheet, start_col, end_col);
25684        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
25685            Some(bounds)
25686        } else {
25687            let sc0 = start_col.saturating_sub(1);
25688            let ec0 = end_col.saturating_sub(1);
25689            self.graph
25690                .used_row_bounds_for_columns(sheet_id, sc0, ec0)
25691                .map(|(a0, b0)| (a0 + 1, b0 + 1))
25692        };
25693
25694        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
25695            guard
25696                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
25697                .put_row_bounds(sheet_id, start_col, end_col, snap, computed);
25698        }
25699
25700        computed
25701    }
25702
25703    fn used_cols_for_rows(&self, sheet: &str, start_row: u32, end_row: u32) -> Option<(u32, u32)> {
25704        // Union Arrow-backed used-region with formula columns that have not been materialized yet.
25705        let sheet_id = self.graph.sheet_id(sheet)?;
25706        let snap = self.data_snapshot_id();
25707        if let Some(cached) = self.used_axis_bounds_cache.read().ok().and_then(|guard| {
25708            guard
25709                .as_ref()
25710                .and_then(|cache| cache.get_col_bounds(sheet_id, start_row, end_row, snap))
25711        }) {
25712            return cached;
25713        }
25714
25715        let arrow_bounds = self
25716            .sheet_store()
25717            .sheet(sheet)
25718            .and_then(|_| self.arrow_used_col_bounds(sheet, start_row, end_row));
25719        let formula_bounds = self.formula_col_bounds_for_rows(sheet, start_row, end_row);
25720        let computed = if let Some(bounds) = Self::union_used_bounds(arrow_bounds, formula_bounds) {
25721            Some(bounds)
25722        } else {
25723            let sr0 = start_row.saturating_sub(1);
25724            let er0 = end_row.saturating_sub(1);
25725            self.graph
25726                .used_col_bounds_for_rows(sheet_id, sr0, er0)
25727                .map(|(a0, b0)| (a0 + 1, b0 + 1))
25728        };
25729
25730        if let Ok(mut guard) = self.used_axis_bounds_cache.write() {
25731            guard
25732                .get_or_insert_with(|| UsedAxisBoundsCache::new(snap))
25733                .put_col_bounds(sheet_id, start_row, end_row, snap, computed);
25734        }
25735
25736        computed
25737    }
25738
25739    fn sheet_bounds(&self, sheet: &str) -> Option<(u32, u32)> {
25740        let _ = self.graph.sheet_id(sheet)?;
25741        // Excel-like upper bounds; we expose something finite but large.
25742        // Backends may override with real bounds.
25743        Some((1_048_576, 16_384)) // 1048576 rows, 16384 cols (XFD)
25744    }
25745
25746    fn data_snapshot_id(&self) -> u64 {
25747        self.snapshot_id.load(std::sync::atomic::Ordering::Relaxed)
25748    }
25749
25750    fn backend_caps(&self) -> crate::traits::BackendCaps {
25751        crate::traits::BackendCaps {
25752            streaming: true,
25753            used_region: true,
25754            write: false,
25755            tables: false,
25756            async_stream: false,
25757        }
25758    }
25759
25760    fn build_lookup_index(
25761        &self,
25762        view: &RangeView<'_>,
25763        axis: LookupAxis,
25764    ) -> Option<Arc<LookupIndex>> {
25765        self.build_lookup_index_impl(view, axis)
25766    }
25767
25768    // Flats removed
25769
25770    fn date_system(&self) -> crate::engine::DateSystem {
25771        self.config.date_system
25772    }
25773    /// New: resolve a reference into a RangeView (Phase 2 API)
25774    fn resolve_range_view<'c>(
25775        &'c self,
25776        reference: &ReferenceType,
25777        current_sheet: &str,
25778    ) -> Result<RangeView<'c>, ExcelError> {
25779        match reference {
25780            ReferenceType::External(ext) => {
25781                let name = ext.raw.as_str();
25782                match ext.kind {
25783                    formualizer_parse::parser::ExternalRefKind::Cell { .. } => {
25784                        let Some(source) = self.graph.resolve_source_scalar_entry(name) else {
25785                            return Err(ExcelError::new(ExcelErrorKind::Name)
25786                                .with_message(format!("Undefined name: {name}")));
25787                        };
25788                        let version = source
25789                            .version
25790                            .or_else(|| self.resolver.source_scalar_version(name));
25791                        let v = self.resolve_source_scalar_cached(name, version)?;
25792                        Ok(RangeView::from_owned_rows(
25793                            vec![vec![v]],
25794                            self.config.date_system,
25795                        ))
25796                    }
25797                    formualizer_parse::parser::ExternalRefKind::Range { .. } => {
25798                        let Some(source) = self.graph.resolve_source_table_entry(name) else {
25799                            return Err(ExcelError::new(ExcelErrorKind::Name)
25800                                .with_message(format!("Undefined table: {name}")));
25801                        };
25802                        let version = source
25803                            .version
25804                            .or_else(|| self.resolver.source_table_version(name));
25805                        let table = self.resolve_source_table_cached(name, version)?;
25806                        let spec = Some(formualizer_parse::parser::TableSpecifier::Data);
25807                        self.source_table_to_range_view(table.as_ref(), &spec)
25808                    }
25809                }
25810            }
25811            ReferenceType::Range { .. } => {
25812                let shared = self.resolve_shared_ref(reference, current_sheet)?;
25813                let formualizer_common::SheetRef::Range(range) = shared else {
25814                    return Err(ExcelError::new(ExcelErrorKind::Ref));
25815                };
25816                // No context sheet is available here, so an unresolved locator
25817                // is #REF! rather than a guess (issue #110).
25818                let sheet_id = match range.sheet {
25819                    formualizer_common::SheetLocator::Id(id) => id,
25820                    formualizer_common::SheetLocator::Current
25821                    | formualizer_common::SheetLocator::Name(_) => {
25822                        return Err(ExcelError::new(ExcelErrorKind::Ref));
25823                    }
25824                };
25825                let sheet_name = self.graph.sheet_name(sheet_id);
25826
25827                let bounded_range = if range.start_row.is_some()
25828                    && range.start_col.is_some()
25829                    && range.end_row.is_some()
25830                    && range.end_col.is_some()
25831                {
25832                    Some(RangeRef::try_from_shared(range.as_ref())?)
25833                } else {
25834                    None
25835                };
25836
25837                let sr = bounded_range
25838                    .as_ref()
25839                    .map(|r| r.start.coord.row() + 1)
25840                    .or_else(|| range.start_row.map(|b| b.index + 1));
25841                let sc = bounded_range
25842                    .as_ref()
25843                    .map(|r| r.start.coord.col() + 1)
25844                    .or_else(|| range.start_col.map(|b| b.index + 1));
25845                let er = bounded_range
25846                    .as_ref()
25847                    .map(|r| r.end.coord.row() + 1)
25848                    .or_else(|| range.end_row.map(|b| b.index + 1));
25849                let ec = bounded_range
25850                    .as_ref()
25851                    .map(|r| r.end.coord.col() + 1)
25852                    .or_else(|| range.end_col.map(|b| b.index + 1));
25853
25854                let extent = resolve_used_extent_with_fallback(
25855                    OpenRangeBounds {
25856                        start_row: sr,
25857                        start_column: sc,
25858                        end_row: er,
25859                        end_column: ec,
25860                    },
25861                    ExtentPolicy::EvaluationCompat {
25862                        fallback_row: None,
25863                        fallback_column: None,
25864                    },
25865                    || {
25866                        self.sheet_bounds(sheet_name)
25867                            .map(|_| self.config.max_open_ended_rows)
25868                    },
25869                    || {
25870                        self.sheet_bounds(sheet_name)
25871                            .map(|_| self.config.max_open_ended_cols)
25872                    },
25873                    |first, last| self.used_rows_for_columns(sheet_name, first, last),
25874                    |first, last| self.used_cols_for_rows(sheet_name, first, last),
25875                );
25876                let (sr, sc, er, ec) = extent
25877                    .map(|extent| {
25878                        (
25879                            extent.start_row,
25880                            extent.start_column,
25881                            extent.end_row,
25882                            extent.end_column,
25883                        )
25884                    })
25885                    .unwrap_or((1, 1, 0, 0));
25886
25887                if self.force_materialize_range_views {
25888                    if er < sr || ec < sc {
25889                        return Ok(RangeView::from_owned_rows(
25890                            Vec::new(),
25891                            self.config.date_system,
25892                        ));
25893                    }
25894                    let h = (er - sr + 1) as u64;
25895                    let w = (ec - sc + 1) as u64;
25896                    let cell_count = h.saturating_mul(w);
25897                    if cell_count <= self.config.spill.max_spill_cells as u64 {
25898                        let mut rows: Vec<Vec<LiteralValue>> = Vec::with_capacity(h as usize);
25899                        for r in sr..=er {
25900                            let mut rowv: Vec<LiteralValue> = Vec::with_capacity(w as usize);
25901                            for c in sc..=ec {
25902                                rowv.push(
25903                                    self.get_cell_value(sheet_name, r, c)
25904                                        .unwrap_or(LiteralValue::Empty),
25905                                );
25906                            }
25907                            rows.push(rowv);
25908                        }
25909                        return Ok(RangeView::from_owned_rows(rows, self.config.date_system));
25910                    }
25911                }
25912
25913                let Some(asheet) = self.sheet_store().sheet(sheet_name) else {
25914                    return Ok(RangeView::from_owned_rows(
25915                        Vec::new(),
25916                        self.config.date_system,
25917                    ));
25918                };
25919
25920                let rv = if er < sr || ec < sc {
25921                    asheet.range_view(1, 1, 0, 0)
25922                } else {
25923                    let sr0 = sr.saturating_sub(1) as usize;
25924                    let sc0 = sc.saturating_sub(1) as usize;
25925                    let er0 = er.saturating_sub(1) as usize;
25926                    let ec0 = ec.saturating_sub(1) as usize;
25927                    asheet.range_view(sr0, sc0, er0, ec0)
25928                };
25929
25930                Ok(rv)
25931            }
25932            ReferenceType::Cell { .. } => {
25933                let shared = self.resolve_shared_ref(reference, current_sheet)?;
25934                let formualizer_common::SheetRef::Cell(cell) = shared else {
25935                    return Err(ExcelError::new(ExcelErrorKind::Ref));
25936                };
25937                let addr = CellRef::try_from_shared(cell)?;
25938                let sheet_id = addr.sheet_id;
25939                let sheet_name = self.graph.sheet_name(sheet_id);
25940                let row = addr.coord.row() + 1;
25941                let col = addr.coord.col() + 1;
25942
25943                if self.force_materialize_range_views {
25944                    let v = self
25945                        .get_cell_value(sheet_name, row, col)
25946                        .unwrap_or(LiteralValue::Empty);
25947                    return Ok(RangeView::from_owned_rows(
25948                        vec![vec![v]],
25949                        self.config.date_system,
25950                    ));
25951                }
25952
25953                if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
25954                    let r0 = row.saturating_sub(1) as usize;
25955                    let c0 = col.saturating_sub(1) as usize;
25956                    let rv = asheet.range_view(r0, c0, r0, c0);
25957                    Ok(rv)
25958                } else {
25959                    let v = self
25960                        .get_cell_value(sheet_name, row, col)
25961                        .unwrap_or(LiteralValue::Empty);
25962                    Ok(RangeView::from_owned_rows(
25963                        vec![vec![v]],
25964                        self.config.date_system,
25965                    ))
25966                }
25967            }
25968            ReferenceType::NamedRange(name) => {
25969                if let Some(current_id) = self.graph.sheet_id(current_sheet)
25970                    && let Some(named) = self.graph.resolve_name_entry(name, current_id)
25971                {
25972                    match &named.definition {
25973                        NamedDefinition::Cell(cell_ref) => {
25974                            let sheet_name = self.graph.sheet_name(cell_ref.sheet_id);
25975                            if self.force_materialize_range_views {
25976                                let v = self
25977                                    .get_cell_value(
25978                                        sheet_name,
25979                                        cell_ref.coord.row() + 1,
25980                                        cell_ref.coord.col() + 1,
25981                                    )
25982                                    .unwrap_or(LiteralValue::Empty);
25983                                return Ok(RangeView::from_owned_rows(
25984                                    vec![vec![v]],
25985                                    self.config.date_system,
25986                                ));
25987                            } else {
25988                                let asheet = self
25989                                    .sheet_store()
25990                                    .sheet(sheet_name)
25991                                    .expect("Arrow sheet missing for named cell");
25992                                let r0 = cell_ref.coord.row() as usize;
25993                                let c0 = cell_ref.coord.col() as usize;
25994                                let rv = asheet.range_view(r0, c0, r0, c0);
25995                                return Ok(rv);
25996                            }
25997                        }
25998                        NamedDefinition::Range(range_ref) => {
25999                            let sheet_name = self.graph.sheet_name(range_ref.start.sheet_id);
26000                            let sr = range_ref.start.coord.row() + 1;
26001                            let sc = range_ref.start.coord.col() + 1;
26002                            let er = range_ref.end.coord.row() + 1;
26003                            let ec = range_ref.end.coord.col() + 1;
26004                            if self.force_materialize_range_views {
26005                                let h = (er.saturating_sub(sr) + 1) as u64;
26006                                let w = (ec.saturating_sub(sc) + 1) as u64;
26007                                let cell_count = h.saturating_mul(w);
26008                                if cell_count <= self.config.spill.max_spill_cells as u64 {
26009                                    let mut rows: Vec<Vec<LiteralValue>> =
26010                                        Vec::with_capacity(h as usize);
26011                                    for r in sr..=er {
26012                                        let mut rowv: Vec<LiteralValue> =
26013                                            Vec::with_capacity(w as usize);
26014                                        for c in sc..=ec {
26015                                            rowv.push(
26016                                                self.get_cell_value(sheet_name, r, c)
26017                                                    .unwrap_or(LiteralValue::Empty),
26018                                            );
26019                                        }
26020                                        rows.push(rowv);
26021                                    }
26022                                    return Ok(RangeView::from_owned_rows(
26023                                        rows,
26024                                        self.config.date_system,
26025                                    ));
26026                                }
26027                            }
26028                            let asheet = self
26029                                .sheet_store()
26030                                .sheet(sheet_name)
26031                                .expect("Arrow sheet missing for named range");
26032                            let sr0 = range_ref.start.coord.row() as usize;
26033                            let sc0 = range_ref.start.coord.col() as usize;
26034                            let er0 = range_ref.end.coord.row() as usize;
26035                            let ec0 = range_ref.end.coord.col() as usize;
26036                            let rv = asheet.range_view(sr0, sc0, er0, ec0);
26037                            return Ok(rv);
26038                        }
26039                        NamedDefinition::Literal(v) => {
26040                            return Ok(RangeView::from_owned_rows(
26041                                vec![vec![v.clone()]],
26042                                self.config.date_system,
26043                            ));
26044                        }
26045                        NamedDefinition::Formula { .. } => {
26046                            if let Some(value) = self.graph.get_value(named.vertex) {
26047                                return Ok(RangeView::from_owned_rows(
26048                                    vec![vec![value]],
26049                                    self.config.date_system,
26050                                ));
26051                            }
26052                        }
26053                    }
26054                }
26055
26056                if let Some(source) = self.graph.resolve_source_scalar_entry(name) {
26057                    let version = source
26058                        .version
26059                        .or_else(|| self.resolver.source_scalar_version(name));
26060                    let v = self.resolve_source_scalar_cached(name, version)?;
26061                    return Ok(RangeView::from_owned_rows(
26062                        vec![vec![v]],
26063                        self.config.date_system,
26064                    ));
26065                }
26066
26067                let data = self.resolver.resolve_named_range_reference(name)?;
26068                Ok(RangeView::from_owned_rows(data, self.config.date_system))
26069            }
26070            ReferenceType::Table(tref) => {
26071                if let Some(table) = self.graph.resolve_table_entry(&tref.name) {
26072                    let sheet_name = self.graph.sheet_name(table.range.start.sheet_id);
26073                    let asheet = self
26074                        .sheet_store()
26075                        .sheet(sheet_name)
26076                        .expect("Arrow sheet missing for table reference");
26077
26078                    let sr0 = table.range.start.coord.row() as usize;
26079                    let sc0 = table.range.start.coord.col() as usize;
26080                    let er0 = table.range.end.coord.row() as usize;
26081                    let ec0 = table.range.end.coord.col() as usize;
26082
26083                    let has_totals = table.totals_row;
26084                    let has_headers = table.header_row;
26085                    let data_sr = if has_headers {
26086                        sr0.saturating_add(1)
26087                    } else {
26088                        sr0
26089                    };
26090                    let data_er = if has_totals {
26091                        er0.saturating_sub(1)
26092                    } else {
26093                        er0
26094                    };
26095
26096                    let select = |sr: usize, sc: usize, er: usize, ec: usize| {
26097                        if sr > er || sc > ec {
26098                            asheet.range_view(1, 1, 0, 0)
26099                        } else {
26100                            asheet.range_view(sr, sc, er, ec)
26101                        }
26102                    };
26103
26104                    let av = match &tref.specifier {
26105                        None => {
26106                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26107                                "Table reference without specifier is unsupported".to_string(),
26108                            ));
26109                        }
26110                        Some(formualizer_parse::parser::TableSpecifier::Column(col)) => {
26111                            let Some(idx) = table.col_index(col) else {
26112                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26113                                    "Column refers to unknown table column".to_string(),
26114                                ));
26115                            };
26116                            let c0 = sc0 + idx;
26117                            select(data_sr, c0, data_er, c0)
26118                        }
26119                        Some(formualizer_parse::parser::TableSpecifier::ColumnRange(
26120                            start,
26121                            end,
26122                        )) => {
26123                            let Some(si) = table.col_index(start) else {
26124                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26125                                    "Column range refers to unknown column(s)".to_string(),
26126                                ));
26127                            };
26128                            let Some(ei) = table.col_index(end) else {
26129                                return Err(ExcelError::new(ExcelErrorKind::Ref).with_message(
26130                                    "Column range refers to unknown column(s)".to_string(),
26131                                ));
26132                            };
26133                            let (mut a, mut b) = (si, ei);
26134                            if a > b {
26135                                std::mem::swap(&mut a, &mut b);
26136                            }
26137                            let c_start = sc0 + a;
26138                            let c_end = sc0 + b;
26139                            select(data_sr, c_start, data_er, c_end)
26140                        }
26141                        Some(formualizer_parse::parser::TableSpecifier::All)
26142                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26143                            formualizer_parse::parser::SpecialItem::All,
26144                        )) => select(sr0, sc0, er0, ec0),
26145                        Some(formualizer_parse::parser::TableSpecifier::Data)
26146                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26147                            formualizer_parse::parser::SpecialItem::Data,
26148                        )) => select(data_sr, sc0, data_er, ec0),
26149                        Some(formualizer_parse::parser::TableSpecifier::Headers)
26150                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26151                            formualizer_parse::parser::SpecialItem::Headers,
26152                        )) => {
26153                            if !has_headers {
26154                                asheet.range_view(1, 1, 0, 0)
26155                            } else {
26156                                select(sr0, sc0, sr0, ec0)
26157                            }
26158                        }
26159                        Some(formualizer_parse::parser::TableSpecifier::Totals)
26160                        | Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26161                            formualizer_parse::parser::SpecialItem::Totals,
26162                        )) => {
26163                            if !has_totals {
26164                                asheet.range_view(1, 1, 0, 0)
26165                            } else {
26166                                select(er0, sc0, er0, ec0)
26167                            }
26168                        }
26169                        Some(formualizer_parse::parser::TableSpecifier::SpecialItem(
26170                            formualizer_parse::parser::SpecialItem::ThisRow,
26171                        )) => {
26172                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26173                                "@ (This Row) requires table-aware context; not yet supported"
26174                                    .to_string(),
26175                            ));
26176                        }
26177                        Some(formualizer_parse::parser::TableSpecifier::Row(_))
26178                        | Some(formualizer_parse::parser::TableSpecifier::Combination(_)) => {
26179                            return Err(ExcelError::new(ExcelErrorKind::NImpl).with_message(
26180                                "Complex structured references not yet supported".to_string(),
26181                            ));
26182                        }
26183                    };
26184
26185                    return Ok(av);
26186                }
26187
26188                if let Some(source) = self.graph.resolve_source_table_entry(&tref.name) {
26189                    let version = source
26190                        .version
26191                        .or_else(|| self.resolver.source_table_version(&tref.name));
26192                    let table = self.resolve_source_table_cached(&tref.name, version)?;
26193                    return self.source_table_to_range_view(table.as_ref(), &tref.specifier);
26194                }
26195
26196                // Fallback: materialize via Resolver::resolve_range_like tranche 1
26197                let boxed = self.resolve_range_like(&ReferenceType::Table(tref.clone()))?;
26198                let owned = boxed.materialise().into_owned();
26199                Ok(RangeView::from_owned_rows(owned, self.config.date_system))
26200            }
26201            ReferenceType::Cell3D { .. } | ReferenceType::Range3D { .. } => {
26202                Err(ExcelError::new(ExcelErrorKind::NImpl)
26203                    .with_message("3D references are not yet supported".to_string()))
26204            }
26205        }
26206    }
26207
26208    fn resolve_cell_format(
26209        &self,
26210        sheet: Option<&str>,
26211        row: u32,
26212        col: u32,
26213        current_sheet: &str,
26214    ) -> Option<crate::format::FormatId> {
26215        self.effective_format_id(sheet.unwrap_or(current_sheet), row, col)
26216    }
26217
26218    fn format_class(
26219        &self,
26220        format: crate::format::FormatId,
26221    ) -> Option<formualizer_common::numfmt::FormatClass> {
26222        self.format_registry.class(format).cloned()
26223    }
26224
26225    fn record_cell_derived_format(
26226        &self,
26227        sheet: &str,
26228        row: u32,
26229        col: u32,
26230        format: Option<crate::format::FormatId>,
26231    ) {
26232        if let Some(sheet_id) = self.graph.sheet_id(sheet) {
26233            let cell = CellRef::new(sheet_id, Coord::from_excel(row, col, true, true));
26234            self.record_derived_format_at(cell, format);
26235        }
26236    }
26237
26238    fn resolve_cell_reference_value(
26239        &self,
26240        sheet: Option<&str>,
26241        row: u32,
26242        col: u32,
26243        current_sheet: &str,
26244    ) -> Result<LiteralValue, ExcelError> {
26245        let sheet_name = sheet.unwrap_or(current_sheet);
26246        if self.graph.sheet_id(sheet_name).is_none() {
26247            return Err(ExcelError::new(ExcelErrorKind::Ref));
26248        }
26249        Ok(self
26250            .get_cell_value(sheet_name, row, col)
26251            .unwrap_or(LiteralValue::Empty))
26252    }
26253
26254    fn build_criteria_mask(
26255        &self,
26256        view: &RangeView<'_>,
26257        col_in_view: usize,
26258        pred: &crate::args::CriteriaPredicate,
26259    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
26260        #[cfg(any(test, feature = "test-support"))]
26261        criteria_mask_test_hooks::note_mask(view.dims().0);
26262        if view.dims().1 == 0 {
26263            return None;
26264        }
26265        // If the view is logically open-ended but the backing sheet has no physical rows,
26266        // treat the mask as empty (0-len) rather than attempting to build a huge mask.
26267        let sheet_rows = view.sheet().nrows as usize;
26268        if sheet_rows == 0 || view.start_row() >= sheet_rows {
26269            return Some(std::sync::Arc::new(arrow_array::BooleanArray::new_null(0)));
26270        }
26271        compute_criteria_mask(view, col_in_view, pred)
26272    }
26273
26274    fn build_row_visibility_mask(
26275        &self,
26276        view: &RangeView<'_>,
26277        mode: VisibilityMaskMode,
26278    ) -> Option<std::sync::Arc<arrow_array::BooleanArray>> {
26279        self.build_row_visibility_mask_for_view(view, mode)
26280    }
26281}
26282
26283impl<R> Engine<R>
26284where
26285    R: EvaluationContext,
26286{
26287    fn clear_spill_projection_and_mirror(
26288        &mut self,
26289        anchor_vertex: VertexId,
26290        delta: Option<&mut DeltaCollector>,
26291    ) {
26292        let spill_cells = self
26293            .graph
26294            .spill_cells_for_anchor(anchor_vertex)
26295            .map(|cells| cells.to_vec())
26296            .unwrap_or_default();
26297        if spill_cells.is_empty() {
26298            return;
26299        }
26300
26301        if let Some(delta) = delta
26302            && delta.mode != DeltaMode::Off
26303        {
26304            let empty = LiteralValue::Empty;
26305            for cell in spill_cells.iter() {
26306                let sheet_name = self.graph.sheet_name(cell.sheet_id);
26307                let old = self
26308                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26309                    .unwrap_or(LiteralValue::Empty);
26310                if old != empty {
26311                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26312                }
26313            }
26314        }
26315
26316        self.graph.clear_spill_region(anchor_vertex);
26317        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
26318            self.record_formula_plane_structural_change(scope);
26319        }
26320
26321        if self.config.arrow_storage_enabled
26322            && self.config.delta_overlay_enabled
26323            && self.config.write_formula_overlay_enabled
26324        {
26325            let empty = LiteralValue::Empty;
26326            for cell in spill_cells.iter() {
26327                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
26328                self.mirror_value_to_computed_overlay(
26329                    &sheet_name,
26330                    cell.coord.row() + 1,
26331                    cell.coord.col() + 1,
26332                    &empty,
26333                );
26334            }
26335        }
26336    }
26337
26338    /// Apply the evaluation outcome for one cyclic SCC: stamp `#CIRC!` on its
26339    /// (optionally filtered) members via `stamp_cycle_error`.
26340    ///
26341    /// This is the single per-SCC application point used by every schedule
26342    /// consumer walking `Schedule::units` (pre-work for #112, where cyclic
26343    /// SCCs will gain runtime verdicts instead of an unconditional stamp).
26344    ///
26345    /// `dirty_filter` preserves the recalc-plan quirk: when `Some(dirty)`,
26346    /// only members present in the set are stamped.
26347    ///
26348    /// Returns the number of vertices stamped (0 when a filter excludes every
26349    /// member), so callers can keep their site-specific `cycle_errors`
26350    /// accounting.
26351    fn apply_cycle_outcome(
26352        &mut self,
26353        cycle: &[VertexId],
26354        mut delta: Option<&mut DeltaCollector>,
26355        dirty_filter: Option<&FxHashSet<VertexId>>,
26356    ) -> usize {
26357        let circ_error = LiteralValue::Error(
26358            ExcelError::new(ExcelErrorKind::Circ)
26359                .with_message("Circular dependency detected".to_string()),
26360        );
26361        let mut stamped = 0usize;
26362        for &vertex_id in cycle {
26363            if let Some(filter) = dirty_filter
26364                && !filter.contains(&vertex_id)
26365            {
26366                continue;
26367            }
26368            self.stamp_cycle_error(vertex_id, &circ_error, delta.as_deref_mut());
26369            stamped += 1;
26370        }
26371        stamped
26372    }
26373
26374    /// Stamp a vertex with `#CIRC!` as part of cycle handling.
26375    ///
26376    /// Unlike a bare `update_vertex_value`, this first tears down any spill the
26377    /// vertex previously anchored: it clears the spilled cells, releases the graph
26378    /// spill registry, drops any lingering region reservation, and mirrors the
26379    /// cleared cells into the computed overlay — the same teardown a normal scalar/
26380    /// error result performs (see `apply_non_array_result_from_parallel` /
26381    /// `clear_spill_projection_and_mirror`). Without this, a #CIRC stamp on a former
26382    /// spill anchor would leave stale spilled values and a reserved region behind
26383    /// (issue #111).
26384    ///
26385    /// When `delta` is provided, the cleared spill cells are recorded (by
26386    /// `clear_spill_projection_and_mirror`) and the anchor's own #CIRC change is
26387    /// recorded here, matching how other result paths emit deltas.
26388    fn stamp_cycle_error(
26389        &mut self,
26390        vertex_id: VertexId,
26391        circ_error: &LiteralValue,
26392        mut delta: Option<&mut DeltaCollector>,
26393    ) {
26394        // Tear down any previous spill projection/region before overwriting the anchor.
26395        if self.graph.spill_registry_has_anchor(vertex_id) {
26396            self.clear_spill_projection_and_mirror(vertex_id, delta.as_deref_mut());
26397        }
26398        // Drop any reservation that was never committed (defensive; normally released
26399        // on the prior successful commit).
26400        self.spill_mgr.release_owner(vertex_id);
26401
26402        // Record the anchor's own #CIRC delta, like other result paths.
26403        if let Some(d) = delta
26404            && d.mode != DeltaMode::Off
26405            && let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id)
26406        {
26407            let sheet_name = self.graph.sheet_name(cell.sheet_id);
26408            let old = self
26409                .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26410                .unwrap_or(LiteralValue::Empty);
26411            if old != *circ_error {
26412                d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26413            }
26414        }
26415
26416        self.graph
26417            .update_vertex_value(vertex_id, circ_error.clone());
26418        self.mirror_vertex_value_to_overlay(vertex_id, circ_error);
26419    }
26420
26421    /// Dispatch point for one `ScheduleUnit::Cycle` (RFC #112, Stage 2).
26422    ///
26423    /// * `CycleDetection::Static` — today's behavior, byte-for-byte: stamp
26424    ///   `#CIRC!` on the (optionally dirty-filtered) members.
26425    /// * `CycleDetection::Runtime` — evaluate the SCC via
26426    ///   [`Self::evaluate_scc_unit`]. The recalc-plan dirty quirk maps to:
26427    ///   no dirty member → skip the task entirely (values stand); any dirty
26428    ///   member → the whole SCC evaluates (an SCC cannot be partially
26429    ///   evaluated).
26430    ///
26431    /// Returns the number of `#CIRC!`-stamped vertices, so call sites can
26432    /// keep their `cycle_errors` accounting (`> 0` ⇒ count the unit).
26433    fn handle_cycle_unit(
26434        &mut self,
26435        cycle: &[VertexId],
26436        mut delta: Option<&mut DeltaCollector>,
26437        dirty_filter: Option<&FxHashSet<VertexId>>,
26438        cancel_flag: Option<&AtomicBool>,
26439    ) -> Result<usize, ExcelError> {
26440        self.resource_checkpoint(cycle.len() as u64)?;
26441        match self.config.cycle.detection {
26442            CycleDetection::Static => {
26443                Ok(self.apply_cycle_outcome(cycle, delta.as_deref_mut(), dirty_filter))
26444            }
26445            CycleDetection::Runtime => {
26446                if let Some(filter) = dirty_filter
26447                    && !cycle.iter().any(|v| filter.contains(v))
26448                {
26449                    return Ok(0);
26450                }
26451                // Both policies share `evaluate_scc_unit`; they differ only
26452                // in the settle loop's live-cycle arm (Error stamps,
26453                // Iterate keeps passing — RFC #113).
26454                self.evaluate_scc_unit(cycle, delta, cancel_flag)
26455            }
26456        }
26457    }
26458
26459    /// Evaluate one statically-cyclic SCC under `CycleDetection::Runtime`
26460    /// (design doc `formualizer-stage2-scc-evaluation-design.md` §3; contract
26461    /// spec §3; Iterate policy arm per RFC #113).
26462    ///
26463    /// Phantom SCCs (live-acyclic) produce ordinary values under both
26464    /// policies; live cycles get `#CIRC!` with live-cycle-only blast radius
26465    /// under `CyclePolicy::Error`, or Excel-style iterative calculation
26466    /// (converge per spec §6 or cap at `max_iterations` passes) under
26467    /// `CyclePolicy::Iterate`. Runs sequentially on the
26468    /// coordinating thread; commits are write-through per member (no
26469    /// `ComputedWriteBuffer` — that buffer is scoped to layer evaluation and
26470    /// always flushed before a Cycle unit runs, G1), so later members' scalar
26471    /// *and* range reads observe earlier members' results through the overlay
26472    /// cascade. Deltas are recorded once per member at end of task (G11).
26473    ///
26474    /// Returns the number of vertices stamped `#CIRC!`.
26475    ///
26476    /// `pub(crate)` so tests can drive SCC shapes (e.g. name-vertex members)
26477    /// that ingest-time cycle rejection makes unreachable via public edits.
26478    pub(crate) fn evaluate_scc_unit(
26479        &mut self,
26480        cycle: &[VertexId],
26481        mut delta: Option<&mut DeltaCollector>,
26482        cancel_flag: Option<&AtomicBool>,
26483    ) -> Result<usize, ExcelError> {
26484        struct SccMember {
26485            vertex: VertexId,
26486            cell: Option<CellRef>,
26487        }
26488
26489        let task_start = crate::instant::FzInstant::now();
26490
26491        // ── 0. Member order (spec §7.13): cells ascending (sheet, row, col);
26492        // name vertices after, lexicographic by folded canonical name; any
26493        // other vertex kind (defensive — `get_evaluation_vertices` only emits
26494        // formula/name kinds) last by id, never evaluated.
26495        let mut cell_members: Vec<(VertexId, CellRef)> = Vec::new();
26496        let mut name_members: Vec<(VertexId, String)> = Vec::new();
26497        let mut other_members: Vec<VertexId> = Vec::new();
26498        for &v in cycle {
26499            match self.graph.get_vertex_kind(v) {
26500                VertexKind::FormulaScalar | VertexKind::FormulaArray => {
26501                    match self.graph.get_cell_ref(v) {
26502                        Some(cell) => cell_members.push((v, cell)),
26503                        None => other_members.push(v),
26504                    }
26505                }
26506                VertexKind::NamedScalar | VertexKind::NamedArray => {
26507                    match self.graph.name_key_for_vertex(v) {
26508                        Some(key) => name_members.push((v, key)),
26509                        None => other_members.push(v),
26510                    }
26511                }
26512                _ => other_members.push(v),
26513            }
26514        }
26515        cell_members.sort_unstable_by_key(|(_, c)| (c.sheet_id, c.coord.row(), c.coord.col()));
26516        name_members.sort_unstable_by(|(av, ak), (bv, bk)| ak.cmp(bk).then(av.cmp(bv)));
26517        other_members.sort_unstable();
26518
26519        let cell_refs: Vec<CellRef> = cell_members.iter().map(|(_, c)| *c).collect();
26520        let name_keys: Vec<String> = name_members.iter().map(|(_, k)| k.clone()).collect();
26521        let mut members: Vec<SccMember> = Vec::with_capacity(cycle.len());
26522        for (v, c) in &cell_members {
26523            members.push(SccMember {
26524                vertex: *v,
26525                cell: Some(*c),
26526            });
26527        }
26528        for (v, _) in &name_members {
26529            members.push(SccMember {
26530                vertex: *v,
26531                cell: None,
26532            });
26533        }
26534        for v in &other_members {
26535            members.push(SccMember {
26536                vertex: *v,
26537                cell: None,
26538            });
26539        }
26540        let n = members.len();
26541        // Indices addressable by the collector (cells + names); `other`
26542        // members can be neither edge sources nor targets.
26543        let recordable = cell_refs.len() + name_keys.len();
26544
26545        let circ_error = LiteralValue::Error(
26546            ExcelError::new(ExcelErrorKind::Circ)
26547                .with_message("Circular dependency detected".to_string()),
26548        );
26549
26550        // ── 0b. Spec-§4 persistence repair: structural edits clear computed
26551        // overlays wholesale (`clear_computed_overlay_after_row/_col`), but
26552        // an iterating member's committed value is cycle STATE, not a
26553        // recomputable cache — and in canonical mode the overlay is its ONLY
26554        // home. If the overlay entry vanished since the last recalc, re-seed
26555        // it from the end-of-recalc snapshot (`iterative_state_values`) so
26556        // pass-1 reads (scalar AND range, via the overlay cascade) observe
26557        // the persisted value instead of silently restarting at Empty→0.
26558        // (Found by the iterate edge corpus: inserting/deleting an unrelated
26559        // row reset accumulators, violating spec §4/§7.15.)
26560        if !self.iterative_state_values.is_empty() {
26561            let restore: Vec<(VertexId, LiteralValue)> = members
26562                .iter()
26563                .filter_map(|m| {
26564                    let cell = m.cell?;
26565                    let persisted = self.iterative_state_values.get(&m.vertex)?;
26566                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
26567                    let overlay = self
26568                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26569                        .unwrap_or(LiteralValue::Empty);
26570                    if matches!(overlay, LiteralValue::Empty) {
26571                        Some((m.vertex, persisted.clone()))
26572                    } else {
26573                        None
26574                    }
26575                })
26576                .collect();
26577            for (vertex, value) in restore {
26578                self.mirror_vertex_value_to_overlay(vertex, &value);
26579            }
26580        }
26581
26582        // ── 1. Pre-task value snapshot (overlay-first for cells — G3; the
26583        // graph value map may be evicted in value-cache-disabled mode).
26584        let snapshot: Vec<LiteralValue> = members
26585            .iter()
26586            .map(|m| match m.cell {
26587                Some(cell) => {
26588                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
26589                    self.get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
26590                        .unwrap_or(LiteralValue::Empty)
26591                }
26592                None => self
26593                    .graph
26594                    .get_value(m.vertex)
26595                    .unwrap_or(LiteralValue::Empty),
26596            })
26597            .collect();
26598
26599        // ── 2. Pre-scan: spill anchors (FormulaArray) are stamped `#CIRC!`
26600        // with full spill teardown (spec §7.9, #115) and excluded from
26601        // evaluation. They stay recordable edge TARGETS (readers see `#CIRC!`
26602        // and propagate). Non-evaluable defensive members are excluded too.
26603        let mut excluded = vec![false; n];
26604        let mut last_value = snapshot.clone();
26605        let mut stamped = 0usize;
26606        for (i, m) in members.iter().enumerate() {
26607            match self.graph.get_vertex_kind(m.vertex) {
26608                VertexKind::FormulaArray => {
26609                    // Deltas for the cleared spill-region cells (non-members)
26610                    // can only be recorded here; the anchor's own delta is
26611                    // covered by the end-of-task snapshot comparison (dedup).
26612                    self.stamp_cycle_error(m.vertex, &circ_error, delta.as_deref_mut());
26613                    excluded[i] = true;
26614                    last_value[i] = circ_error.clone();
26615                    stamped += 1;
26616                }
26617                VertexKind::FormulaScalar | VertexKind::NamedScalar | VertexKind::NamedArray => {}
26618                _ => excluded[i] = true,
26619            }
26620        }
26621
26622        let collector = LiveEdgeCollector::new_with_names(&cell_refs, &name_keys);
26623
26624        // Per-member live out-edges, refreshed whenever a member re-runs.
26625        let mut out_edges: Vec<Vec<u32>> = vec![Vec::new(); n];
26626        // Position of each member in the most recent pass (-1 = did not run).
26627        let mut pos: Vec<i64> = vec![-1; n];
26628        // Whether each member's committed value changed in the most recent pass.
26629        let mut changed = vec![false; n];
26630
26631        // Evaluate-and-commit one member; returns Ok(true) when the member was
26632        // stamped `#CIRC!` (array result — would-be spill anchor, spec §7.9).
26633        macro_rules! run_member {
26634            ($i:expr) => {{
26635                let i: usize = $i;
26636                let m = &members[i];
26637                if i < recordable {
26638                    collector.set_current(i as u32);
26639                }
26640                let value = {
26641                    let ctx = RecordingContext::new(&*self, &collector);
26642                    match self.evaluate_vertex_recorded(m.vertex, &ctx, &collector) {
26643                        Ok(v) => v,
26644                        Err(e) => LiteralValue::Error(e),
26645                    }
26646                };
26647                let is_cell_formula = m.cell.is_some();
26648                if is_cell_formula && matches!(value, LiteralValue::Array(_)) {
26649                    // A member that *would* spill inside an SCC gets the
26650                    // conservative §7.9 verdict. It has never spilled before
26651                    // (a prior spill would make it FormulaArray, pre-stamped
26652                    // above), so there is no projection to tear down.
26653                    self.stamp_cycle_error(m.vertex, &circ_error, None);
26654                    excluded[i] = true;
26655                    stamped += 1;
26656                    changed[i] = last_value[i] != circ_error;
26657                    last_value[i] = circ_error.clone();
26658                } else {
26659                    self.graph.update_vertex_value(m.vertex, value.clone());
26660                    self.mirror_vertex_value_to_overlay(m.vertex, &value);
26661                    // §7.14 invariant (G2): a formula member must never be
26662                    // shadowed by a user/delta overlay entry, or iteration
26663                    // reads would silently diverge from committed values.
26664                    #[cfg(debug_assertions)]
26665                    if let Some(cell) = m.cell {
26666                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
26667                        debug_assert!(
26668                            self.read_delta_overlay_cell(
26669                                &sheet_name,
26670                                cell.coord.row() + 1,
26671                                cell.coord.col() + 1
26672                            )
26673                            .is_none(),
26674                            "user overlay must never shadow a formula SCC member ({sheet_name}!r{}c{})",
26675                            cell.coord.row() + 1,
26676                            cell.coord.col() + 1
26677                        );
26678                    }
26679                    changed[i] = last_value[i] != value;
26680                    last_value[i] = value;
26681                }
26682            }};
26683        }
26684
26685        let check_cancel = |flag: Option<&AtomicBool>| -> Result<(), ExcelError> {
26686            if let Some(flag) = flag
26687                && flag.load(Ordering::Relaxed)
26688            {
26689                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
26690                    .with_message("Evaluation cancelled during SCC evaluation".to_string()));
26691            }
26692            Ok(())
26693        };
26694
26695        // ── 3. Pass 1: all evaluable members in member order.
26696        check_cancel(cancel_flag)?;
26697        let mut passes = 1usize;
26698        {
26699            let mut p = 0i64;
26700            for i in 0..n {
26701                if excluded[i] {
26702                    continue;
26703                }
26704                run_member!(i);
26705                pos[i] = p;
26706                p += 1;
26707            }
26708        }
26709
26710        // ── 4. Settle loop (design doc §3 step 4; RFC #113 policy arm).
26711        //
26712        // Acyclic classifications settle stale readers exactly (identical
26713        // under both policies — phantom SCCs never iterate). A witnessed
26714        // live cycle dispatches on policy: `Error` stamps `#CIRC!` and
26715        // stops; `Iterate` keeps running full passes over all members in
26716        // member order until converged (spec §6) or capped at
26717        // `max_iterations` total passes. A live cycle that only appears
26718        // mid-settle takes the same arm, and a cycle that dissolves
26719        // mid-iteration falls back to exact acyclic settling.
26720        //
26721        // Defensive acyclic budget: the acyclic settle is monotone, so more
26722        // than |SCC| + 2 settle passes can only be a bug; cap hits stamp the
26723        // remainder and set telemetry. Tracked via `settle_passes` so
26724        // iteration passes (legitimately many) don't consume the budget.
26725        let policy = self.config.cycle.policy;
26726        let cap = n + 2;
26727        let mut witnessed_cycles = 0usize;
26728        let mut capped = false;
26729        // ── Iterate-policy state ──
26730        let mut iterating = false;
26731        let mut converged = false;
26732        let mut exact_fixed_point = false;
26733        // Values committed by the last *full* pass; `None` until the first
26734        // iteration pass runs (pass 1 has no predecessor to compare against)
26735        // and reset when a settle pass runs (no cross-kind comparisons).
26736        let mut prev_pass: Option<Vec<LiteralValue>> = None;
26737        // Final-round convergence stats (overwritten per round so the values
26738        // reported are the ones observed at stop).
26739        let mut iter_max_delta = 0f64;
26740        let mut iter_nan_converged = 0usize;
26741        // Acyclic stale-reader re-eval passes (defensive budget; under pure
26742        // Error flow `1 + settle_passes == passes`, preserving Stage-2
26743        // behavior exactly).
26744        let mut settle_passes = 0usize;
26745        loop {
26746            // Drain this pass's recordings; members that ran replace their
26747            // out-edge set, members that didn't keep last-known edges.
26748            let drained = collector.take_edges();
26749            for i in 0..n {
26750                if pos[i] >= 0 {
26751                    out_edges[i].clear();
26752                }
26753            }
26754            for (from, to) in drained {
26755                debug_assert!(
26756                    pos[from as usize] >= 0,
26757                    "edge from a member that did not run"
26758                );
26759                out_edges[from as usize].push(to);
26760            }
26761            let mut edges: Vec<(u32, u32)> = Vec::new();
26762            for (i, outs) in out_edges.iter().enumerate() {
26763                if excluded[i] {
26764                    continue;
26765                }
26766                for &t in outs {
26767                    edges.push((i as u32, t));
26768                }
26769            }
26770            edges.sort_unstable();
26771            edges.dedup();
26772
26773            let analysis = analyze_live_graph(n, &edges);
26774
26775            if analysis.cycle_count > 0 {
26776                // Classification repeats every iteration pass under
26777                // `Iterate`; record the widest single witness instead of
26778                // accumulating so the count stays "distinct live cycles".
26779                witnessed_cycles = witnessed_cycles.max(analysis.cycle_count);
26780                match policy {
26781                    CyclePolicy::Error => {
26782                        // POLICY (Error): stamp every member of a live cycle,
26783                        // then one settling pass over the remaining members in
26784                        // live-topological order so error propagation
26785                        // downstream is consistent (spec §3.4). Blast radius =
26786                        // live cycles only.
26787                        for i in 0..n {
26788                            if analysis.in_cycle[i] && !excluded[i] {
26789                                self.stamp_cycle_error(members[i].vertex, &circ_error, None);
26790                                excluded[i] = true;
26791                                last_value[i] = circ_error.clone();
26792                                stamped += 1;
26793                            }
26794                        }
26795                        check_cancel(cancel_flag)?;
26796                        let order: Vec<usize> = analysis
26797                            .topo
26798                            .iter()
26799                            .map(|&i| i as usize)
26800                            .filter(|&i| !excluded[i])
26801                            .collect();
26802                        if !order.is_empty() {
26803                            passes += 1;
26804                            for i in order {
26805                                run_member!(i);
26806                            }
26807                        }
26808                        break;
26809                    }
26810                    CyclePolicy::Iterate {
26811                        max_iterations,
26812                        max_change,
26813                    } => {
26814                        // POLICY (Iterate), spec §3.5/§6.
26815                        iterating = true;
26816
26817                        // Convergence test: the full pass that just completed
26818                        // vs the previous full pass, per the spec-§6 rules.
26819                        // `prev_pass` is `None` until an iteration pass has
26820                        // run — pass 1 has no predecessor, so no convergence
26821                        // test occurs before the second pass (spec §6).
26822                        if let Some(prev) = &prev_pass {
26823                            let mut round_max_delta = 0f64;
26824                            let mut round_nan = 0usize;
26825                            let mut all_converged = true;
26826                            let mut round_exact = true;
26827                            for i in 0..n {
26828                                if excluded[i] {
26829                                    // Stamped mid-iteration (array result,
26830                                    // §7.9): the value is pinned and cannot
26831                                    // change again — trivially settled.
26832                                    continue;
26833                                }
26834                                let out = crate::engine::convergence::values_converged(
26835                                    &prev[i],
26836                                    &last_value[i],
26837                                    max_change,
26838                                    self.config.date_system,
26839                                );
26840                                if out.nan_converged {
26841                                    round_nan += 1;
26842                                }
26843                                if let Some(d) = out.abs_delta {
26844                                    round_max_delta = round_max_delta.max(d);
26845                                    if d != 0.0 {
26846                                        round_exact = false;
26847                                    }
26848                                }
26849                                if !out.converged {
26850                                    all_converged = false;
26851                                }
26852                            }
26853                            // Overwrite (not max): telemetry reports the
26854                            // round observed at stop.
26855                            iter_max_delta = round_max_delta;
26856                            iter_nan_converged = round_nan;
26857                            if all_converged {
26858                                exact_fixed_point = round_exact;
26859                                converged = true;
26860                                break;
26861                            }
26862                        }
26863
26864                        // ── Pass-counting reconciliation (spec §6/§7.6):
26865                        // `max_iterations` counts TOTAL passes, pass 1
26866                        // included, and pass 1 has already run by the time a
26867                        // live cycle is first witnessed here. The budget is
26868                        // therefore checked BEFORE evaluating anything more:
26869                        // with `max_iterations: 1` we stop right here — each
26870                        // member was evaluated exactly once this recalc (the
26871                        // Excel accumulator contract) and no convergence test
26872                        // ran (`prev_pass` is still `None`). Capping keeps
26873                        // the last committed values and is NOT an error
26874                        // (Excel parity); telemetry records it.
26875                        if passes >= max_iterations as usize {
26876                            capped = true;
26877                            break;
26878                        }
26879
26880                        check_cancel(cancel_flag)?;
26881                        // One more full pass over every evaluable member in
26882                        // member order (Gauss–Seidel: each commit is visible
26883                        // to later members within the pass). Live edges
26884                        // re-record — guards can flip near convergence
26885                        // (§7.3) — so classification repeats next time
26886                        // around, and a cycle that dissolves drops back to
26887                        // the exact acyclic settle below.
26888                        prev_pass = Some(last_value.clone());
26889                        for x in pos.iter_mut() {
26890                            *x = -1;
26891                        }
26892                        changed.fill(false);
26893                        passes += 1;
26894                        let mut p = 0i64;
26895                        for i in 0..n {
26896                            if excluded[i] {
26897                                continue;
26898                            }
26899                            run_member!(i);
26900                            pos[i] = p;
26901                            p += 1;
26902                        }
26903                        continue;
26904                    }
26905                }
26906            }
26907
26908            // Acyclic: find stale readers — members whose live read of `to`
26909            // happened before `to`'s value changed in the pass that just ran.
26910            let mut stale: Vec<usize> = Vec::new();
26911            for i in 0..n {
26912                if excluded[i] {
26913                    continue;
26914                }
26915                let is_stale = out_edges[i].iter().any(|&t| {
26916                    let t = t as usize;
26917                    changed[t] && (pos[i] < 0 || (pos[t] >= 0 && pos[i] < pos[t]))
26918                });
26919                if is_stale {
26920                    stale.push(i);
26921                }
26922            }
26923            if stale.is_empty() {
26924                break; // values exact — phantom SCC (or dissolved live cycle)
26925            }
26926            if 1 + settle_passes >= cap {
26927                // Defensive only; hitting this is a bug (loud telemetry).
26928                capped = true;
26929                for (i, m) in members.iter().enumerate() {
26930                    if !excluded[i] {
26931                        self.stamp_cycle_error(m.vertex, &circ_error, None);
26932                        excluded[i] = true;
26933                        last_value[i] = circ_error.clone();
26934                        stamped += 1;
26935                    }
26936                }
26937                break;
26938            }
26939
26940            check_cancel(cancel_flag)?;
26941            // Re-evaluate stale readers in live-topo order, recording fresh
26942            // edges (branches may flip on re-eval — spec §7.3 — which is why
26943            // classification repeats).
26944            // A settle pass is a partial sweep: drop the full-pass baseline
26945            // so a live cycle (re)appearing afterwards never compares values
26946            // across mixed pass kinds.
26947            prev_pass = None;
26948            let topo_pos = analysis.topo_positions();
26949            stale.sort_unstable_by_key(|&i| topo_pos[i]);
26950            for x in pos.iter_mut() {
26951                *x = -1;
26952            }
26953            changed.fill(false);
26954            passes += 1;
26955            settle_passes += 1;
26956            for (p, i) in stale.into_iter().enumerate() {
26957                run_member!(i);
26958                pos[i] = p as i64;
26959            }
26960        }
26961
26962        // Iteration that ended because the live cycle dissolved and the
26963        // acyclic settle reached exactness counts as converged (values are
26964        // exact, strictly better than threshold-converged). The defensive
26965        // settle cap (`capped` + stamping) is not.
26966        if iterating && !converged && !capped {
26967            converged = true;
26968        }
26969
26970        // ── 5. End of task: one delta per member whose final value differs
26971        // from the pre-task snapshot (spec §3 side-effect rule, G11).
26972        collector.clear_current();
26973        if let Some(d) = delta
26974            && d.mode != DeltaMode::Off
26975        {
26976            for (i, m) in members.iter().enumerate() {
26977                if let Some(cell) = m.cell
26978                    && last_value[i] != snapshot[i]
26979                {
26980                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
26981                }
26982            }
26983        }
26984
26985        // Members of an SCC that iterated re-evaluate on EVERY recalc, like
26986        // Excel's circular cells: register them for the end-of-recalc
26987        // volatile-like redirty (see `pending_iterative_redirty`). Marking
26988        // any one member propagates around the (strongly connected) SCC and
26989        // to downstream dependents, but all members are registered so the
26990        // contract survives partial structural edits between recalcs.
26991        //
26992        // Exception (#368): an SCC that stopped on an
26993        // exact fixed point — every member reproduced its previous value
26994        // bit-for-bit before the pass cap, no NaN identity, no volatile or
26995        // dynamic-reference member — cannot change on a re-run with the same
26996        // inputs, so it is retained clean. The dirty graph decides when it
26997        // runs again. Tolerance-only convergence (|Δ| < max_change but ≠ 0),
26998        // capped SCCs (including the `max_iterations: 1` accumulator
26999        // contract) and volatile cycles keep the per-recalc redirty.
27000        // Whatever the outcome, this task supersedes any earlier retention
27001        // of its members.
27002        if !self.retained_scc_members.is_empty() {
27003            for m in members.iter() {
27004                self.retained_scc_members.remove(&m.vertex);
27005            }
27006        }
27007        if iterating {
27008            let retain = converged
27009                && !capped
27010                && exact_fixed_point
27011                && iter_nan_converged == 0
27012                && members
27013                    .iter()
27014                    .all(|m| !self.graph.is_volatile(m.vertex) && !self.graph.is_dynamic(m.vertex));
27015            if retain {
27016                if self.retained_scc_members.is_empty() {
27017                    self.retained_scc_config_fingerprint = self.retained_scc_config_fingerprint();
27018                    self.retained_scc_function_epoch_seen =
27019                        crate::function_registry::semantic_epoch();
27020                    self.retained_scc_provider_revision_seen =
27021                        self.resolver.planning_semantic_revision();
27022                }
27023                let scc_id = self.next_retained_scc_id;
27024                self.next_retained_scc_id = self.next_retained_scc_id.wrapping_add(1);
27025                for (i, m) in members.iter().enumerate() {
27026                    self.retained_scc_members.insert(m.vertex, scc_id);
27027                    // §4 persistence snapshot, written once: retained
27028                    // members do not pass through `redirty_for_next_recalc`.
27029                    if matches!(last_value[i], LiteralValue::Empty) {
27030                        self.iterative_state_values.remove(&m.vertex);
27031                    } else {
27032                        self.iterative_state_values
27033                            .insert(m.vertex, last_value[i].clone());
27034                    }
27035                }
27036            } else {
27037                self.pending_iterative_redirty
27038                    .extend(members.iter().map(|m| m.vertex));
27039            }
27040        } else if !self.iterative_state_values.is_empty() {
27041            // The cycle dissolved (phantom settle or `#CIRC!` stamping):
27042            // these members are ordinary formulas again and must not carry
27043            // stale iteration state into a future cycle.
27044            for m in members.iter() {
27045                self.iterative_state_values.remove(&m.vertex);
27046            }
27047        }
27048
27049        {
27050            let t = &mut self.last_cycle_telemetry;
27051            t.static_sccs += 1;
27052            if witnessed_cycles == 0 && stamped == 0 && !capped {
27053                t.phantom_sccs += 1;
27054            }
27055            t.live_cycles_witnessed += witnessed_cycles;
27056            t.circ_cells_stamped += stamped;
27057            t.settle_passes_total += passes;
27058            t.max_passes_single_scc = t.max_passes_single_scc.max(passes);
27059            if iterating {
27060                t.iterated_sccs += 1;
27061                if converged {
27062                    t.converged_sccs += 1;
27063                }
27064                t.max_abs_delta_at_stop = t.max_abs_delta_at_stop.max(iter_max_delta);
27065                t.nan_converged += iter_nan_converged;
27066            }
27067            if capped {
27068                t.capped_sccs += 1;
27069            }
27070            t.elapsed_ms += task_start.elapsed().as_millis();
27071        }
27072
27073        Ok(stamped)
27074    }
27075
27076    /// Recorded sibling of [`Self::evaluate_vertex_immutable`]: evaluates one
27077    /// SCC member's AST via an [`Interpreter`] over a [`RecordingContext`] so
27078    /// reads that actually occur are captured as live edges. Value semantics
27079    /// must match `evaluate_vertex_immutable` exactly (including the missing-
27080    /// AST `Number(0.0)` quirk, G14); named Cell/Range/Literal definitions
27081    /// delegate to it after recording the definition region by hand (those
27082    /// reads bypass the context).
27083    fn evaluate_vertex_recorded(
27084        &self,
27085        vertex_id: VertexId,
27086        ctx: &RecordingContext<'_, R>,
27087        collector: &LiveEdgeCollector,
27088    ) -> Result<LiteralValue, ExcelError> {
27089        if !self.graph.vertex_exists(vertex_id) {
27090            return Err(ExcelError::new(formualizer_common::ExcelErrorKind::Ref)
27091                .with_message(format!("Vertex not found: {vertex_id:?}")));
27092        }
27093
27094        let kind = self.graph.get_vertex_kind(vertex_id);
27095        let sheet_id = self.graph.get_vertex_sheet_id(vertex_id);
27096
27097        match kind {
27098            VertexKind::FormulaScalar | VertexKind::FormulaArray => {
27099                let Some(ast_id) = self.graph.get_formula_id(vertex_id) else {
27100                    return Ok(LiteralValue::Number(0.0)); // G14 quirk
27101                };
27102                let sheet_name = self.graph.sheet_name(sheet_id);
27103                let cell_ref = self
27104                    .graph
27105                    .get_cell_ref(vertex_id)
27106                    .expect("cell ref for vertex");
27107                let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
27108                interpreter
27109                    .evaluate_arena_ast(ast_id, self.graph.data_store(), self.graph.sheet_reg())
27110                    .map(|cv| {
27111                        let format = cv.format_id();
27112                        self.derived_format_results
27113                            .write()
27114                            .unwrap()
27115                            .insert(vertex_id, format);
27116                        self.record_derived_format(vertex_id, format);
27117                        crate::engine::result_finalization::finalize_formula_result(
27118                            cv.into_literal(),
27119                        )
27120                    })
27121            }
27122            VertexKind::NamedScalar | VertexKind::NamedArray => {
27123                let named_range = self.graph.named_range_by_vertex(vertex_id).ok_or_else(|| {
27124                    ExcelError::new(ExcelErrorKind::Name)
27125                        .with_message("Named range metadata missing".to_string())
27126                })?;
27127
27128                match &named_range.definition {
27129                    NamedDefinition::Formula { ast, .. } => {
27130                        let context_sheet = match named_range.scope {
27131                            NameScope::Sheet(id) => id,
27132                            NameScope::Workbook => sheet_id,
27133                        };
27134                        let sheet_name = self.graph.sheet_name(context_sheet);
27135                        let cell_ref = self
27136                            .graph
27137                            .get_cell_ref(vertex_id)
27138                            .unwrap_or_else(|| self.graph.make_cell_ref(sheet_name, 0, 0));
27139                        let interpreter = Interpreter::new_with_cell(ctx, sheet_name, cell_ref);
27140                        if kind == VertexKind::NamedScalar {
27141                            interpreter.evaluate_ast(ast).map(|cv| cv.into_literal())
27142                        } else {
27143                            match interpreter.evaluate_ast(ast) {
27144                                Ok(cv) => match cv.into_literal() {
27145                                    v @ LiteralValue::Array(_) => Ok(v),
27146                                    other => Ok(LiteralValue::Array(vec![vec![other]])),
27147                                },
27148                                Err(err) => Ok(LiteralValue::Error(err)),
27149                            }
27150                        }
27151                    }
27152                    NamedDefinition::Cell(cell_ref) => {
27153                        // The definition is read via direct grid access in
27154                        // `evaluate_vertex_immutable`; record the live edge
27155                        // by hand before delegating.
27156                        collector.record_scalar(
27157                            cell_ref.sheet_id,
27158                            cell_ref.coord.row(),
27159                            cell_ref.coord.col(),
27160                        );
27161                        self.evaluate_vertex_immutable(vertex_id)
27162                    }
27163                    NamedDefinition::Range(range_ref) => {
27164                        if range_ref.start.sheet_id == range_ref.end.sheet_id {
27165                            collector.record_rect(
27166                                range_ref.start.sheet_id,
27167                                range_ref.start.coord.row(),
27168                                range_ref.start.coord.col(),
27169                                range_ref.end.coord.row(),
27170                                range_ref.end.coord.col(),
27171                            );
27172                        }
27173                        self.evaluate_vertex_immutable(vertex_id)
27174                    }
27175                    NamedDefinition::Literal(_) => self.evaluate_vertex_immutable(vertex_id),
27176                }
27177            }
27178            _ => self.evaluate_vertex_immutable(vertex_id),
27179        }
27180    }
27181
27182    /// Helper: commit spill via shim and mirror resulting cells into Arrow overlay when enabled.
27183    fn commit_spill_and_mirror(
27184        &mut self,
27185        anchor_vertex: VertexId,
27186        targets: &[CellRef],
27187        rows: Vec<Vec<LiteralValue>>,
27188        delta: Option<&mut DeltaCollector>,
27189        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27190    ) -> Result<(), ExcelError> {
27191        let prev_spill_cells = self
27192            .graph
27193            .spill_cells_for_anchor(anchor_vertex)
27194            .map(|cells| cells.to_vec())
27195            .unwrap_or_default();
27196
27197        if let Some(delta) = delta
27198            && delta.mode != DeltaMode::Off
27199        {
27200            let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
27201                targets.iter().copied().collect();
27202            let empty = LiteralValue::Empty;
27203
27204            // Clears (prev - targets)
27205            for cell in prev_spill_cells.iter() {
27206                if target_set.contains(cell) {
27207                    continue;
27208                }
27209                let sheet_name = self.graph.sheet_name(cell.sheet_id);
27210                let old = self
27211                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27212                    .unwrap_or(LiteralValue::Empty);
27213                if old != empty {
27214                    delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27215                }
27216            }
27217
27218            // Writes (targets)
27219            if !targets.is_empty() && !rows.is_empty() && !rows[0].is_empty() {
27220                let width = rows[0].len();
27221                for (idx, cell) in targets.iter().enumerate() {
27222                    let r_off = idx / width;
27223                    let c_off = idx % width;
27224                    let new = rows
27225                        .get(r_off)
27226                        .and_then(|r| r.get(c_off))
27227                        .cloned()
27228                        .unwrap_or(LiteralValue::Empty);
27229                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
27230                    let old = self
27231                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27232                        .unwrap_or(LiteralValue::Empty);
27233                    if old != new {
27234                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27235                    }
27236                }
27237            } else {
27238                // Degenerate shapes: if we have targets but no rows, treat as writing Empty.
27239                for cell in targets.iter() {
27240                    let sheet_name = self.graph.sheet_name(cell.sheet_id);
27241                    let old = self
27242                        .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27243                        .unwrap_or(LiteralValue::Empty);
27244                    if !matches!(old, LiteralValue::Empty) {
27245                        delta.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27246                    }
27247                }
27248            }
27249        }
27250
27251        // Commit via shim (releases locks). When the graph value cache is disabled (Arrow-canonical
27252        // values), plan/commit must consult Arrow storage to detect non-empty value blockers.
27253        let arrow_sheets = &self.arrow_sheets;
27254        self.spill_mgr.commit_array_with_value_probe(
27255            &mut self.graph,
27256            anchor_vertex,
27257            targets,
27258            rows.clone(),
27259            overwritable_formulas,
27260            |g, cell| {
27261                let sheet_name = g.sheet_name(cell.sheet_id);
27262                let asheet = arrow_sheets.sheet(sheet_name)?;
27263                let r0 = cell.coord.row() as usize;
27264                let c0 = cell.coord.col() as usize;
27265                let v = asheet.get_cell_value(r0, c0);
27266                if matches!(v, LiteralValue::Empty) {
27267                    None
27268                } else {
27269                    Some(v)
27270                }
27271            },
27272        )?;
27273
27274        if let Some(scope) = Self::formula_plane_region_from_cells(&prev_spill_cells) {
27275            self.record_formula_plane_structural_change(scope);
27276        }
27277        if let Some(scope) = Self::formula_plane_region_from_cells(targets) {
27278            self.record_formula_plane_structural_change(scope);
27279        }
27280
27281        if self.config.arrow_storage_enabled
27282            && self.config.delta_overlay_enabled
27283            && self.config.write_formula_overlay_enabled
27284        {
27285            if !prev_spill_cells.is_empty() {
27286                let target_set: std::collections::HashSet<CellRef, CoordBuildHasher> =
27287                    targets.iter().copied().collect();
27288                let empty = LiteralValue::Empty;
27289                for cell in prev_spill_cells.iter() {
27290                    if !target_set.contains(cell) {
27291                        let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27292                        self.mirror_value_to_computed_overlay(
27293                            &sheet_name,
27294                            cell.coord.row() + 1,
27295                            cell.coord.col() + 1,
27296                            &empty,
27297                        );
27298                    }
27299                }
27300            }
27301
27302            for (idx, cell) in targets.iter().enumerate() {
27303                if rows.is_empty() || rows[0].is_empty() {
27304                    break;
27305                }
27306                let width = rows[0].len();
27307                let r_off = idx / width;
27308                let c_off = idx % width;
27309                let v = rows[r_off][c_off].clone();
27310                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27311                self.mirror_value_to_computed_overlay(
27312                    &sheet_name,
27313                    cell.coord.row() + 1,
27314                    cell.coord.col() + 1,
27315                    &v,
27316                );
27317            }
27318        }
27319        Ok(())
27320    }
27321}
27322
27323// ── Effects pipeline (ticket 603) ──────────────────────────────────────────
27324//
27325// Compute → Plan → Apply separation for evaluation side-effects.
27326
27327use crate::engine::effects::Effect;
27328use crate::engine::graph::editor::change_log::{ChangeEvent, ChangeLog, SpillSnapshot};
27329
27330impl<R> Engine<R>
27331where
27332    R: EvaluationContext,
27333{
27334    /// Plan effects for a single vertex after its value has been computed.
27335    ///
27336    /// This reads graph state but only performs lightweight mutations
27337    /// (`set_kind`, `spill_mgr.reserve`) that are needed for correctness
27338    /// during the planning phase.  Value-changing mutations are deferred to
27339    /// `apply_effect`.
27340    pub(crate) fn plan_vertex_effects(
27341        &mut self,
27342        vertex_id: VertexId,
27343        computed_value: LiteralValue,
27344        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27345    ) -> Result<Vec<Effect>, ExcelError> {
27346        let kind = self.graph.get_vertex_kind(vertex_id);
27347        let is_formula = matches!(kind, VertexKind::FormulaScalar | VertexKind::FormulaArray);
27348
27349        // If this vertex's cell is currently covered by a spill from a different
27350        // anchor, ignore the computed result.  Formula vertices are exempt:
27351        // they must still evaluate so that overlapping spills produce #SPILL!.
27352        if !is_formula {
27353            if let Some(cell) = self.graph.get_cell_ref(vertex_id)
27354                && let Some(owner) = self.graph.spill_registry_anchor_for_cell(cell)
27355                && owner != vertex_id
27356            {
27357                return Ok(Vec::new());
27358            }
27359            // Non-formula vertices: store value as-is (arrays remain arrays; no spill).
27360            return Ok(vec![Effect::WriteCell {
27361                vertex_id,
27362                value: computed_value,
27363            }]);
27364        }
27365
27366        match computed_value {
27367            LiteralValue::Array(rows) => {
27368                self.plan_array_effects(vertex_id, rows, overwritable_formulas)
27369            }
27370            other => self.plan_scalar_effects(vertex_id, other),
27371        }
27372    }
27373
27374    /// Plan effects for a formula vertex that produced a scalar/error result.
27375    fn plan_scalar_effects(
27376        &self,
27377        vertex_id: VertexId,
27378        value: LiteralValue,
27379    ) -> Result<Vec<Effect>, ExcelError> {
27380        let has_spill = self
27381            .graph
27382            .spill_cells_for_anchor(vertex_id)
27383            .is_some_and(|c| !c.is_empty());
27384
27385        let mut effects = Vec::new();
27386        if has_spill {
27387            effects.push(Effect::SpillClear {
27388                anchor_vertex: vertex_id,
27389            });
27390        }
27391        effects.push(Effect::WriteCell { vertex_id, value });
27392        Ok(effects)
27393    }
27394
27395    /// Plan effects for a formula vertex that produced an array result.
27396    fn plan_array_effects(
27397        &mut self,
27398        vertex_id: VertexId,
27399        rows: Vec<Vec<LiteralValue>>,
27400        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27401    ) -> Result<Vec<Effect>, ExcelError> {
27402        // Lightweight mutation needed for correct spill-blocking checks.
27403        self.graph.set_kind(vertex_id, VertexKind::FormulaArray);
27404
27405        let anchor = self
27406            .graph
27407            .get_cell_ref(vertex_id)
27408            .expect("cell ref for vertex");
27409        let sheet_id = anchor.sheet_id;
27410        let h = rows.len() as u32;
27411        let w = rows.first().map(|r| r.len()).unwrap_or(0) as u32;
27412
27413        // Hard cap to avoid vertex explosion from huge dynamic arrays.
27414        let spill_cells = (h as u64).saturating_mul(w as u64);
27415        if spill_cells > self.config.spill.max_spill_cells as u64 {
27416            return self.plan_spill_error_effects(vertex_id, "SpillTooLarge", h, w);
27417        }
27418
27419        // Bounds check to avoid out-of-range writes (align to AbsCoord capacity).
27420        const PACKED_MAX_ROW: u32 = 1_048_575;
27421        const PACKED_MAX_COL: u32 = 16_383;
27422        let end_row = anchor.coord.row().saturating_add(h).saturating_sub(1);
27423        let end_col = anchor.coord.col().saturating_add(w).saturating_sub(1);
27424        if end_row > PACKED_MAX_ROW || end_col > PACKED_MAX_COL {
27425            return self.plan_spill_error_effects(vertex_id, "Spill exceeds sheet bounds", h, w);
27426        }
27427
27428        let mut targets = Vec::new();
27429        for r in 0..h {
27430            for c in 0..w {
27431                targets.push(self.graph.make_cell_ref_internal(
27432                    sheet_id,
27433                    anchor.coord.row() + r,
27434                    anchor.coord.col() + c,
27435                ));
27436            }
27437        }
27438
27439        // Region lock via spill manager.
27440        match self.spill_mgr.reserve(
27441            vertex_id,
27442            anchor,
27443            SpillShape { rows: h, cols: w },
27444            SpillMeta {
27445                epoch: self.recalc_epoch,
27446                config: self.config.spill,
27447            },
27448        ) {
27449            Ok(()) => {
27450                // Validate spill region is available.
27451                if let Err(_e) = self.graph.plan_spill_region_allowing_formula_overwrite(
27452                    vertex_id,
27453                    &targets,
27454                    overwritable_formulas,
27455                ) {
27456                    return self.plan_spill_error_effects(vertex_id, "Spill blocked", h, w);
27457                }
27458
27459                // Arrow-canonical mode: graph planning cannot see non-empty value blockers because
27460                // cell values are not cached in the dependency graph. Consult Arrow storage to
27461                // detect occupied cells in the target region.
27462                if !self.graph.value_cache_enabled() {
27463                    let sheet_name = self.graph.sheet_name(sheet_id);
27464                    if let Some(asheet) = self.sheet_store().sheet(sheet_name) {
27465                        for cell in targets.iter() {
27466                            // Allow overwriting the anchor itself.
27467                            if *cell == anchor {
27468                                continue;
27469                            }
27470                            // Allow cells already owned by a spill (plan() validated spill ownership).
27471                            if self.graph.spill_registry_anchor_for_cell(*cell).is_some() {
27472                                continue;
27473                            }
27474                            // Skip formula blockers; plan() handled them (or allowed).
27475                            if let Some(&vid) = self.graph.get_vertex_id_for_address(cell)
27476                                && vid != vertex_id
27477                            {
27478                                match self.graph.get_vertex_kind(vid) {
27479                                    VertexKind::FormulaScalar | VertexKind::FormulaArray => {
27480                                        continue;
27481                                    }
27482                                    _ => {}
27483                                }
27484                            }
27485
27486                            let v = asheet.get_cell_value(
27487                                cell.coord.row() as usize,
27488                                cell.coord.col() as usize,
27489                            );
27490                            if !matches!(v, LiteralValue::Empty) {
27491                                return self.plan_spill_error_effects(
27492                                    vertex_id,
27493                                    "BlockedByValue",
27494                                    h,
27495                                    w,
27496                                );
27497                            }
27498                        }
27499                    }
27500                }
27501
27502                let top_left = rows
27503                    .first()
27504                    .and_then(|r| r.first())
27505                    .cloned()
27506                    .unwrap_or(LiteralValue::Empty);
27507
27508                let mut effects = Vec::new();
27509                // Clear previous spill if any.
27510                let has_prev = self
27511                    .graph
27512                    .spill_cells_for_anchor(vertex_id)
27513                    .is_some_and(|c| !c.is_empty());
27514                if has_prev {
27515                    effects.push(Effect::SpillClear {
27516                        anchor_vertex: vertex_id,
27517                    });
27518                }
27519                effects.push(Effect::SpillCommit {
27520                    anchor_vertex: vertex_id,
27521                    anchor_cell: anchor,
27522                    target_cells: targets,
27523                    values: rows,
27524                });
27525                effects.push(Effect::WriteCell {
27526                    vertex_id,
27527                    value: top_left,
27528                });
27529                Ok(effects)
27530            }
27531            Err(e) => {
27532                let msg = e.message.unwrap_or_else(|| "Spill blocked".to_string());
27533                self.plan_spill_error_effects(vertex_id, &msg, h, w)
27534            }
27535        }
27536    }
27537
27538    /// Build the effect list for a spill that failed validation.
27539    fn plan_spill_error_effects(
27540        &self,
27541        vertex_id: VertexId,
27542        message: &str,
27543        expected_rows: u32,
27544        expected_cols: u32,
27545    ) -> Result<Vec<Effect>, ExcelError> {
27546        let spill_err = ExcelError::new(ExcelErrorKind::Spill)
27547            .with_message(message)
27548            .with_extra(formualizer_common::ExcelErrorExtra::Spill {
27549                expected_rows,
27550                expected_cols,
27551            });
27552        let spill_val = LiteralValue::Error(spill_err);
27553
27554        let effects = vec![
27555            Effect::SpillClear {
27556                anchor_vertex: vertex_id,
27557            },
27558            Effect::WriteCell {
27559                vertex_id,
27560                value: spill_val,
27561            },
27562        ];
27563        Ok(effects)
27564    }
27565
27566    /// Apply a single effect, performing the actual graph mutations.
27567    pub(crate) fn apply_effect(
27568        &mut self,
27569        effect: &Effect,
27570        delta: Option<&mut DeltaCollector>,
27571        log: Option<&mut ChangeLog>,
27572    ) -> Result<(), ExcelError> {
27573        self.apply_effect_with_computed_writes(effect, delta, log, None)
27574    }
27575
27576    fn apply_effect_with_computed_writes(
27577        &mut self,
27578        effect: &Effect,
27579        delta: Option<&mut DeltaCollector>,
27580        log: Option<&mut ChangeLog>,
27581        computed_writes: Option<&mut ComputedWriteBuffer>,
27582    ) -> Result<(), ExcelError> {
27583        match effect {
27584            Effect::WriteCell { vertex_id, value } => {
27585                self.apply_write_cell(*vertex_id, value, delta, computed_writes)?;
27586            }
27587            Effect::SpillClear { anchor_vertex } => {
27588                self.apply_spill_clear(*anchor_vertex, delta, log, computed_writes)?;
27589            }
27590            Effect::SpillCommit {
27591                anchor_vertex,
27592                anchor_cell: _,
27593                target_cells,
27594                values,
27595            } => {
27596                self.apply_spill_commit(
27597                    *anchor_vertex,
27598                    target_cells,
27599                    values.clone(),
27600                    delta,
27601                    log,
27602                    computed_writes,
27603                )?;
27604            }
27605        }
27606        Ok(())
27607    }
27608
27609    /// Apply a WriteCell effect.
27610    fn apply_write_cell(
27611        &mut self,
27612        vertex_id: VertexId,
27613        value: &LiteralValue,
27614        delta: Option<&mut DeltaCollector>,
27615        mut computed_writes: Option<&mut ComputedWriteBuffer>,
27616    ) -> Result<(), ExcelError> {
27617        if let Some(d) = delta
27618            && d.mode != DeltaMode::Off
27619        {
27620            if let Some(buffer) = computed_writes.as_deref_mut() {
27621                self.flush_computed_write_buffer(buffer)?;
27622            }
27623            if let Some(cell) = self.graph.get_cell_ref_for_vertex(vertex_id) {
27624                let sheet_name = self.graph.sheet_name(cell.sheet_id);
27625                let old = self
27626                    .read_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27627                    .unwrap_or(LiteralValue::Empty);
27628                if old != *value {
27629                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27630                }
27631            }
27632        }
27633        self.graph.update_vertex_value(vertex_id, value.clone());
27634        self.record_vertex_value_to_overlay(vertex_id, value, computed_writes)?;
27635        Ok(())
27636    }
27637
27638    /// Apply a SpillClear effect.
27639    fn apply_spill_clear(
27640        &mut self,
27641        anchor_vertex: VertexId,
27642        delta: Option<&mut DeltaCollector>,
27643        log: Option<&mut ChangeLog>,
27644        computed_writes: Option<&mut ComputedWriteBuffer>,
27645    ) -> Result<(), ExcelError> {
27646        if let Some(buffer) = computed_writes {
27647            self.flush_computed_write_buffer(buffer)?;
27648        }
27649
27650        let spill_cells = self
27651            .graph
27652            .spill_cells_for_anchor(anchor_vertex)
27653            .map(|cells| cells.to_vec())
27654            .unwrap_or_default();
27655        if spill_cells.is_empty() {
27656            return Ok(());
27657        }
27658
27659        // Snapshot for ChangeLog before clearing.
27660        let snapshot = if log.is_some() {
27661            self.snapshot_spill_for_anchor(anchor_vertex)
27662        } else {
27663            None
27664        };
27665
27666        // Record delta for cleared cells.
27667        if let Some(d) = delta
27668            && d.mode != DeltaMode::Off
27669        {
27670            let empty = LiteralValue::Empty;
27671            for cell in spill_cells.iter() {
27672                let sheet_name = self.graph.sheet_name(cell.sheet_id);
27673                let old = self
27674                    .get_cell_value(sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27675                    .unwrap_or(LiteralValue::Empty);
27676                if old != empty {
27677                    d.record_cell(cell.sheet_id, cell.coord.row(), cell.coord.col());
27678                }
27679            }
27680        }
27681
27682        self.graph.clear_spill_region(anchor_vertex);
27683        if let Some(scope) = Self::formula_plane_region_from_cells(&spill_cells) {
27684            self.record_formula_plane_structural_change(scope);
27685        }
27686
27687        // Mirror Empty to Arrow overlay for cleared cells.
27688        if self.config.arrow_storage_enabled
27689            && self.config.delta_overlay_enabled
27690            && self.config.write_formula_overlay_enabled
27691        {
27692            let empty = LiteralValue::Empty;
27693            for cell in spill_cells.iter() {
27694                let sheet_name = self.graph.sheet_name(cell.sheet_id).to_string();
27695                self.mirror_value_to_computed_overlay(
27696                    &sheet_name,
27697                    cell.coord.row() + 1,
27698                    cell.coord.col() + 1,
27699                    &empty,
27700                );
27701            }
27702        }
27703
27704        // ChangeLog.
27705        if let Some(log) = log
27706            && let Some(old) = snapshot
27707        {
27708            log.record(ChangeEvent::SpillCleared {
27709                anchor: anchor_vertex,
27710                old,
27711            });
27712        }
27713        Ok(())
27714    }
27715
27716    /// Apply a SpillCommit effect.
27717    fn apply_spill_commit(
27718        &mut self,
27719        anchor_vertex: VertexId,
27720        target_cells: &[CellRef],
27721        values: Vec<Vec<LiteralValue>>,
27722        delta: Option<&mut DeltaCollector>,
27723        log: Option<&mut ChangeLog>,
27724        computed_writes: Option<&mut ComputedWriteBuffer>,
27725    ) -> Result<(), ExcelError> {
27726        if let Some(buffer) = computed_writes {
27727            self.flush_computed_write_buffer(buffer)?;
27728        }
27729
27730        // Snapshot for ChangeLog before commit.
27731        let old_snapshot = if log.is_some() {
27732            self.snapshot_spill_for_anchor(anchor_vertex)
27733        } else {
27734            None
27735        };
27736
27737        // Delegate to existing commit_spill_and_mirror for delta + overlay logic.
27738        self.commit_spill_and_mirror(
27739            anchor_vertex,
27740            target_cells,
27741            values.clone(),
27742            delta,
27743            None, // overwritable_formulas already validated in plan phase
27744        )?;
27745
27746        // ChangeLog.
27747        if let Some(log) = log {
27748            log.record(ChangeEvent::SpillCommitted {
27749                anchor: anchor_vertex,
27750                old: old_snapshot,
27751                new: SpillSnapshot {
27752                    target_cells: target_cells.to_vec(),
27753                    values,
27754                },
27755            });
27756        }
27757        Ok(())
27758    }
27759
27760    /// Snapshot a spill region for ChangeLog recording.
27761    ///
27762    /// Extracted from `VertexEditor::snapshot_spill_for_anchor` to be usable
27763    /// without creating a `VertexEditor`.
27764    fn snapshot_spill_for_anchor(&self, anchor: VertexId) -> Option<SpillSnapshot> {
27765        let cells = self.graph.spill_cells_for_anchor(anchor)?.to_vec();
27766        if cells.is_empty() {
27767            return None;
27768        }
27769
27770        let max = self.config.spill.max_spill_cells as usize;
27771        let mut cells = cells;
27772        if cells.len() > max {
27773            cells.truncate(max);
27774        }
27775
27776        let first = *cells.first().expect("non-empty spill cells");
27777        let sheet_name = self.graph.sheet_name(first.sheet_id).to_string();
27778        let row0 = first.coord.row();
27779        let col0 = first.coord.col();
27780
27781        let mut max_row = row0;
27782        let mut max_col = col0;
27783        let mut by_coord: FxHashMap<(u32, u32), LiteralValue> = FxHashMap::default();
27784        for cell in &cells {
27785            max_row = max_row.max(cell.coord.row());
27786            max_col = max_col.max(cell.coord.col());
27787            let v = self
27788                .get_cell_value(&sheet_name, cell.coord.row() + 1, cell.coord.col() + 1)
27789                .unwrap_or(LiteralValue::Empty);
27790            by_coord.insert((cell.coord.row(), cell.coord.col()), v);
27791        }
27792
27793        let rows = (max_row - row0 + 1) as usize;
27794        let cols = (max_col - col0 + 1) as usize;
27795        let mut values: Vec<Vec<LiteralValue>> = Vec::with_capacity(rows);
27796        for r in 0..rows {
27797            let mut row: Vec<LiteralValue> = Vec::with_capacity(cols);
27798            for c in 0..cols {
27799                row.push(
27800                    by_coord
27801                        .get(&(row0 + r as u32, col0 + c as u32))
27802                        .cloned()
27803                        .unwrap_or(LiteralValue::Empty),
27804                );
27805            }
27806            values.push(row);
27807        }
27808
27809        Some(SpillSnapshot {
27810            target_cells: cells,
27811            values,
27812        })
27813    }
27814
27815    fn flush_before_range_dependent_vertex(
27816        &mut self,
27817        vertex_id: VertexId,
27818        computed_writes: &mut ComputedWriteBuffer,
27819    ) -> Result<(), ExcelError> {
27820        if self.graph.get_range_dependencies(vertex_id).is_some() {
27821            self.flush_computed_write_buffer(computed_writes)?;
27822        }
27823        Ok(())
27824    }
27825
27826    fn plan_vertex_effects_with_computed_flush(
27827        &mut self,
27828        vertex_id: VertexId,
27829        computed_value: LiteralValue,
27830        overwritable_formulas: Option<&rustc_hash::FxHashSet<VertexId>>,
27831        computed_writes: &mut ComputedWriteBuffer,
27832    ) -> Result<Vec<Effect>, ExcelError> {
27833        if matches!(&computed_value, LiteralValue::Array(_)) {
27834            self.flush_computed_write_buffer(computed_writes)?;
27835        }
27836        self.plan_vertex_effects(vertex_id, computed_value, overwritable_formulas)
27837    }
27838
27839    // ── Layer evaluation via effects pipeline ──────────────────────────────
27840
27841    fn evaluate_small_layer_direct_effects(
27842        &mut self,
27843        layer: &super::scheduler::Layer,
27844        mut delta: Option<&mut DeltaCollector>,
27845        mut log: Option<&mut ChangeLog>,
27846        cancel_flag: Option<&AtomicBool>,
27847        cancel_check_every: usize,
27848        cancel_message: &'static str,
27849    ) -> Result<usize, ExcelError> {
27850        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
27851            if cancel_check_every > 0
27852                && i % cancel_check_every == 0
27853                && cancel_flag.is_some_and(|flag| flag.load(Ordering::Relaxed))
27854            {
27855                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
27856                    .with_message(cancel_message.to_string()));
27857            }
27858            let value = match self.evaluate_vertex_immutable(vertex_id) {
27859                Ok(v) => v,
27860                Err(e) => LiteralValue::Error(e),
27861            };
27862            let effects = self.plan_vertex_effects(vertex_id, value, None)?;
27863            for effect in &effects {
27864                self.apply_effect_with_computed_writes(
27865                    effect,
27866                    delta.as_deref_mut(),
27867                    log.as_deref_mut(),
27868                    None,
27869                )?;
27870            }
27871        }
27872        Ok(layer.vertices.len())
27873    }
27874
27875    /// Evaluate a layer sequentially using the effects pipeline.
27876    fn evaluate_layer_sequential_effects(
27877        &mut self,
27878        layer: &super::scheduler::Layer,
27879    ) -> Result<usize, ExcelError> {
27880        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27881            return self.evaluate_small_layer_direct_effects(
27882                layer,
27883                None,
27884                None,
27885                None,
27886                0,
27887                "Evaluation cancelled within layer",
27888            );
27889        }
27890
27891        let mut computed_writes = ComputedWriteBuffer::default();
27892        for &vertex_id in &layer.vertices {
27893            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
27894            let value = match self.evaluate_vertex_immutable(vertex_id) {
27895                Ok(v) => v,
27896                Err(e) => LiteralValue::Error(e),
27897            };
27898            let effects = match self.plan_vertex_effects_with_computed_flush(
27899                vertex_id,
27900                value,
27901                None,
27902                &mut computed_writes,
27903            ) {
27904                Ok(effects) => effects,
27905                Err(e) => {
27906                    self.flush_computed_write_buffer(&mut computed_writes)?;
27907                    return Err(e);
27908                }
27909            };
27910            for effect in &effects {
27911                if let Err(e) = self.apply_effect_with_computed_writes(
27912                    effect,
27913                    None,
27914                    None,
27915                    Some(&mut computed_writes),
27916                ) {
27917                    self.flush_computed_write_buffer(&mut computed_writes)?;
27918                    return Err(e);
27919                }
27920            }
27921        }
27922        self.flush_computed_write_buffer(&mut computed_writes)?;
27923        Ok(layer.vertices.len())
27924    }
27925
27926    /// Evaluate a layer sequentially with delta collection via effects pipeline.
27927    fn evaluate_layer_sequential_with_delta_effects(
27928        &mut self,
27929        layer: &super::scheduler::Layer,
27930        delta: &mut DeltaCollector,
27931    ) -> Result<usize, ExcelError> {
27932        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27933            return self.evaluate_small_layer_direct_effects(
27934                layer,
27935                Some(delta),
27936                None,
27937                None,
27938                0,
27939                "Evaluation cancelled within layer",
27940            );
27941        }
27942
27943        let mut computed_writes = ComputedWriteBuffer::default();
27944        for &vertex_id in &layer.vertices {
27945            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
27946            let value = match self.evaluate_vertex_immutable(vertex_id) {
27947                Ok(v) => v,
27948                Err(e) => LiteralValue::Error(e),
27949            };
27950            let effects = match self.plan_vertex_effects_with_computed_flush(
27951                vertex_id,
27952                value,
27953                None,
27954                &mut computed_writes,
27955            ) {
27956                Ok(effects) => effects,
27957                Err(e) => {
27958                    self.flush_computed_write_buffer(&mut computed_writes)?;
27959                    return Err(e);
27960                }
27961            };
27962            for effect in &effects {
27963                if let Err(e) = self.apply_effect_with_computed_writes(
27964                    effect,
27965                    Some(delta),
27966                    None,
27967                    Some(&mut computed_writes),
27968                ) {
27969                    self.flush_computed_write_buffer(&mut computed_writes)?;
27970                    return Err(e);
27971                }
27972            }
27973        }
27974        self.flush_computed_write_buffer(&mut computed_writes)?;
27975        Ok(layer.vertices.len())
27976    }
27977
27978    /// Evaluate a layer sequentially with cancellation support via effects pipeline.
27979    fn evaluate_layer_sequential_cancellable_effects(
27980        &mut self,
27981        layer: &super::scheduler::Layer,
27982        cancel_flag: &AtomicBool,
27983    ) -> Result<usize, ExcelError> {
27984        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
27985            return self.evaluate_small_layer_direct_effects(
27986                layer,
27987                None,
27988                None,
27989                Some(cancel_flag),
27990                256,
27991                "Evaluation cancelled within layer",
27992            );
27993        }
27994
27995        let mut computed_writes = ComputedWriteBuffer::default();
27996        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
27997            if i % 256 == 0 && cancel_flag.load(Ordering::Relaxed) {
27998                self.flush_computed_write_buffer(&mut computed_writes)?;
27999                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28000                    .with_message("Evaluation cancelled within layer".to_string()));
28001            }
28002            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28003            let value = match self.evaluate_vertex_immutable(vertex_id) {
28004                Ok(v) => v,
28005                Err(e) => LiteralValue::Error(e),
28006            };
28007            let effects = match self.plan_vertex_effects_with_computed_flush(
28008                vertex_id,
28009                value,
28010                None,
28011                &mut computed_writes,
28012            ) {
28013                Ok(effects) => effects,
28014                Err(e) => {
28015                    self.flush_computed_write_buffer(&mut computed_writes)?;
28016                    return Err(e);
28017                }
28018            };
28019            for effect in &effects {
28020                if let Err(e) = self.apply_effect_with_computed_writes(
28021                    effect,
28022                    None,
28023                    None,
28024                    Some(&mut computed_writes),
28025                ) {
28026                    self.flush_computed_write_buffer(&mut computed_writes)?;
28027                    return Err(e);
28028                }
28029            }
28030        }
28031        self.flush_computed_write_buffer(&mut computed_writes)?;
28032        Ok(layer.vertices.len())
28033    }
28034
28035    /// Evaluate a layer sequentially with more frequent cancellation for demand-driven eval.
28036    fn evaluate_layer_sequential_cancellable_demand_driven_effects(
28037        &mut self,
28038        layer: &super::scheduler::Layer,
28039        cancel_flag: &AtomicBool,
28040    ) -> Result<usize, ExcelError> {
28041        if layer.vertices.len() < COMPUTED_WRITE_COALESCING_MIN_LAYER_WIDTH {
28042            return self.evaluate_small_layer_direct_effects(
28043                layer,
28044                None,
28045                None,
28046                Some(cancel_flag),
28047                128,
28048                "Demand-driven evaluation cancelled within layer",
28049            );
28050        }
28051
28052        let mut computed_writes = ComputedWriteBuffer::default();
28053        for (i, &vertex_id) in layer.vertices.iter().enumerate() {
28054            if i % 128 == 0 && cancel_flag.load(Ordering::Relaxed) {
28055                self.flush_computed_write_buffer(&mut computed_writes)?;
28056                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28057                    .with_message("Demand-driven evaluation cancelled within layer".to_string()));
28058            }
28059            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28060            let value = match self.evaluate_vertex_immutable(vertex_id) {
28061                Ok(v) => v,
28062                Err(e) => LiteralValue::Error(e),
28063            };
28064            let effects = match self.plan_vertex_effects_with_computed_flush(
28065                vertex_id,
28066                value,
28067                None,
28068                &mut computed_writes,
28069            ) {
28070                Ok(effects) => effects,
28071                Err(e) => {
28072                    self.flush_computed_write_buffer(&mut computed_writes)?;
28073                    return Err(e);
28074                }
28075            };
28076            for effect in &effects {
28077                if let Err(e) = self.apply_effect_with_computed_writes(
28078                    effect,
28079                    None,
28080                    None,
28081                    Some(&mut computed_writes),
28082                ) {
28083                    self.flush_computed_write_buffer(&mut computed_writes)?;
28084                    return Err(e);
28085                }
28086            }
28087        }
28088        self.flush_computed_write_buffer(&mut computed_writes)?;
28089        Ok(layer.vertices.len())
28090    }
28091
28092    /// Evaluate a layer in parallel, applying via effects pipeline.
28093    fn evaluate_layer_parallel_effects(
28094        &mut self,
28095        layer: &super::scheduler::Layer,
28096    ) -> Result<usize, ExcelError> {
28097        use rayon::prelude::*;
28098
28099        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28100
28101        let mut phase1: Vec<VertexId> = Vec::new();
28102        let mut phase2: Vec<VertexId> = Vec::new();
28103        for &vid in &layer.vertices {
28104            if self.graph.get_range_dependencies(vid).is_some() {
28105                phase2.push(vid);
28106            } else {
28107                phase1.push(vid);
28108            }
28109        }
28110
28111        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28112        let mut applied = 0usize;
28113
28114        for group in [&phase1[..], &phase2[..]] {
28115            if group.is_empty() {
28116                continue;
28117            }
28118            let mut computed_writes = ComputedWriteBuffer::default();
28119
28120            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28121                thread_pool.install(|| {
28122                    group
28123                        .par_iter()
28124                        .map(
28125                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
28126                                Ok(v) => Ok((vertex_id, v)),
28127                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28128                            },
28129                        )
28130                        .collect()
28131                });
28132
28133            match results {
28134                Ok(vertex_results) => {
28135                    // Arrays first, then scalars — establishes spill regions before
28136                    // scalar results that might land inside a spilled region.
28137                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28138                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28139                    for (vertex_id, result) in vertex_results {
28140                        if matches!(result, LiteralValue::Array(_)) {
28141                            arrays.push((vertex_id, result));
28142                        } else {
28143                            others.push((vertex_id, result));
28144                        }
28145                    }
28146                    for (vertex_id, result) in arrays {
28147                        let effects = match self.plan_vertex_effects_with_computed_flush(
28148                            vertex_id,
28149                            result,
28150                            Some(&inflight),
28151                            &mut computed_writes,
28152                        ) {
28153                            Ok(effects) => effects,
28154                            Err(e) => {
28155                                self.flush_computed_write_buffer(&mut computed_writes)?;
28156                                return Err(e);
28157                            }
28158                        };
28159                        for effect in &effects {
28160                            if let Err(e) = self.apply_effect_with_computed_writes(
28161                                effect,
28162                                None,
28163                                None,
28164                                Some(&mut computed_writes),
28165                            ) {
28166                                self.flush_computed_write_buffer(&mut computed_writes)?;
28167                                return Err(e);
28168                            }
28169                        }
28170                        applied = applied.saturating_add(1);
28171                    }
28172                    // Make all array spill/top-left writes visible before scalar effects in this group.
28173                    self.flush_computed_write_buffer(&mut computed_writes)?;
28174                    for (vertex_id, result) in others {
28175                        let effects = match self.plan_vertex_effects_with_computed_flush(
28176                            vertex_id,
28177                            result,
28178                            Some(&inflight),
28179                            &mut computed_writes,
28180                        ) {
28181                            Ok(effects) => effects,
28182                            Err(e) => {
28183                                self.flush_computed_write_buffer(&mut computed_writes)?;
28184                                return Err(e);
28185                            }
28186                        };
28187                        for effect in &effects {
28188                            if let Err(e) = self.apply_effect_with_computed_writes(
28189                                effect,
28190                                None,
28191                                None,
28192                                Some(&mut computed_writes),
28193                            ) {
28194                                self.flush_computed_write_buffer(&mut computed_writes)?;
28195                                return Err(e);
28196                            }
28197                        }
28198                        applied = applied.saturating_add(1);
28199                    }
28200                    // Flush at the group boundary; phase1 must be visible before phase2.
28201                    self.flush_computed_write_buffer(&mut computed_writes)?;
28202                }
28203                Err(e) => {
28204                    self.flush_computed_write_buffer(&mut computed_writes)?;
28205                    return Err(e);
28206                }
28207            }
28208        }
28209
28210        Ok(applied)
28211    }
28212
28213    /// Evaluate a layer in parallel with delta collection via effects pipeline.
28214    fn evaluate_layer_parallel_with_delta_effects(
28215        &mut self,
28216        layer: &super::scheduler::Layer,
28217        delta: &mut DeltaCollector,
28218    ) -> Result<usize, ExcelError> {
28219        use rayon::prelude::*;
28220
28221        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28222
28223        let mut phase1: Vec<VertexId> = Vec::new();
28224        let mut phase2: Vec<VertexId> = Vec::new();
28225        for &vid in &layer.vertices {
28226            if self.graph.get_range_dependencies(vid).is_some() {
28227                phase2.push(vid);
28228            } else {
28229                phase1.push(vid);
28230            }
28231        }
28232
28233        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28234        let mut applied = 0usize;
28235
28236        for group in [&phase1[..], &phase2[..]] {
28237            if group.is_empty() {
28238                continue;
28239            }
28240            let mut computed_writes = ComputedWriteBuffer::default();
28241            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28242                thread_pool.install(|| {
28243                    group
28244                        .par_iter()
28245                        .map(
28246                            |&vertex_id| match self.evaluate_vertex_immutable(vertex_id) {
28247                                Ok(v) => Ok((vertex_id, v)),
28248                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28249                            },
28250                        )
28251                        .collect()
28252                });
28253
28254            match results {
28255                Ok(vertex_results) => {
28256                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28257                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28258                    for (vertex_id, result) in vertex_results {
28259                        if matches!(result, LiteralValue::Array(_)) {
28260                            arrays.push((vertex_id, result));
28261                        } else {
28262                            others.push((vertex_id, result));
28263                        }
28264                    }
28265                    for (vertex_id, result) in arrays {
28266                        let effects = match self.plan_vertex_effects_with_computed_flush(
28267                            vertex_id,
28268                            result,
28269                            Some(&inflight),
28270                            &mut computed_writes,
28271                        ) {
28272                            Ok(effects) => effects,
28273                            Err(e) => {
28274                                self.flush_computed_write_buffer(&mut computed_writes)?;
28275                                return Err(e);
28276                            }
28277                        };
28278                        for effect in &effects {
28279                            if let Err(e) = self.apply_effect_with_computed_writes(
28280                                effect,
28281                                Some(delta),
28282                                None,
28283                                Some(&mut computed_writes),
28284                            ) {
28285                                self.flush_computed_write_buffer(&mut computed_writes)?;
28286                                return Err(e);
28287                            }
28288                        }
28289                        applied = applied.saturating_add(1);
28290                    }
28291                    self.flush_computed_write_buffer(&mut computed_writes)?;
28292                    for (vertex_id, result) in others {
28293                        let effects = match self.plan_vertex_effects_with_computed_flush(
28294                            vertex_id,
28295                            result,
28296                            Some(&inflight),
28297                            &mut computed_writes,
28298                        ) {
28299                            Ok(effects) => effects,
28300                            Err(e) => {
28301                                self.flush_computed_write_buffer(&mut computed_writes)?;
28302                                return Err(e);
28303                            }
28304                        };
28305                        for effect in &effects {
28306                            if let Err(e) = self.apply_effect_with_computed_writes(
28307                                effect,
28308                                Some(delta),
28309                                None,
28310                                Some(&mut computed_writes),
28311                            ) {
28312                                self.flush_computed_write_buffer(&mut computed_writes)?;
28313                                return Err(e);
28314                            }
28315                        }
28316                        applied = applied.saturating_add(1);
28317                    }
28318                    self.flush_computed_write_buffer(&mut computed_writes)?;
28319                }
28320                Err(e) => {
28321                    self.flush_computed_write_buffer(&mut computed_writes)?;
28322                    return Err(e);
28323                }
28324            }
28325        }
28326
28327        Ok(applied)
28328    }
28329
28330    /// Evaluate a layer in parallel with cancellation support via effects pipeline.
28331    fn evaluate_layer_parallel_cancellable_effects(
28332        &mut self,
28333        layer: &super::scheduler::Layer,
28334        cancel_flag: &AtomicBool,
28335    ) -> Result<usize, ExcelError> {
28336        use rayon::prelude::*;
28337
28338        let thread_pool = self.thread_pool.as_ref().unwrap().clone();
28339
28340        if cancel_flag.load(Ordering::Relaxed) {
28341            return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28342                .with_message("Parallel evaluation cancelled before starting".to_string()));
28343        }
28344
28345        let mut phase1: Vec<VertexId> = Vec::new();
28346        let mut phase2: Vec<VertexId> = Vec::new();
28347        for &vid in &layer.vertices {
28348            if self.graph.get_range_dependencies(vid).is_some() {
28349                phase2.push(vid);
28350            } else {
28351                phase1.push(vid);
28352            }
28353        }
28354
28355        let inflight: rustc_hash::FxHashSet<VertexId> = layer.vertices.iter().copied().collect();
28356        let mut applied = 0usize;
28357
28358        for group in [&phase1[..], &phase2[..]] {
28359            if group.is_empty() {
28360                continue;
28361            }
28362            let mut computed_writes = ComputedWriteBuffer::default();
28363
28364            let results: Result<Vec<(VertexId, LiteralValue)>, ExcelError> =
28365                thread_pool.install(|| {
28366                    group
28367                        .par_iter()
28368                        .map(|&vertex_id| {
28369                            if cancel_flag.load(Ordering::Relaxed) {
28370                                return Err(ExcelError::new(ExcelErrorKind::Cancelled)
28371                                    .with_message(
28372                                        "Parallel evaluation cancelled during execution"
28373                                            .to_string(),
28374                                    ));
28375                            }
28376                            match self.evaluate_vertex_immutable(vertex_id) {
28377                                Ok(v) => Ok((vertex_id, v)),
28378                                Err(e) => Ok((vertex_id, LiteralValue::Error(e))),
28379                            }
28380                        })
28381                        .collect()
28382                });
28383
28384            match results {
28385                Ok(vertex_results) => {
28386                    let mut arrays: Vec<(VertexId, LiteralValue)> = Vec::new();
28387                    let mut others: Vec<(VertexId, LiteralValue)> = Vec::new();
28388                    for (vertex_id, result) in vertex_results {
28389                        if matches!(result, LiteralValue::Array(_)) {
28390                            arrays.push((vertex_id, result));
28391                        } else {
28392                            others.push((vertex_id, result));
28393                        }
28394                    }
28395                    for (vertex_id, result) in arrays {
28396                        let effects = match self.plan_vertex_effects_with_computed_flush(
28397                            vertex_id,
28398                            result,
28399                            Some(&inflight),
28400                            &mut computed_writes,
28401                        ) {
28402                            Ok(effects) => effects,
28403                            Err(e) => {
28404                                self.flush_computed_write_buffer(&mut computed_writes)?;
28405                                return Err(e);
28406                            }
28407                        };
28408                        for effect in &effects {
28409                            if let Err(e) = self.apply_effect_with_computed_writes(
28410                                effect,
28411                                None,
28412                                None,
28413                                Some(&mut computed_writes),
28414                            ) {
28415                                self.flush_computed_write_buffer(&mut computed_writes)?;
28416                                return Err(e);
28417                            }
28418                        }
28419                        applied = applied.saturating_add(1);
28420                    }
28421                    self.flush_computed_write_buffer(&mut computed_writes)?;
28422                    for (vertex_id, result) in others {
28423                        let effects = match self.plan_vertex_effects_with_computed_flush(
28424                            vertex_id,
28425                            result,
28426                            Some(&inflight),
28427                            &mut computed_writes,
28428                        ) {
28429                            Ok(effects) => effects,
28430                            Err(e) => {
28431                                self.flush_computed_write_buffer(&mut computed_writes)?;
28432                                return Err(e);
28433                            }
28434                        };
28435                        for effect in &effects {
28436                            if let Err(e) = self.apply_effect_with_computed_writes(
28437                                effect,
28438                                None,
28439                                None,
28440                                Some(&mut computed_writes),
28441                            ) {
28442                                self.flush_computed_write_buffer(&mut computed_writes)?;
28443                                return Err(e);
28444                            }
28445                        }
28446                        applied = applied.saturating_add(1);
28447                    }
28448                    self.flush_computed_write_buffer(&mut computed_writes)?;
28449                }
28450                Err(e) => {
28451                    self.flush_computed_write_buffer(&mut computed_writes)?;
28452                    return Err(e);
28453                }
28454            }
28455        }
28456
28457        Ok(applied)
28458    }
28459
28460    // ── Top-level evaluate_all_logged ───────────────────────────────────────
28461
28462    /// Evaluate all dirty/volatile vertices, recording effects into a ChangeLog.
28463    ///
28464    /// This is the same flow as `evaluate_all` but threads a ChangeLog through
28465    /// every effect application so that spill commits/clears are captured.
28466    pub fn evaluate_all_logged(&mut self, log: &mut ChangeLog) -> Result<EvalResult, ExcelError> {
28467        self.observe_evaluation_resource_request(EvaluationRequestKind::FullLogged, |engine| {
28468            engine.evaluate_all_logged_unobserved(log)
28469        })
28470    }
28471
28472    fn evaluate_all_logged_unobserved(
28473        &mut self,
28474        log: &mut ChangeLog,
28475    ) -> Result<EvalResult, ExcelError> {
28476        self.observe_function_semantic_epoch()?;
28477        let _source_cache = self.source_cache_session();
28478        self.validate_deterministic_mode()?;
28479        if self.config.defer_graph_building {
28480            self.build_graph_all()?;
28481        }
28482        self.transition_off_mode_spans_to_legacy()?;
28483        self.begin_evaluation_request();
28484        if self.config.formula_plane_mode == FormulaPlaneMode::AuthoritativeExperimental
28485            && self.graph.formula_authority().active_span_count() > 0
28486        {
28487            return self.evaluate_authoritative_formula_plane_all();
28488        }
28489        self.reset_virtual_dep_telemetry_if_disabled();
28490        let start = crate::instant::FzInstant::now();
28491        let mut computed_vertices = 0;
28492        let mut cycle_errors = 0;
28493
28494        let mut replan_iterations = 0;
28495        const MAX_REPLAN: usize = 5;
28496        let mut telemetry = self
28497            .config
28498            .enable_virtual_dep_telemetry
28499            .then(|| self.start_virtual_dep_telemetry());
28500
28501        log.begin_compound(format!("evaluate_all(epoch={})", self.recalc_epoch));
28502
28503        let result = (|| -> Result<EvalResult, ExcelError> {
28504            loop {
28505                let to_evaluate = self.graph.get_evaluation_vertices();
28506                if to_evaluate.is_empty() {
28507                    if let Some(t) = telemetry.as_mut()
28508                        && t.bailout_reason.is_none()
28509                    {
28510                        t.bailout_reason = Some("no_work");
28511                    }
28512                    break;
28513                }
28514
28515                let (schedule, old_vdeps, meta) = self.create_evaluation_schedule(&to_evaluate)?;
28516                if let Some(t) = telemetry.as_mut() {
28517                    Self::accumulate_schedule_meta(t, &meta);
28518                }
28519
28520                // Walk units in condensation order: stamp cycles at their
28521                // position, evaluate layers with ChangeLog recording.
28522                for &unit in &schedule.units {
28523                    match unit {
28524                        ScheduleUnit::Cycle(i) => {
28525                            // Journal integration (design doc §4 last row): the
28526                            // ChangeLog in this path only records SpillClear /
28527                            // SpillCommit events; WriteCell effects are never
28528                            // logged (see `apply_write_cell`). Runtime SCC tasks
28529                            // write values directly and never spill (§7.9 stamps
28530                            // would-be anchors), and their spill *teardown* is the
28531                            // same unlogged `stamp_cycle_error` the Static path
28532                            // already uses here — so direct commits coexist with
28533                            // the journal cleanly, with identical semantics to
28534                            // Static. Pinned by `scc_runtime_cycles` tests.
28535                            if self.handle_cycle_unit(schedule.unit_cycle(i), None, None, None)? > 0
28536                            {
28537                                cycle_errors += 1;
28538                            }
28539                        }
28540                        ScheduleUnit::Layer(i) => {
28541                            computed_vertices +=
28542                                self.evaluate_layer_logged(schedule.unit_layer(i), log)?;
28543                        }
28544                    }
28545                }
28546
28547                let changed_vertices = self.changed_virtual_dep_vertices(&to_evaluate, &old_vdeps);
28548                if let Some(t) = telemetry.as_mut() {
28549                    t.changed_vdeps_total += changed_vertices.len();
28550                }
28551                self.resource_checkpoint(0)?;
28552                self.graph.clear_dirty_flags(&to_evaluate);
28553                for v in &changed_vertices {
28554                    self.graph.set_dirty(*v, true);
28555                }
28556
28557                if changed_vertices.is_empty() {
28558                    if let Some(t) = telemetry.as_mut() {
28559                        t.bailout_reason = Some("converged");
28560                    }
28561                    break;
28562                }
28563                if replan_iterations >= MAX_REPLAN {
28564                    if let Some(mut t) = telemetry.take() {
28565                        t.bailout_reason = Some("max_replan");
28566                        t.replan_iterations = replan_iterations;
28567                        self.last_virtual_dep_telemetry = t;
28568                    }
28569                    return Err(
28570                        self.replan_exhausted_error(MAX_REPLAN, "dynamic dependency evaluation")
28571                    );
28572                }
28573                replan_iterations += 1;
28574            }
28575
28576            if let Some(mut t) = telemetry {
28577                t.replan_iterations = replan_iterations;
28578                self.last_virtual_dep_telemetry = t;
28579            }
28580
28581            self.redirty_for_next_recalc();
28582            self.recalc_epoch = self.recalc_epoch.wrapping_add(1);
28583
28584            Ok(EvalResult {
28585                computed_vertices,
28586                cycle_errors,
28587                elapsed: start.elapsed(),
28588            })
28589        })();
28590        log.end_compound();
28591        result
28592    }
28593
28594    /// Evaluate a single layer with ChangeLog recording.
28595    fn evaluate_layer_logged(
28596        &mut self,
28597        layer: &super::scheduler::Layer,
28598        log: &mut ChangeLog,
28599    ) -> Result<usize, ExcelError> {
28600        self.resource_checkpoint(layer.vertices.len() as u64)?;
28601        let mut computed_writes = ComputedWriteBuffer::default();
28602        for &vertex_id in &layer.vertices {
28603            self.flush_before_range_dependent_vertex(vertex_id, &mut computed_writes)?;
28604            let value = match self.evaluate_vertex_immutable(vertex_id) {
28605                Ok(v) => v,
28606                Err(e) => LiteralValue::Error(e),
28607            };
28608            let effects = match self.plan_vertex_effects_with_computed_flush(
28609                vertex_id,
28610                value,
28611                None,
28612                &mut computed_writes,
28613            ) {
28614                Ok(effects) => effects,
28615                Err(e) => {
28616                    self.flush_computed_write_buffer(&mut computed_writes)?;
28617                    return Err(e);
28618                }
28619            };
28620            for effect in &effects {
28621                if let Err(e) = self.apply_effect_with_computed_writes(
28622                    effect,
28623                    None,
28624                    Some(log),
28625                    Some(&mut computed_writes),
28626                ) {
28627                    self.flush_computed_write_buffer(&mut computed_writes)?;
28628                    return Err(e);
28629                }
28630            }
28631        }
28632        self.flush_computed_write_buffer(&mut computed_writes)?;
28633        Ok(layer.vertices.len())
28634    }
28635}